From e52155bc4ba4cdba9875f4579ecf0b97afe9b498 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 28 Jun 2023 14:47:30 +0200 Subject: [PATCH 01/93] Introduce SharedAttributableData --- include/openPMD/backend/Attributable.hpp | 79 +++++++++++++++++------- include/openPMD/backend/Writable.hpp | 7 +++ src/Iteration.cpp | 6 +- src/Series.cpp | 10 +-- src/backend/Attributable.cpp | 11 +++- src/backend/Writable.cpp | 1 + 6 files changed, 84 insertions(+), 30 deletions(-) diff --git a/include/openPMD/backend/Attributable.hpp b/include/openPMD/backend/Attributable.hpp index 0f7b722ae5..05b140de32 100644 --- a/include/openPMD/backend/Attributable.hpp +++ b/include/openPMD/backend/Attributable.hpp @@ -56,18 +56,19 @@ namespace internal class IterationData; class SeriesData; - class AttributableData + class SharedAttributableData { friend class openPMD::Attributable; public: - AttributableData(); - AttributableData(AttributableData const &) = delete; - AttributableData(AttributableData &&) = delete; - virtual ~AttributableData() = default; + SharedAttributableData(AttributableData *); + SharedAttributableData(SharedAttributableData const &) = delete; + SharedAttributableData(SharedAttributableData &&) = delete; + virtual ~SharedAttributableData() = default; - AttributableData &operator=(AttributableData const &) = delete; - AttributableData &operator=(AttributableData &&) = delete; + SharedAttributableData & + operator=(SharedAttributableData const &) = delete; + SharedAttributableData &operator=(SharedAttributableData &&) = delete; using A_MAP = std::map; /** @@ -77,6 +78,46 @@ namespace internal */ Writable m_writable; + private: + /** + * The attributes defined by this Attributable. + */ + A_MAP m_attributes; + }; + + /* + * This is essentially a two-level pointer. + * + * 1. level: Our public API hands out handles to users that are (shared) + * pointers to an internal object (PIMPL). + * 2. level: Multiple internal objects might refer to the same item in an + * openPMD file, e.g. to the same backend object. + * So, the internal object for an Attributable is a shared pointer to the + * unique object identifying this item. + * + * Such sharing occurs in the CustomHierarchy class where multiple + * containers refer to the same group in the openPMD hierarchy + * (container of groups, of meshes, of particle species, of datasets). + * This might also become relevant for links as in HDF5 if we choose to + * implement them. + */ + + class AttributableData : public std::shared_ptr + { + friend class openPMD::Attributable; + + using SharedData_t = std::shared_ptr; + + public: + AttributableData(); + AttributableData(SharedAttributableData *); + AttributableData(AttributableData const &) = delete; + AttributableData(AttributableData &&) = delete; + virtual ~AttributableData() = default; + + AttributableData &operator=(AttributableData const &) = delete; + AttributableData &operator=(AttributableData &&) = delete; + template T asInternalCopyOf() { @@ -112,12 +153,6 @@ namespace internal std::shared_ptr(self, [](auto const *) {})); return res; } - - private: - /** - * The attributes defined by this Attributable. - */ - A_MAP m_attributes; }; template @@ -410,7 +445,7 @@ OPENPMD_protected } AbstractIOHandler const *IOHandler() const { - auto &opt = m_attri->m_writable.IOHandler; + auto &opt = writable().IOHandler; if (!opt || !opt->has_value()) { return nullptr; @@ -419,19 +454,19 @@ OPENPMD_protected } Writable *&parent() { - return m_attri->m_writable.parent; + return writable().parent; } Writable const *parent() const { - return m_attri->m_writable.parent; + return writable().parent; } Writable &writable() { - return m_attri->m_writable; + return (*m_attri)->m_writable; } Writable const &writable() const { - return m_attri->m_writable; + return (*m_attri)->m_writable; } inline void setData(std::shared_ptr attri) @@ -439,13 +474,13 @@ OPENPMD_protected m_attri = std::move(attri); } - inline internal::AttributableData &get() + inline internal::SharedAttributableData &get() { - return *m_attri; + return **m_attri; } - inline internal::AttributableData const &get() const + inline internal::SharedAttributableData const &get() const { - return *m_attri; + return **m_attri; } bool dirty() const diff --git a/include/openPMD/backend/Writable.hpp b/include/openPMD/backend/Writable.hpp index d0b8b4f3c7..7ca9f4c3bd 100644 --- a/include/openPMD/backend/Writable.hpp +++ b/include/openPMD/backend/Writable.hpp @@ -48,6 +48,7 @@ class Series; namespace internal { + class SharedAttributableData; class AttributableData; class SeriesData; } // namespace internal @@ -73,6 +74,7 @@ namespace debug */ class Writable final { + friend class internal::SharedAttributableData; friend class internal::AttributableData; friend class internal::SeriesData; friend class Attributable; @@ -140,6 +142,11 @@ OPENPMD_private */ std::shared_ptr>> IOHandler = nullptr; + /* + * Link to the containing Attributable. + * If multiple Attributables share the same Writable, then the creating one. + * (See SharedAttributableData) + */ internal::AttributableData *attributable = nullptr; Writable *parent = nullptr; diff --git a/src/Iteration.cpp b/src/Iteration.cpp index 366fea0de1..93318fdeb0 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -27,6 +27,7 @@ #include "openPMD/auxiliary/DerefDynamicCast.hpp" #include "openPMD/auxiliary/Filesystem.hpp" #include "openPMD/auxiliary/StringManip.hpp" +#include "openPMD/backend/Attributable.hpp" #include "openPMD/backend/Writable.hpp" #include @@ -708,7 +709,8 @@ auto Iteration::beginStep( case IE::fileBased: if (thisObject.has_value()) { - file = &static_cast(*thisObject).get(); + file = static_cast( + thisObject.value().m_attri.get()); } else { @@ -790,7 +792,7 @@ void Iteration::endStep() switch (series.iterationEncoding()) { case IE::fileBased: - file = &Attributable::get(); + file = m_attri.get(); break; case IE::groupBased: case IE::variableBased: diff --git a/src/Series.cpp b/src/Series.cpp index d587575b44..8202523367 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -1060,7 +1060,7 @@ void Series::initSeries( std::unique_ptr input) { auto &series = get(); - auto &writable = series.m_writable; + auto &writable = series->m_writable; /* * In Access modes READ_LINEAR and APPEND, the Series constructor might have @@ -2419,7 +2419,7 @@ AdvanceStatus Series::advance( // If the backend does not support steps, we cannot continue here param.isThisStepMandatory = true; } - IOTask task(&file.m_writable, param); + IOTask task(&file->m_writable, param); IOHandler()->enqueue(task); } @@ -2516,7 +2516,7 @@ AdvanceStatus Series::advance(AdvanceMode mode) // If the backend does not support steps, we cannot continue here param.isThisStepMandatory = true; } - IOTask task(&series.m_writable, param); + IOTask task(&series->m_writable, param); IOHandler()->enqueue(task); // We cannot call Series::flush now, since the IO handler is still filled @@ -2880,9 +2880,9 @@ namespace internal // This releases the openPMD hierarchy iterations.container().clear(); // Release the IO Handler - if (m_writable.IOHandler) + if (operator*().m_writable.IOHandler) { - *m_writable.IOHandler = std::nullopt; + *operator*().m_writable.IOHandler = std::nullopt; } } } // namespace internal diff --git a/src/backend/Attributable.cpp b/src/backend/Attributable.cpp index d5ff005389..5d296a28f5 100644 --- a/src/backend/Attributable.cpp +++ b/src/backend/Attributable.cpp @@ -38,7 +38,16 @@ namespace openPMD { namespace internal { - AttributableData::AttributableData() : m_writable{this} + SharedAttributableData::SharedAttributableData(AttributableData *attr) + : m_writable{attr} + {} + + AttributableData::AttributableData() + : SharedData_t(std::make_shared(this)) + {} + + AttributableData::AttributableData(SharedAttributableData *raw_ptr) + : SharedData_t({raw_ptr, [](auto const *) {}}) {} } // namespace internal diff --git a/src/backend/Writable.cpp b/src/backend/Writable.cpp index 0e399a3a81..c012051d9e 100644 --- a/src/backend/Writable.cpp +++ b/src/backend/Writable.cpp @@ -21,6 +21,7 @@ #include "openPMD/backend/Writable.hpp" #include "openPMD/Series.hpp" #include "openPMD/auxiliary/DerefDynamicCast.hpp" +#include "openPMD/backend/Attributable.hpp" namespace openPMD { From 9e31b09da9abefbd8ceaa0faf39faf7d32977e7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 25 Jan 2024 17:03:57 +0100 Subject: [PATCH 02/93] Add AbstractSeriesIterator --- CMakeLists.txt | 1 + include/openPMD/Iteration.hpp | 4 + include/openPMD/SeriesIterator.hpp | 82 ++++++++++++++++ src/SeriesIterator.cpp | 153 +++++++++++++++++++++++++++++ 4 files changed, 240 insertions(+) create mode 100644 include/openPMD/SeriesIterator.hpp create mode 100644 src/SeriesIterator.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index c17631c6cb..4b2f44975b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -463,6 +463,7 @@ set(CORE_SOURCE src/Record.cpp src/RecordComponent.cpp src/Series.cpp + src/SeriesIterator.cpp src/version.cpp src/WriteIterations.cpp src/auxiliary/Date.cpp diff --git a/include/openPMD/Iteration.hpp b/include/openPMD/Iteration.hpp index 52bf43293a..1d60185d5c 100644 --- a/include/openPMD/Iteration.hpp +++ b/include/openPMD/Iteration.hpp @@ -440,6 +440,10 @@ class IndexedIteration : public Iteration using index_t = Iteration::IterationIndex_t; index_t const iterationIndex; + inline IndexedIteration(std::pair &&pair) + : IndexedIteration(std::move(pair.second), pair.first) + {} + private: template IndexedIteration(Iteration_t &&it, index_t index) diff --git a/include/openPMD/SeriesIterator.hpp b/include/openPMD/SeriesIterator.hpp new file mode 100644 index 0000000000..9cfb2d4be9 --- /dev/null +++ b/include/openPMD/SeriesIterator.hpp @@ -0,0 +1,82 @@ +/* Copyright 2024 Franz Poeschel + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ +#pragma once + +#include "openPMD/Iteration.hpp" + +namespace openPMD +{ +template +class AbstractSeriesIterator +{ +public: + using difference_type = Iteration::IterationIndex_t; + using value_type = Container::value_type; + + // dereference + virtual value_type const &operator*() const = 0; + virtual value_type &operator*(); + virtual value_type const *operator->() const; + virtual value_type *operator->(); + + // member access + virtual const value_type &operator[](difference_type) const; + virtual value_type &operator[](difference_type); + + // arithmetic random-access + virtual ChildClass operator+(difference_type) const = 0; + virtual ChildClass operator+(difference_type); + virtual ChildClass operator-(difference_type) const; + virtual ChildClass operator-(difference_type); + + // increment/decrement + virtual ChildClass &operator++(); + virtual ChildClass &operator--(); + virtual ChildClass operator++(int); + virtual ChildClass operator--(int); + + // comparison + virtual difference_type operator-(AbstractSeriesIterator const &) const = 0; + virtual bool operator==(ChildClass const &) const = 0; + virtual bool operator!=(ChildClass const &) const; + virtual bool operator<(ChildClass const &) const = 0; + virtual bool operator>(ChildClass const &) const; + virtual bool operator<=(ChildClass const &) const; + virtual bool operator>=(ChildClass const &) const; + +private: + ChildClass *this_child(); + ChildClass const *this_child() const; +}; + +template +ChildClass operator+( + Iteration::IterationIndex_t, AbstractSeriesIterator const &); +template +ChildClass +operator+(Iteration::IterationIndex_t, AbstractSeriesIterator &); +template +ChildClass operator-( + Iteration::IterationIndex_t, AbstractSeriesIterator const &); +template +ChildClass +operator-(Iteration::IterationIndex_t, AbstractSeriesIterator &); +} // namespace openPMD diff --git a/src/SeriesIterator.cpp b/src/SeriesIterator.cpp new file mode 100644 index 0000000000..5ac9a4ffeb --- /dev/null +++ b/src/SeriesIterator.cpp @@ -0,0 +1,153 @@ +#include "openPMD/SeriesIterator.hpp" +#include "openPMD/ReadIterations.hpp" + +namespace openPMD +{ +// dereference +template +auto AbstractSeriesIterator::operator*() -> value_type & +{ + return const_cast( + static_cast(this)->operator*()); +} +template +auto AbstractSeriesIterator::operator->() const + -> value_type const * +{ + return &operator*(); +} +template +auto AbstractSeriesIterator::operator->() -> value_type * +{ + return &operator*(); +} + +// member access +template +auto AbstractSeriesIterator::operator[](difference_type diff) const + -> value_type const & +{ + return *(*this + diff); +} +template +auto AbstractSeriesIterator::operator[](difference_type diff) + -> value_type & +{ + return *(*this + diff); +} + +// arithmetic random-access +template +ChildClass AbstractSeriesIterator::operator+(difference_type diff) +{ + static_cast(this)->operator+(diff); + return *this_child(); +} +template +ChildClass +AbstractSeriesIterator::operator-(difference_type diff) const +{ + return this->operator+(-diff); +} +template +ChildClass AbstractSeriesIterator::operator-(difference_type diff) +{ + return this->operator+(-diff); +} + +// increment/decrement +template +ChildClass &AbstractSeriesIterator::operator++() +{ + *this_child() = *this + 1; + return *this_child(); +} +template +ChildClass &AbstractSeriesIterator::operator--() +{ + *this_child() = *this - 1; + return *this_child(); +} +template +ChildClass AbstractSeriesIterator::operator++(int) +{ + auto prev = *this_child(); + operator++(); + return prev; +} +template +ChildClass AbstractSeriesIterator::operator--(int) +{ + auto prev = *this_child(); + operator--(); + return prev; +} + +// comparison +template +bool AbstractSeriesIterator::operator!=( + ChildClass const &other) const +{ + return !operator==(other); +} +template +bool AbstractSeriesIterator::operator>( + ChildClass const &other) const +{ + return other.operator<(*this_child()); +} +template +bool AbstractSeriesIterator::operator<=( + ChildClass const &other) const +{ + return !operator>(other); +} +template +bool AbstractSeriesIterator::operator>=( + ChildClass const &other) const +{ + return !operator<(other); +} + +// helpers +template +ChildClass *AbstractSeriesIterator::this_child() +{ + return static_cast(this); +} +template +ChildClass const *AbstractSeriesIterator::this_child() const +{ + return static_cast(this); +} + +// out-of-line arithmetic operators +template +ChildClass operator+( + Iteration::IterationIndex_t index, + AbstractSeriesIterator const &iterator) +{ + return iterator.operator+(index); +} +template +ChildClass operator+( + Iteration::IterationIndex_t index, + AbstractSeriesIterator &iterator) +{ + return iterator.operator+(index); +} +template +ChildClass operator-( + Iteration::IterationIndex_t index, + AbstractSeriesIterator const &iterator) +{ + return iterator.operator-(index); +} +template +ChildClass operator-( + Iteration::IterationIndex_t index, + AbstractSeriesIterator &iterator) +{ + return iterator.operator-(index); +} +} // namespace openPMD From 16fe71c86b0d6a6053aa172f3a02a2dd1334f061 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 25 Jan 2024 18:32:13 +0100 Subject: [PATCH 03/93] Derive SeriesIterator from AbstractSeriesIterator --- include/openPMD/Iteration.hpp | 9 ++- include/openPMD/ReadIterations.hpp | 94 ++++++++++++++++++++++++++++-- src/ReadIterations.cpp | 20 +++---- src/SeriesIterator.cpp | 15 +++++ src/binding/python/Series.cpp | 11 ++-- 5 files changed, 123 insertions(+), 26 deletions(-) diff --git a/include/openPMD/Iteration.hpp b/include/openPMD/Iteration.hpp index 1d60185d5c..02166892f3 100644 --- a/include/openPMD/Iteration.hpp +++ b/include/openPMD/Iteration.hpp @@ -136,7 +136,9 @@ class Iteration : public Attributable public: Iteration(Iteration const &) = default; + Iteration(Iteration &&) = default; Iteration &operator=(Iteration const &) = default; + Iteration &operator=(Iteration &&) = default; using IterationIndex_t = uint64_t; @@ -435,16 +437,17 @@ class IndexedIteration : public Iteration { friend class SeriesIterator; friend class WriteIterations; + friend class LegacyIteratorAdaptor; public: using index_t = Iteration::IterationIndex_t; index_t const iterationIndex; - inline IndexedIteration(std::pair &&pair) - : IndexedIteration(std::move(pair.second), pair.first) +private: + inline IndexedIteration(std::pair pair) + : Iteration(std::move(pair.second)), iterationIndex(pair.first) {} -private: template IndexedIteration(Iteration_t &&it, index_t index) : Iteration(std::forward(it)), iterationIndex(index) diff --git a/include/openPMD/ReadIterations.hpp b/include/openPMD/ReadIterations.hpp index 35f2f740ce..b4eaad93a6 100644 --- a/include/openPMD/ReadIterations.hpp +++ b/include/openPMD/ReadIterations.hpp @@ -20,8 +20,10 @@ */ #pragma once +#include "openPMD/Error.hpp" #include "openPMD/Iteration.hpp" #include "openPMD/Series.hpp" +#include "openPMD/SeriesIterator.hpp" #include "openPMD/backend/ParsePreference.hpp" #include @@ -31,7 +33,7 @@ namespace openPMD { -class SeriesIterator +class SeriesIterator : public AbstractSeriesIterator { using iteration_index_t = IndexedIteration::index_t; @@ -72,6 +74,8 @@ class SeriesIterator return m_data->value(); } + using parent_t = AbstractSeriesIterator; + public: //! construct the end() iterator explicit SeriesIterator(); @@ -80,13 +84,51 @@ class SeriesIterator Series const &, std::optional const &parsePreference); - SeriesIterator &operator++(); + // dereference + using parent_t::operator*; + value_type const &operator*() const override; - IndexedIteration operator*(); + // arithmetic random-access + using parent_t::operator-; + using parent_t::operator+; + SeriesIterator operator+(difference_type) const override + { + throw error::WrongAPIUsage( + "Random-access in global stateful iterator is inherently " + "non-const."); + } + SeriesIterator operator+(difference_type) override + { + throw error::WrongAPIUsage( + "Global stateful iterator supports no random access (yet)."); + } - bool operator==(SeriesIterator const &other) const; + // increment/decrement + SeriesIterator &operator++() override; + using parent_t::operator--; + inline SeriesIterator &operator--() override + { + throw error::WrongAPIUsage( + "Global stateful iterator supports no decrement (yet)."); + } + SeriesIterator operator++(int) override + { + throw error::WrongAPIUsage( + "Global stateful iterator supports no post-increment."); + } - bool operator!=(SeriesIterator const &other) const; + // comparison + difference_type operator-(AbstractSeriesIterator const &) const override + { + throw error::WrongAPIUsage( + "Global stateful iterator supports no relative comparison."); + } + bool operator==(SeriesIterator const &other) const override; + inline bool operator<(SeriesIterator const &) const override + { + throw error::WrongAPIUsage( + "Global stateful iterator supports no relative comparison."); + } static SeriesIterator end(); @@ -142,6 +184,46 @@ class SeriesIterator void close(); }; +class LegacyIteratorAdaptor +{ + using value_type = IndexedIteration; + using parent_t = SeriesIterator; + +private: + friend class ReadIterations; + SeriesIterator m_iterator; + LegacyIteratorAdaptor(SeriesIterator iterator) + : m_iterator(std::move(iterator)) + {} + +public: + inline value_type operator*() const + { + return m_iterator.operator*(); + } + + inline LegacyIteratorAdaptor &operator++() + { + ++m_iterator; + return *this; + } + + static LegacyIteratorAdaptor end() + { + return SeriesIterator::end(); + } + + inline bool operator==(LegacyIteratorAdaptor const &other) const + { + return m_iterator == other.m_iterator; + }; + + inline bool operator!=(LegacyIteratorAdaptor const &other) const + { + return m_iterator != other.m_iterator; + }; +}; + /** * @brief Reading side of the streaming API. * @@ -164,7 +246,7 @@ class ReadIterations private: using iterations_t = decltype(internal::SeriesData::iterations); - using iterator_t = SeriesIterator; + using iterator_t = LegacyIteratorAdaptor; Series m_series; std::optional m_parsePreference; diff --git a/src/ReadIterations.cpp b/src/ReadIterations.cpp index fff398c8cd..02442a78fd 100644 --- a/src/ReadIterations.cpp +++ b/src/ReadIterations.cpp @@ -597,12 +597,13 @@ SeriesIterator &SeriesIterator::operator++() return *resvalue; } -IndexedIteration SeriesIterator::operator*() +auto SeriesIterator::operator*() const -> value_type const & { auto &data = get(); - return IndexedIteration( - data.series.value().iterations[data.currentIteration], - data.currentIteration); + auto iterator = static_cast( + data.series.value().iterations) + .find(data.currentIteration); + return iterator.operator*(); } bool SeriesIterator::operator==(SeriesIterator const &other) const @@ -615,11 +616,6 @@ bool SeriesIterator::operator==(SeriesIterator const &other) const (!this->m_data->has_value() && !other.m_data->has_value()); } -bool SeriesIterator::operator!=(SeriesIterator const &other) const -{ - return !operator==(other); -} - SeriesIterator SeriesIterator::end() { return SeriesIterator{}; @@ -636,7 +632,7 @@ ReadIterations::ReadIterations( { // Open the iterator now already, so that metadata may already be read data.m_sharedStatefulIterator = - std::make_unique(m_series, m_parsePreference); + std::make_unique(m_series, m_parsePreference); } } @@ -646,13 +642,13 @@ ReadIterations::iterator_t ReadIterations::begin() if (!series.m_sharedStatefulIterator) { series.m_sharedStatefulIterator = - std::make_unique(m_series, m_parsePreference); + std::make_unique(m_series, m_parsePreference); } return *series.m_sharedStatefulIterator; } ReadIterations::iterator_t ReadIterations::end() { - return SeriesIterator::end(); + return iterator_t::end(); } } // namespace openPMD diff --git a/src/SeriesIterator.cpp b/src/SeriesIterator.cpp index 5ac9a4ffeb..9f24dbb659 100644 --- a/src/SeriesIterator.cpp +++ b/src/SeriesIterator.cpp @@ -150,4 +150,19 @@ ChildClass operator-( { return iterator.operator-(index); } + +#define OPENPMD_INSTANTIATE(type) \ + template class AbstractSeriesIterator; \ + template auto operator+( \ + Iteration::IterationIndex_t, AbstractSeriesIterator const &) \ + -> type; \ + template auto operator+( \ + Iteration::IterationIndex_t, AbstractSeriesIterator &) -> type; \ + template auto operator-( \ + Iteration::IterationIndex_t, AbstractSeriesIterator const &) \ + -> type; \ + template auto operator-( \ + Iteration::IterationIndex_t, AbstractSeriesIterator &) -> type; +OPENPMD_INSTANTIATE(SeriesIterator) +#undef OPENPMD_INSTANTIATE } // namespace openPMD diff --git a/src/binding/python/Series.cpp b/src/binding/python/Series.cpp index 37de823f2a..a7e53a415d 100644 --- a/src/binding/python/Series.cpp +++ b/src/binding/python/Series.cpp @@ -21,6 +21,7 @@ #include "openPMD/Series.hpp" #include "openPMD/IO/Access.hpp" #include "openPMD/IterationEncoding.hpp" +#include "openPMD/ReadIterations.hpp" #include "openPMD/auxiliary/JSON.hpp" #include "openPMD/binding/python/Pickle.hpp" #include "openPMD/config.hpp" @@ -37,10 +38,10 @@ #include #include -struct SeriesIteratorPythonAdaptor : SeriesIterator +struct SeriesIteratorPythonAdaptor : LegacyIteratorAdaptor { - SeriesIteratorPythonAdaptor(SeriesIterator it) - : SeriesIterator(std::move(it)) + SeriesIteratorPythonAdaptor(LegacyIteratorAdaptor it) + : LegacyIteratorAdaptor(std::move(it)) {} /* @@ -94,7 +95,7 @@ not possible once it has been closed. .def( "__next__", [](SeriesIteratorPythonAdaptor &iterator) { - if (iterator == SeriesIterator::end()) + if (iterator == LegacyIteratorAdaptor::end()) { throw py::stop_iteration(); } @@ -112,7 +113,7 @@ not possible once it has been closed. ++iterator; } iterator.first_iteration = false; - if (iterator == SeriesIterator::end()) + if (iterator == LegacyIteratorAdaptor::end()) { throw py::stop_iteration(); } From 8ceec083a762ceb520540d51d755b0b3505512ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 26 Jan 2024 10:52:42 +0100 Subject: [PATCH 04/93] Little fix --- src/SeriesIterator.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/SeriesIterator.cpp b/src/SeriesIterator.cpp index 9f24dbb659..7728eefb28 100644 --- a/src/SeriesIterator.cpp +++ b/src/SeriesIterator.cpp @@ -40,8 +40,7 @@ auto AbstractSeriesIterator::operator[](difference_type diff) template ChildClass AbstractSeriesIterator::operator+(difference_type diff) { - static_cast(this)->operator+(diff); - return *this_child(); + return static_cast(this)->operator+(diff); } template ChildClass From 8ba982092066e121a2827bfbeb961f6bed4fdf80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 26 Jan 2024 13:10:49 +0100 Subject: [PATCH 05/93] Introduce Snapshots.hpp --- CMakeLists.txt | 1 + include/openPMD/ReadIterations.hpp | 2 +- include/openPMD/SeriesIterator.hpp | 71 ++++++++++++- include/openPMD/Snapshots.hpp | 136 +++++++++++++++++++++++++ src/SeriesIterator.cpp | 156 ++++++++++++++++++++++++++--- src/Snapshots.cpp | 9 ++ 6 files changed, 355 insertions(+), 20 deletions(-) create mode 100644 include/openPMD/Snapshots.hpp create mode 100644 src/Snapshots.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 4b2f44975b..ca8f0301f7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -464,6 +464,7 @@ set(CORE_SOURCE src/RecordComponent.cpp src/Series.cpp src/SeriesIterator.cpp + src/Snapshots.cpp src/version.cpp src/WriteIterations.cpp src/auxiliary/Date.cpp diff --git a/include/openPMD/ReadIterations.hpp b/include/openPMD/ReadIterations.hpp index b4eaad93a6..278d2b8c7b 100644 --- a/include/openPMD/ReadIterations.hpp +++ b/include/openPMD/ReadIterations.hpp @@ -118,7 +118,7 @@ class SeriesIterator : public AbstractSeriesIterator } // comparison - difference_type operator-(AbstractSeriesIterator const &) const override + difference_type operator-(SeriesIterator const &) const override { throw error::WrongAPIUsage( "Global stateful iterator supports no relative comparison."); diff --git a/include/openPMD/SeriesIterator.hpp b/include/openPMD/SeriesIterator.hpp index 9cfb2d4be9..73b7c9e6dd 100644 --- a/include/openPMD/SeriesIterator.hpp +++ b/include/openPMD/SeriesIterator.hpp @@ -21,11 +21,11 @@ #pragma once #include "openPMD/Iteration.hpp" +#include namespace openPMD { -template -class AbstractSeriesIterator +class DynamicSeriesIterator { public: using difference_type = Iteration::IterationIndex_t; @@ -38,9 +38,41 @@ class AbstractSeriesIterator virtual value_type *operator->(); // member access - virtual const value_type &operator[](difference_type) const; + virtual value_type const &operator[](difference_type) const; virtual value_type &operator[](difference_type); +protected: + friend class OpaqueSeriesIterator; + // arithmetic random-access + virtual std::unique_ptr + plus_operator(difference_type) const = 0; + virtual std::unique_ptr + plus_operator(difference_type); + virtual std::unique_ptr + minus_operator(difference_type) const; + virtual std::unique_ptr + minus_operator(difference_type); + + // increment/decrement + virtual DynamicSeriesIterator &increment_operator(); + virtual DynamicSeriesIterator &decrement_operator(); + + // comparison + virtual difference_type + difference_operator(DynamicSeriesIterator const &) const = 0; + virtual bool equality_operator(DynamicSeriesIterator const &) const = 0; + virtual bool less_than_operator(DynamicSeriesIterator const &) const = 0; + + std::unique_ptr clone() const; +}; + +template +class AbstractSeriesIterator : public DynamicSeriesIterator +{ +public: + using difference_type = Iteration::IterationIndex_t; + using value_type = Container::value_type; + // arithmetic random-access virtual ChildClass operator+(difference_type) const = 0; virtual ChildClass operator+(difference_type); @@ -54,7 +86,7 @@ class AbstractSeriesIterator virtual ChildClass operator--(int); // comparison - virtual difference_type operator-(AbstractSeriesIterator const &) const = 0; + virtual difference_type operator-(ChildClass const &) const = 0; virtual bool operator==(ChildClass const &) const = 0; virtual bool operator!=(ChildClass const &) const; virtual bool operator<(ChildClass const &) const = 0; @@ -62,6 +94,37 @@ class AbstractSeriesIterator virtual bool operator<=(ChildClass const &) const; virtual bool operator>=(ChildClass const &) const; + /************* + * overrides * + *************/ + + // member access + value_type const &operator[](difference_type) const override; + value_type &operator[](difference_type) override; + +protected: + // arithmetic random-access + std::unique_ptr + plus_operator(difference_type) const override; + std::unique_ptr + plus_operator(difference_type) override; + std::unique_ptr + minus_operator(difference_type) const override; + std::unique_ptr + minus_operator(difference_type) override; + + // increment/decrement + DynamicSeriesIterator &increment_operator() override; + DynamicSeriesIterator &decrement_operator() override; + + // comparison + difference_type + difference_operator(DynamicSeriesIterator const &) const override; + bool equality_operator(DynamicSeriesIterator const &) const override; + bool less_than_operator(DynamicSeriesIterator const &) const override; + + std::unique_ptr clone() const; + private: ChildClass *this_child(); ChildClass const *this_child() const; diff --git a/include/openPMD/Snapshots.hpp b/include/openPMD/Snapshots.hpp new file mode 100644 index 0000000000..476eba397f --- /dev/null +++ b/include/openPMD/Snapshots.hpp @@ -0,0 +1,136 @@ +/* Copyright 2024 Franz Poeschel + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ +#pragma once + +#include "openPMD/SeriesIterator.hpp" +#include + +namespace openPMD +{ + +class OpaqueSeriesIterator : public AbstractSeriesIterator +{ +private: + // no shared_ptr since copied iterators should not share state + std::unique_ptr m_internal_iterator; + + OpaqueSeriesIterator( + std::unique_ptr internal_iterator) + : m_internal_iterator(std::move(internal_iterator)) + {} + +public: + OpaqueSeriesIterator(OpaqueSeriesIterator &other) + : m_internal_iterator(other.m_internal_iterator->clone()) + {} + OpaqueSeriesIterator(OpaqueSeriesIterator &&other) = default; + OpaqueSeriesIterator &operator=(OpaqueSeriesIterator &other) + { + m_internal_iterator = other.m_internal_iterator->clone(); + return *this; + } + OpaqueSeriesIterator &operator=(OpaqueSeriesIterator &&other) = default; + + ~OpaqueSeriesIterator() = default; + + // dereference + inline value_type const &operator*() const override + { + return **m_internal_iterator; + } + + // member access + inline value_type const &operator[](difference_type diff) const override + { + return m_internal_iterator->operator[](diff); + } + inline value_type &operator[](difference_type diff) override + { + return m_internal_iterator->operator[](diff); + } + + // arithmetic random-access + inline OpaqueSeriesIterator operator+(difference_type diff) const override + { + return OpaqueSeriesIterator(m_internal_iterator->plus_operator(diff)); + } + inline OpaqueSeriesIterator operator-(difference_type diff) const override + { + return OpaqueSeriesIterator(m_internal_iterator->minus_operator(diff)); + } + inline OpaqueSeriesIterator operator+(difference_type diff) override + { + return OpaqueSeriesIterator(m_internal_iterator->plus_operator(diff)); + } + inline OpaqueSeriesIterator operator-(difference_type diff) override + { + return OpaqueSeriesIterator(m_internal_iterator->minus_operator(diff)); + } + + // increment/decrement + OpaqueSeriesIterator &operator++() override + { + m_internal_iterator->increment_operator(); + return *this; + } + OpaqueSeriesIterator &operator--() override + { + m_internal_iterator->decrement_operator(); + return *this; + } + OpaqueSeriesIterator operator++(int) override + { + auto prev = *this; + ++(*this); + return prev; + } + OpaqueSeriesIterator operator--(int) override + { + auto prev = *this; + --(*this); + return prev; + } + + // comparison + inline difference_type + operator-(OpaqueSeriesIterator const &other) const override + { + return m_internal_iterator->difference_operator( + *other.m_internal_iterator); + } + + inline bool operator==(OpaqueSeriesIterator const &other) const override + { + return m_internal_iterator->equality_operator( + *other.m_internal_iterator); + } + inline bool operator<(OpaqueSeriesIterator const &other) const override + { + return m_internal_iterator->less_than_operator( + *other.m_internal_iterator); + } +}; + +class Snapshots +{ + OpaqueSeriesIterator begin(); +}; +} // namespace openPMD diff --git a/src/SeriesIterator.cpp b/src/SeriesIterator.cpp index 7728eefb28..6cee4ec872 100644 --- a/src/SeriesIterator.cpp +++ b/src/SeriesIterator.cpp @@ -1,39 +1,63 @@ #include "openPMD/SeriesIterator.hpp" #include "openPMD/ReadIterations.hpp" +#include namespace openPMD { // dereference -template -auto AbstractSeriesIterator::operator*() -> value_type & +auto DynamicSeriesIterator::operator*() -> value_type & { return const_cast( - static_cast(this)->operator*()); + static_cast(this)->operator*()); } -template -auto AbstractSeriesIterator::operator->() const - -> value_type const * +auto DynamicSeriesIterator::operator->() const -> value_type const * { return &operator*(); } -template -auto AbstractSeriesIterator::operator->() -> value_type * +auto DynamicSeriesIterator::operator->() -> value_type * { return &operator*(); } // member access -template -auto AbstractSeriesIterator::operator[](difference_type diff) const +auto DynamicSeriesIterator::operator[](difference_type diff) const -> value_type const & { - return *(*this + diff); + return **this->plus_operator(diff); } -template -auto AbstractSeriesIterator::operator[](difference_type diff) - -> value_type & +auto DynamicSeriesIterator::operator[](difference_type diff) -> value_type & { - return *(*this + diff); + return **this->plus_operator(diff); +} + +// arithmetic random-access +std::unique_ptr +DynamicSeriesIterator::plus_operator(difference_type diff) +{ + return static_cast(this)->plus_operator( + diff); +} +std::unique_ptr +DynamicSeriesIterator::minus_operator(difference_type diff) const +{ + return this->plus_operator(-diff); +} +std::unique_ptr +DynamicSeriesIterator::minus_operator(difference_type diff) +{ + return this->plus_operator(-diff); +} + +// increment/decrement +DynamicSeriesIterator &DynamicSeriesIterator::increment_operator() +{ + *this = *plus_operator(1); + return *this; +} +DynamicSeriesIterator &DynamicSeriesIterator::decrement_operator() +{ + *this = *minus_operator(1); + return *this; } // arithmetic random-access @@ -150,6 +174,108 @@ ChildClass operator-( return iterator.operator-(index); } +/************* + * overrides * + *************/ + +// member access +template +auto AbstractSeriesIterator::operator[](difference_type diff) const + -> value_type const & +{ + return *(*this + diff); +} +template +auto AbstractSeriesIterator::operator[](difference_type diff) + -> value_type & +{ + return *(*this + diff); +} + +// arithmetic random-access +template +std::unique_ptr +AbstractSeriesIterator::plus_operator(difference_type diff) const +{ + return std::unique_ptr{ + new ChildClass(operator+(diff))}; +} +template +std::unique_ptr +AbstractSeriesIterator::plus_operator(difference_type diff) +{ + return std::unique_ptr{ + new ChildClass(operator+(diff))}; +} +template +std::unique_ptr +AbstractSeriesIterator::minus_operator(difference_type diff) const +{ + return std::unique_ptr{ + new ChildClass(operator-(diff))}; +} +template +std::unique_ptr +AbstractSeriesIterator::minus_operator(difference_type diff) +{ + return std::unique_ptr{ + new ChildClass(operator-(diff))}; +} + +// increment/decrement +template +DynamicSeriesIterator &AbstractSeriesIterator::increment_operator() +{ + return ++*this; +} +template +DynamicSeriesIterator &AbstractSeriesIterator::decrement_operator() +{ + return --*this; +} + +// comparison +template +auto AbstractSeriesIterator::difference_operator( + DynamicSeriesIterator const &other) const -> difference_type +{ + return operator-(dynamic_cast(other)); +} +template +bool AbstractSeriesIterator::equality_operator( + DynamicSeriesIterator const &other) const +{ + if (auto child = dynamic_cast(&other); child) + { + return operator==(*child); + } + else + { + return false; // or throw error? + } +} +template +bool AbstractSeriesIterator::less_than_operator( + DynamicSeriesIterator const &other) const +{ + if (auto child = dynamic_cast(&other); child) + { + return operator<(*child); + } + else + { + return false; // or throw error? + } +} + +template +std::unique_ptr +AbstractSeriesIterator::clone() const +{ + return std::unique_ptr( + new ChildClass(*static_cast(this))); +} + #define OPENPMD_INSTANTIATE(type) \ template class AbstractSeriesIterator; \ template auto operator+( \ diff --git a/src/Snapshots.cpp b/src/Snapshots.cpp new file mode 100644 index 0000000000..5a8289ba98 --- /dev/null +++ b/src/Snapshots.cpp @@ -0,0 +1,9 @@ +#include "openPMD/Snapshots.hpp" + +namespace openPMD +{ +OpaqueSeriesIterator Snapshots::begin() +{ + // return OpaqueSeriesIterator(); +} +} // namespace openPMD From 37bd136f93bb373e7b49398bcec462ca1c5d1cec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 26 Jan 2024 13:39:41 +0100 Subject: [PATCH 06/93] Make AbstractSeriesIterator non-virtual --- include/openPMD/ReadIterations.hpp | 16 ++++---- include/openPMD/SeriesIterator.hpp | 50 ++++++++++++++---------- include/openPMD/Snapshots.hpp | 33 ++++++++-------- src/SeriesIterator.cpp | 62 +++++++++++++++--------------- src/Snapshots.cpp | 2 + 5 files changed, 87 insertions(+), 76 deletions(-) diff --git a/include/openPMD/ReadIterations.hpp b/include/openPMD/ReadIterations.hpp index 278d2b8c7b..fd03b91718 100644 --- a/include/openPMD/ReadIterations.hpp +++ b/include/openPMD/ReadIterations.hpp @@ -91,40 +91,40 @@ class SeriesIterator : public AbstractSeriesIterator // arithmetic random-access using parent_t::operator-; using parent_t::operator+; - SeriesIterator operator+(difference_type) const override + SeriesIterator operator+(difference_type) const { throw error::WrongAPIUsage( "Random-access in global stateful iterator is inherently " "non-const."); } - SeriesIterator operator+(difference_type) override + SeriesIterator operator+(difference_type) { throw error::WrongAPIUsage( "Global stateful iterator supports no random access (yet)."); } // increment/decrement - SeriesIterator &operator++() override; + SeriesIterator &operator++(); using parent_t::operator--; - inline SeriesIterator &operator--() override + inline SeriesIterator &operator--() { throw error::WrongAPIUsage( "Global stateful iterator supports no decrement (yet)."); } - SeriesIterator operator++(int) override + SeriesIterator operator++(int) { throw error::WrongAPIUsage( "Global stateful iterator supports no post-increment."); } // comparison - difference_type operator-(SeriesIterator const &) const override + difference_type operator-(SeriesIterator const &) const { throw error::WrongAPIUsage( "Global stateful iterator supports no relative comparison."); } - bool operator==(SeriesIterator const &other) const override; - inline bool operator<(SeriesIterator const &) const override + bool operator==(SeriesIterator const &other) const; + inline bool operator<(SeriesIterator const &) const { throw error::WrongAPIUsage( "Global stateful iterator supports no relative comparison."); diff --git a/include/openPMD/SeriesIterator.hpp b/include/openPMD/SeriesIterator.hpp index 73b7c9e6dd..3c3730fa56 100644 --- a/include/openPMD/SeriesIterator.hpp +++ b/include/openPMD/SeriesIterator.hpp @@ -25,6 +25,8 @@ namespace openPMD { +// Abstract class that can be used as an abstract interface to an opaque +// iterator implementation class DynamicSeriesIterator { public: @@ -38,8 +40,8 @@ class DynamicSeriesIterator virtual value_type *operator->(); // member access - virtual value_type const &operator[](difference_type) const; - virtual value_type &operator[](difference_type); + virtual value_type const &index_operator(difference_type) const; + virtual value_type &index_operator(difference_type); protected: friend class OpaqueSeriesIterator; @@ -66,6 +68,12 @@ class DynamicSeriesIterator std::unique_ptr clone() const; }; +// Class template with default method implementations for iterators. +// No virtual classes since there is no use. +// Commented methods must be implemented by child classes. +// Implement as `class MyIterator : public AbstractSeriesIterator` +// Use `using AbstractSeriesIterator::operator-` to pull default +// implementations. template class AbstractSeriesIterator : public DynamicSeriesIterator { @@ -74,34 +82,34 @@ class AbstractSeriesIterator : public DynamicSeriesIterator using value_type = Container::value_type; // arithmetic random-access - virtual ChildClass operator+(difference_type) const = 0; - virtual ChildClass operator+(difference_type); - virtual ChildClass operator-(difference_type) const; - virtual ChildClass operator-(difference_type); + // ChildClass operator+(difference_type) const = 0; + ChildClass operator+(difference_type); + ChildClass operator-(difference_type) const; + ChildClass operator-(difference_type); + + // member access + value_type const &operator[](difference_type) const; + value_type &operator[](difference_type); // increment/decrement - virtual ChildClass &operator++(); - virtual ChildClass &operator--(); - virtual ChildClass operator++(int); - virtual ChildClass operator--(int); + ChildClass &operator++(); + ChildClass &operator--(); + ChildClass operator++(int); + ChildClass operator--(int); // comparison - virtual difference_type operator-(ChildClass const &) const = 0; - virtual bool operator==(ChildClass const &) const = 0; - virtual bool operator!=(ChildClass const &) const; - virtual bool operator<(ChildClass const &) const = 0; - virtual bool operator>(ChildClass const &) const; - virtual bool operator<=(ChildClass const &) const; - virtual bool operator>=(ChildClass const &) const; + // difference_type operator-(ChildClass const &) const = 0; + // bool operator==(ChildClass const &) const = 0; + bool operator!=(ChildClass const &) const; + // bool operator<(ChildClass const &) const = 0; + bool operator>(ChildClass const &) const; + bool operator<=(ChildClass const &) const; + bool operator>=(ChildClass const &) const; /************* * overrides * *************/ - // member access - value_type const &operator[](difference_type) const override; - value_type &operator[](difference_type) override; - protected: // arithmetic random-access std::unique_ptr diff --git a/include/openPMD/Snapshots.hpp b/include/openPMD/Snapshots.hpp index 476eba397f..48f07aac1c 100644 --- a/include/openPMD/Snapshots.hpp +++ b/include/openPMD/Snapshots.hpp @@ -29,6 +29,7 @@ namespace openPMD class OpaqueSeriesIterator : public AbstractSeriesIterator { private: + using parent_t = AbstractSeriesIterator; // no shared_ptr since copied iterators should not share state std::unique_ptr m_internal_iterator; @@ -52,57 +53,58 @@ class OpaqueSeriesIterator : public AbstractSeriesIterator ~OpaqueSeriesIterator() = default; // dereference + using parent_t::operator*; inline value_type const &operator*() const override { return **m_internal_iterator; } // member access - inline value_type const &operator[](difference_type diff) const override + inline value_type const &operator[](difference_type diff) const { - return m_internal_iterator->operator[](diff); + return m_internal_iterator->index_operator(diff); } - inline value_type &operator[](difference_type diff) override + inline value_type &operator[](difference_type diff) { - return m_internal_iterator->operator[](diff); + return m_internal_iterator->index_operator(diff); } // arithmetic random-access - inline OpaqueSeriesIterator operator+(difference_type diff) const override + inline OpaqueSeriesIterator operator+(difference_type diff) const { return OpaqueSeriesIterator(m_internal_iterator->plus_operator(diff)); } - inline OpaqueSeriesIterator operator-(difference_type diff) const override + inline OpaqueSeriesIterator operator-(difference_type diff) const { return OpaqueSeriesIterator(m_internal_iterator->minus_operator(diff)); } - inline OpaqueSeriesIterator operator+(difference_type diff) override + inline OpaqueSeriesIterator operator+(difference_type diff) { return OpaqueSeriesIterator(m_internal_iterator->plus_operator(diff)); } - inline OpaqueSeriesIterator operator-(difference_type diff) override + inline OpaqueSeriesIterator operator-(difference_type diff) { return OpaqueSeriesIterator(m_internal_iterator->minus_operator(diff)); } // increment/decrement - OpaqueSeriesIterator &operator++() override + OpaqueSeriesIterator &operator++() { m_internal_iterator->increment_operator(); return *this; } - OpaqueSeriesIterator &operator--() override + OpaqueSeriesIterator &operator--() { m_internal_iterator->decrement_operator(); return *this; } - OpaqueSeriesIterator operator++(int) override + OpaqueSeriesIterator operator++(int) { auto prev = *this; ++(*this); return prev; } - OpaqueSeriesIterator operator--(int) override + OpaqueSeriesIterator operator--(int) { auto prev = *this; --(*this); @@ -110,19 +112,18 @@ class OpaqueSeriesIterator : public AbstractSeriesIterator } // comparison - inline difference_type - operator-(OpaqueSeriesIterator const &other) const override + inline difference_type operator-(OpaqueSeriesIterator const &other) const { return m_internal_iterator->difference_operator( *other.m_internal_iterator); } - inline bool operator==(OpaqueSeriesIterator const &other) const override + inline bool operator==(OpaqueSeriesIterator const &other) const { return m_internal_iterator->equality_operator( *other.m_internal_iterator); } - inline bool operator<(OpaqueSeriesIterator const &other) const override + inline bool operator<(OpaqueSeriesIterator const &other) const { return m_internal_iterator->less_than_operator( *other.m_internal_iterator); diff --git a/src/SeriesIterator.cpp b/src/SeriesIterator.cpp index 6cee4ec872..1f90b149a7 100644 --- a/src/SeriesIterator.cpp +++ b/src/SeriesIterator.cpp @@ -20,12 +20,12 @@ auto DynamicSeriesIterator::operator->() -> value_type * } // member access -auto DynamicSeriesIterator::operator[](difference_type diff) const +auto DynamicSeriesIterator::index_operator(difference_type diff) const -> value_type const & { return **this->plus_operator(diff); } -auto DynamicSeriesIterator::operator[](difference_type diff) -> value_type & +auto DynamicSeriesIterator::index_operator(difference_type diff) -> value_type & { return **this->plus_operator(diff); } @@ -64,18 +64,32 @@ DynamicSeriesIterator &DynamicSeriesIterator::decrement_operator() template ChildClass AbstractSeriesIterator::operator+(difference_type diff) { - return static_cast(this)->operator+(diff); + return static_cast(this)->operator+(diff); } template ChildClass AbstractSeriesIterator::operator-(difference_type diff) const { - return this->operator+(-diff); + return this_child()->operator+(-diff); } template ChildClass AbstractSeriesIterator::operator-(difference_type diff) { - return this->operator+(-diff); + return this_child()->operator+(-diff); +} + +// member access +template +auto AbstractSeriesIterator::operator[](difference_type diff) const + -> value_type const & +{ + return *(*this_child() + diff); +} +template +auto AbstractSeriesIterator::operator[](difference_type diff) + -> value_type & +{ + return *(*this_child() + diff); } // increment/decrement @@ -111,7 +125,7 @@ template bool AbstractSeriesIterator::operator!=( ChildClass const &other) const { - return !operator==(other); + return !this_child()->operator==(other); } template bool AbstractSeriesIterator::operator>( @@ -129,7 +143,7 @@ template bool AbstractSeriesIterator::operator>=( ChildClass const &other) const { - return !operator<(other); + return !this_child()->operator<(other); } // helpers @@ -150,7 +164,7 @@ ChildClass operator+( Iteration::IterationIndex_t index, AbstractSeriesIterator const &iterator) { - return iterator.operator+(index); + return static_cast(iterator).operator+(index); } template ChildClass operator+( @@ -178,60 +192,46 @@ ChildClass operator-( * overrides * *************/ -// member access -template -auto AbstractSeriesIterator::operator[](difference_type diff) const - -> value_type const & -{ - return *(*this + diff); -} -template -auto AbstractSeriesIterator::operator[](difference_type diff) - -> value_type & -{ - return *(*this + diff); -} - // arithmetic random-access template std::unique_ptr AbstractSeriesIterator::plus_operator(difference_type diff) const { return std::unique_ptr{ - new ChildClass(operator+(diff))}; + new ChildClass(this_child()->operator+(diff))}; } template std::unique_ptr AbstractSeriesIterator::plus_operator(difference_type diff) { return std::unique_ptr{ - new ChildClass(operator+(diff))}; + new ChildClass(this_child()->operator+(diff))}; } template std::unique_ptr AbstractSeriesIterator::minus_operator(difference_type diff) const { return std::unique_ptr{ - new ChildClass(operator-(diff))}; + new ChildClass(this_child()->operator-(diff))}; } template std::unique_ptr AbstractSeriesIterator::minus_operator(difference_type diff) { return std::unique_ptr{ - new ChildClass(operator-(diff))}; + new ChildClass(this_child()->operator-(diff))}; } // increment/decrement template DynamicSeriesIterator &AbstractSeriesIterator::increment_operator() { - return ++*this; + return ++*this_child(); } template DynamicSeriesIterator &AbstractSeriesIterator::decrement_operator() { - return --*this; + return --*this_child(); } // comparison @@ -239,7 +239,7 @@ template auto AbstractSeriesIterator::difference_operator( DynamicSeriesIterator const &other) const -> difference_type { - return operator-(dynamic_cast(other)); + return this_child()->operator-(dynamic_cast(other)); } template bool AbstractSeriesIterator::equality_operator( @@ -247,7 +247,7 @@ bool AbstractSeriesIterator::equality_operator( { if (auto child = dynamic_cast(&other); child) { - return operator==(*child); + return this_child()->operator==(*child); } else { @@ -260,7 +260,7 @@ bool AbstractSeriesIterator::less_than_operator( { if (auto child = dynamic_cast(&other); child) { - return operator<(*child); + return this_child()->operator<(*child); } else { diff --git a/src/Snapshots.cpp b/src/Snapshots.cpp index 5a8289ba98..bea53fa9c3 100644 --- a/src/Snapshots.cpp +++ b/src/Snapshots.cpp @@ -1,9 +1,11 @@ #include "openPMD/Snapshots.hpp" +#include namespace openPMD { OpaqueSeriesIterator Snapshots::begin() { // return OpaqueSeriesIterator(); + throw std::runtime_error("unimplemented"); } } // namespace openPMD From 8cbd84becb861d0f9cd0db11a988c4da2d3d39a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 26 Jan 2024 14:43:30 +0100 Subject: [PATCH 07/93] Working commit for Series::snapshots() --- include/openPMD/Series.hpp | 3 +++ include/openPMD/SeriesIterator.hpp | 4 ++-- include/openPMD/Snapshots.hpp | 28 +++++++++++++++++++++++++--- src/Series.cpp | 29 +++++++++++++++++++++++++++++ src/SeriesIterator.cpp | 2 ++ src/Snapshots.cpp | 10 +++++----- 6 files changed, 66 insertions(+), 10 deletions(-) diff --git a/include/openPMD/Series.hpp b/include/openPMD/Series.hpp index 04907eda40..708e77b63d 100644 --- a/include/openPMD/Series.hpp +++ b/include/openPMD/Series.hpp @@ -26,6 +26,7 @@ #include "openPMD/IO/Format.hpp" #include "openPMD/Iteration.hpp" #include "openPMD/IterationEncoding.hpp" +#include "openPMD/Snapshots.hpp" #include "openPMD/Streaming.hpp" #include "openPMD/WriteIterations.hpp" #include "openPMD/auxiliary/Variant.hpp" @@ -645,6 +646,8 @@ class Series : public Attributable */ ReadIterations readIterations(); + Snapshots snapshots(); + /** * @brief Parse the Series. * diff --git a/include/openPMD/SeriesIterator.hpp b/include/openPMD/SeriesIterator.hpp index 3c3730fa56..d954fbdacc 100644 --- a/include/openPMD/SeriesIterator.hpp +++ b/include/openPMD/SeriesIterator.hpp @@ -65,7 +65,7 @@ class DynamicSeriesIterator virtual bool equality_operator(DynamicSeriesIterator const &) const = 0; virtual bool less_than_operator(DynamicSeriesIterator const &) const = 0; - std::unique_ptr clone() const; + virtual std::unique_ptr clone() const = 0; }; // Class template with default method implementations for iterators. @@ -131,7 +131,7 @@ class AbstractSeriesIterator : public DynamicSeriesIterator bool equality_operator(DynamicSeriesIterator const &) const override; bool less_than_operator(DynamicSeriesIterator const &) const override; - std::unique_ptr clone() const; + std::unique_ptr clone() const override; private: ChildClass *this_child(); diff --git a/include/openPMD/Snapshots.hpp b/include/openPMD/Snapshots.hpp index 48f07aac1c..452172e9b1 100644 --- a/include/openPMD/Snapshots.hpp +++ b/include/openPMD/Snapshots.hpp @@ -21,6 +21,7 @@ #pragma once #include "openPMD/SeriesIterator.hpp" +#include #include namespace openPMD @@ -29,6 +30,7 @@ namespace openPMD class OpaqueSeriesIterator : public AbstractSeriesIterator { private: + friend class Series; using parent_t = AbstractSeriesIterator; // no shared_ptr since copied iterators should not share state std::unique_ptr m_internal_iterator; @@ -39,11 +41,11 @@ class OpaqueSeriesIterator : public AbstractSeriesIterator {} public: - OpaqueSeriesIterator(OpaqueSeriesIterator &other) + OpaqueSeriesIterator(OpaqueSeriesIterator const &other) : m_internal_iterator(other.m_internal_iterator->clone()) {} OpaqueSeriesIterator(OpaqueSeriesIterator &&other) = default; - OpaqueSeriesIterator &operator=(OpaqueSeriesIterator &other) + OpaqueSeriesIterator &operator=(OpaqueSeriesIterator const &other) { m_internal_iterator = other.m_internal_iterator->clone(); return *this; @@ -132,6 +134,26 @@ class OpaqueSeriesIterator : public AbstractSeriesIterator class Snapshots { - OpaqueSeriesIterator begin(); +private: + friend class Series; + + std::function m_begin; + std::function m_end; + + inline Snapshots( + std::function begin, + std::function end) + : m_begin(std::move(begin)), m_end(std::move(end)) + {} + +public: + inline OpaqueSeriesIterator begin() + { + return m_begin(); + } + inline OpaqueSeriesIterator end() + { + return m_end(); + } }; } // namespace openPMD diff --git a/src/Series.cpp b/src/Series.cpp index 8202523367..c8b187ddc1 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -30,6 +30,8 @@ #include "openPMD/IO/IOTask.hpp" #include "openPMD/IterationEncoding.hpp" #include "openPMD/ReadIterations.hpp" +#include "openPMD/SeriesIterator.hpp" +#include "openPMD/Snapshots.hpp" #include "openPMD/ThrowError.hpp" #include "openPMD/auxiliary/Date.hpp" #include "openPMD/auxiliary/Filesystem.hpp" @@ -2928,6 +2930,33 @@ ReadIterations Series::readIterations() std::move(res), IOHandler()->m_frontendAccess, get().m_parsePreference}; } +Snapshots Series::snapshots() +{ + // Use private constructor instead of copy constructor to avoid + // object slicing + Series copied_series; + copied_series.setData( + std::dynamic_pointer_cast(this->m_attri)); + auto begin = [s = std::move(copied_series)]() mutable { + auto &series = s.get(); + if (!series.m_sharedStatefulIterator) + { + auto parse_preference = series.m_parsePreference; + series.m_sharedStatefulIterator = std::make_unique( + std::move(s), parse_preference); + } + std::unique_ptr internal_iterator_cloned{ + new SeriesIterator(*series.m_sharedStatefulIterator)}; + return OpaqueSeriesIterator(std::move(internal_iterator_cloned)); + }; + auto end = []() mutable { + std::unique_ptr internal_iterator{ + new SeriesIterator()}; + return OpaqueSeriesIterator(std::move(internal_iterator)); + }; + return Snapshots(std::move(begin), std::move(end)); +} + void Series::parseBase() { readIterations(); diff --git a/src/SeriesIterator.cpp b/src/SeriesIterator.cpp index 1f90b149a7..6cddfa97ab 100644 --- a/src/SeriesIterator.cpp +++ b/src/SeriesIterator.cpp @@ -1,5 +1,6 @@ #include "openPMD/SeriesIterator.hpp" #include "openPMD/ReadIterations.hpp" +#include "openPMD/Snapshots.hpp" #include namespace openPMD @@ -289,5 +290,6 @@ AbstractSeriesIterator::clone() const template auto operator-( \ Iteration::IterationIndex_t, AbstractSeriesIterator &) -> type; OPENPMD_INSTANTIATE(SeriesIterator) +OPENPMD_INSTANTIATE(OpaqueSeriesIterator) #undef OPENPMD_INSTANTIATE } // namespace openPMD diff --git a/src/Snapshots.cpp b/src/Snapshots.cpp index bea53fa9c3..ec472d866c 100644 --- a/src/Snapshots.cpp +++ b/src/Snapshots.cpp @@ -3,9 +3,9 @@ namespace openPMD { -OpaqueSeriesIterator Snapshots::begin() -{ - // return OpaqueSeriesIterator(); - throw std::runtime_error("unimplemented"); -} +// OpaqueSeriesIterator Snapshots::begin() +// { +// // return OpaqueSeriesIterator(); +// throw std::runtime_error("unimplemented"); +// } } // namespace openPMD From 71065ad3354ed97d918db559b53eb4d85b2ad880 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 26 Jan 2024 15:49:53 +0100 Subject: [PATCH 08/93] No virtual operator[] --- include/openPMD/ReadIterations.hpp | 2 +- include/openPMD/SeriesIterator.hpp | 19 +++++++++---- include/openPMD/Snapshots.hpp | 4 +-- src/SeriesIterator.cpp | 45 +++++++++++++++++++++--------- 4 files changed, 49 insertions(+), 21 deletions(-) diff --git a/include/openPMD/ReadIterations.hpp b/include/openPMD/ReadIterations.hpp index fd03b91718..218a34f526 100644 --- a/include/openPMD/ReadIterations.hpp +++ b/include/openPMD/ReadIterations.hpp @@ -86,7 +86,7 @@ class SeriesIterator : public AbstractSeriesIterator // dereference using parent_t::operator*; - value_type const &operator*() const override; + value_type const &operator*() const; // arithmetic random-access using parent_t::operator-; diff --git a/include/openPMD/SeriesIterator.hpp b/include/openPMD/SeriesIterator.hpp index d954fbdacc..95fa70169f 100644 --- a/include/openPMD/SeriesIterator.hpp +++ b/include/openPMD/SeriesIterator.hpp @@ -33,17 +33,15 @@ class DynamicSeriesIterator using difference_type = Iteration::IterationIndex_t; using value_type = Container::value_type; +protected: // dereference - virtual value_type const &operator*() const = 0; - virtual value_type &operator*(); - virtual value_type const *operator->() const; - virtual value_type *operator->(); + virtual value_type const &dereference_operator() const = 0; + virtual value_type &dereference_operator(); // member access virtual value_type const &index_operator(difference_type) const; virtual value_type &index_operator(difference_type); -protected: friend class OpaqueSeriesIterator; // arithmetic random-access virtual std::unique_ptr @@ -81,6 +79,12 @@ class AbstractSeriesIterator : public DynamicSeriesIterator using difference_type = Iteration::IterationIndex_t; using value_type = Container::value_type; + // dereference + // value_type const &operator*() const = 0; + value_type &operator*(); + value_type const *operator->() const; + value_type *operator->(); + // arithmetic random-access // ChildClass operator+(difference_type) const = 0; ChildClass operator+(difference_type); @@ -111,6 +115,11 @@ class AbstractSeriesIterator : public DynamicSeriesIterator *************/ protected: + using parent_t = DynamicSeriesIterator; + // dereference + using parent_t::dereference_operator; + value_type const &dereference_operator() const override; + // arithmetic random-access std::unique_ptr plus_operator(difference_type) const override; diff --git a/include/openPMD/Snapshots.hpp b/include/openPMD/Snapshots.hpp index 452172e9b1..4965ce5753 100644 --- a/include/openPMD/Snapshots.hpp +++ b/include/openPMD/Snapshots.hpp @@ -56,9 +56,9 @@ class OpaqueSeriesIterator : public AbstractSeriesIterator // dereference using parent_t::operator*; - inline value_type const &operator*() const override + inline value_type const &operator*() const { - return **m_internal_iterator; + return m_internal_iterator->dereference_operator(); } // member access diff --git a/src/SeriesIterator.cpp b/src/SeriesIterator.cpp index 6cddfa97ab..643a4ad2a2 100644 --- a/src/SeriesIterator.cpp +++ b/src/SeriesIterator.cpp @@ -5,30 +5,22 @@ namespace openPMD { -// dereference -auto DynamicSeriesIterator::operator*() -> value_type & +auto DynamicSeriesIterator::dereference_operator() -> value_type & { return const_cast( - static_cast(this)->operator*()); -} -auto DynamicSeriesIterator::operator->() const -> value_type const * -{ - return &operator*(); -} -auto DynamicSeriesIterator::operator->() -> value_type * -{ - return &operator*(); + static_cast(this) + ->dereference_operator()); } // member access auto DynamicSeriesIterator::index_operator(difference_type diff) const -> value_type const & { - return **this->plus_operator(diff); + return this->plus_operator(diff)->dereference_operator(); } auto DynamicSeriesIterator::index_operator(difference_type diff) -> value_type & { - return **this->plus_operator(diff); + return this->plus_operator(diff)->dereference_operator(); } // arithmetic random-access @@ -61,6 +53,25 @@ DynamicSeriesIterator &DynamicSeriesIterator::decrement_operator() return *this; } +// dereference +template +auto AbstractSeriesIterator::operator*() -> value_type & +{ + return const_cast( + static_cast(this)->operator*()); +} +template +auto AbstractSeriesIterator::operator->() const + -> value_type const * +{ + return &this_child()->operator*(); +} +template +auto AbstractSeriesIterator::operator->() -> value_type * +{ + return &this_child()->operator*(); +} + // arithmetic random-access template ChildClass AbstractSeriesIterator::operator+(difference_type diff) @@ -193,6 +204,14 @@ ChildClass operator-( * overrides * *************/ +// dereference +template +auto AbstractSeriesIterator::dereference_operator() const + -> value_type const & +{ + return this_child()->operator*(); +} + // arithmetic random-access template std::unique_ptr From 52d8d3ec62850a2f0add14b8e4179d008748fad8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 26 Jan 2024 16:26:42 +0100 Subject: [PATCH 09/93] Remove random-accessing from iterator --- include/openPMD/ReadIterations.hpp | 15 --- include/openPMD/SeriesIterator.hpp | 49 ++------- include/openPMD/Snapshots.hpp | 34 ------ src/SeriesIterator.cpp | 159 +---------------------------- 4 files changed, 9 insertions(+), 248 deletions(-) diff --git a/include/openPMD/ReadIterations.hpp b/include/openPMD/ReadIterations.hpp index 218a34f526..50b9fdd25e 100644 --- a/include/openPMD/ReadIterations.hpp +++ b/include/openPMD/ReadIterations.hpp @@ -88,21 +88,6 @@ class SeriesIterator : public AbstractSeriesIterator using parent_t::operator*; value_type const &operator*() const; - // arithmetic random-access - using parent_t::operator-; - using parent_t::operator+; - SeriesIterator operator+(difference_type) const - { - throw error::WrongAPIUsage( - "Random-access in global stateful iterator is inherently " - "non-const."); - } - SeriesIterator operator+(difference_type) - { - throw error::WrongAPIUsage( - "Global stateful iterator supports no random access (yet)."); - } - // increment/decrement SeriesIterator &operator++(); using parent_t::operator--; diff --git a/include/openPMD/SeriesIterator.hpp b/include/openPMD/SeriesIterator.hpp index 95fa70169f..aebe7657e6 100644 --- a/include/openPMD/SeriesIterator.hpp +++ b/include/openPMD/SeriesIterator.hpp @@ -34,32 +34,17 @@ class DynamicSeriesIterator using value_type = Container::value_type; protected: + friend class OpaqueSeriesIterator; + // dereference virtual value_type const &dereference_operator() const = 0; virtual value_type &dereference_operator(); - // member access - virtual value_type const &index_operator(difference_type) const; - virtual value_type &index_operator(difference_type); - - friend class OpaqueSeriesIterator; - // arithmetic random-access - virtual std::unique_ptr - plus_operator(difference_type) const = 0; - virtual std::unique_ptr - plus_operator(difference_type); - virtual std::unique_ptr - minus_operator(difference_type) const; - virtual std::unique_ptr - minus_operator(difference_type); - // increment/decrement - virtual DynamicSeriesIterator &increment_operator(); - virtual DynamicSeriesIterator &decrement_operator(); + virtual DynamicSeriesIterator &increment_operator() = 0; + virtual DynamicSeriesIterator &decrement_operator() = 0; // comparison - virtual difference_type - difference_operator(DynamicSeriesIterator const &) const = 0; virtual bool equality_operator(DynamicSeriesIterator const &) const = 0; virtual bool less_than_operator(DynamicSeriesIterator const &) const = 0; @@ -85,19 +70,9 @@ class AbstractSeriesIterator : public DynamicSeriesIterator value_type const *operator->() const; value_type *operator->(); - // arithmetic random-access - // ChildClass operator+(difference_type) const = 0; - ChildClass operator+(difference_type); - ChildClass operator-(difference_type) const; - ChildClass operator-(difference_type); - - // member access - value_type const &operator[](difference_type) const; - value_type &operator[](difference_type); - // increment/decrement - ChildClass &operator++(); - ChildClass &operator--(); + // ChildClass &operator++(); + // ChildClass &operator--(); ChildClass operator++(int); ChildClass operator--(int); @@ -120,23 +95,11 @@ class AbstractSeriesIterator : public DynamicSeriesIterator using parent_t::dereference_operator; value_type const &dereference_operator() const override; - // arithmetic random-access - std::unique_ptr - plus_operator(difference_type) const override; - std::unique_ptr - plus_operator(difference_type) override; - std::unique_ptr - minus_operator(difference_type) const override; - std::unique_ptr - minus_operator(difference_type) override; - // increment/decrement DynamicSeriesIterator &increment_operator() override; DynamicSeriesIterator &decrement_operator() override; // comparison - difference_type - difference_operator(DynamicSeriesIterator const &) const override; bool equality_operator(DynamicSeriesIterator const &) const override; bool less_than_operator(DynamicSeriesIterator const &) const override; diff --git a/include/openPMD/Snapshots.hpp b/include/openPMD/Snapshots.hpp index 4965ce5753..cef52f8ca9 100644 --- a/include/openPMD/Snapshots.hpp +++ b/include/openPMD/Snapshots.hpp @@ -61,34 +61,6 @@ class OpaqueSeriesIterator : public AbstractSeriesIterator return m_internal_iterator->dereference_operator(); } - // member access - inline value_type const &operator[](difference_type diff) const - { - return m_internal_iterator->index_operator(diff); - } - inline value_type &operator[](difference_type diff) - { - return m_internal_iterator->index_operator(diff); - } - - // arithmetic random-access - inline OpaqueSeriesIterator operator+(difference_type diff) const - { - return OpaqueSeriesIterator(m_internal_iterator->plus_operator(diff)); - } - inline OpaqueSeriesIterator operator-(difference_type diff) const - { - return OpaqueSeriesIterator(m_internal_iterator->minus_operator(diff)); - } - inline OpaqueSeriesIterator operator+(difference_type diff) - { - return OpaqueSeriesIterator(m_internal_iterator->plus_operator(diff)); - } - inline OpaqueSeriesIterator operator-(difference_type diff) - { - return OpaqueSeriesIterator(m_internal_iterator->minus_operator(diff)); - } - // increment/decrement OpaqueSeriesIterator &operator++() { @@ -114,12 +86,6 @@ class OpaqueSeriesIterator : public AbstractSeriesIterator } // comparison - inline difference_type operator-(OpaqueSeriesIterator const &other) const - { - return m_internal_iterator->difference_operator( - *other.m_internal_iterator); - } - inline bool operator==(OpaqueSeriesIterator const &other) const { return m_internal_iterator->equality_operator( diff --git a/src/SeriesIterator.cpp b/src/SeriesIterator.cpp index 643a4ad2a2..8650c93f3a 100644 --- a/src/SeriesIterator.cpp +++ b/src/SeriesIterator.cpp @@ -12,47 +12,6 @@ auto DynamicSeriesIterator::dereference_operator() -> value_type & ->dereference_operator()); } -// member access -auto DynamicSeriesIterator::index_operator(difference_type diff) const - -> value_type const & -{ - return this->plus_operator(diff)->dereference_operator(); -} -auto DynamicSeriesIterator::index_operator(difference_type diff) -> value_type & -{ - return this->plus_operator(diff)->dereference_operator(); -} - -// arithmetic random-access -std::unique_ptr -DynamicSeriesIterator::plus_operator(difference_type diff) -{ - return static_cast(this)->plus_operator( - diff); -} -std::unique_ptr -DynamicSeriesIterator::minus_operator(difference_type diff) const -{ - return this->plus_operator(-diff); -} -std::unique_ptr -DynamicSeriesIterator::minus_operator(difference_type diff) -{ - return this->plus_operator(-diff); -} - -// increment/decrement -DynamicSeriesIterator &DynamicSeriesIterator::increment_operator() -{ - *this = *plus_operator(1); - return *this; -} -DynamicSeriesIterator &DynamicSeriesIterator::decrement_operator() -{ - *this = *minus_operator(1); - return *this; -} - // dereference template auto AbstractSeriesIterator::operator*() -> value_type & @@ -72,63 +31,19 @@ auto AbstractSeriesIterator::operator->() -> value_type * return &this_child()->operator*(); } -// arithmetic random-access -template -ChildClass AbstractSeriesIterator::operator+(difference_type diff) -{ - return static_cast(this)->operator+(diff); -} -template -ChildClass -AbstractSeriesIterator::operator-(difference_type diff) const -{ - return this_child()->operator+(-diff); -} -template -ChildClass AbstractSeriesIterator::operator-(difference_type diff) -{ - return this_child()->operator+(-diff); -} - -// member access -template -auto AbstractSeriesIterator::operator[](difference_type diff) const - -> value_type const & -{ - return *(*this_child() + diff); -} -template -auto AbstractSeriesIterator::operator[](difference_type diff) - -> value_type & -{ - return *(*this_child() + diff); -} - // increment/decrement template -ChildClass &AbstractSeriesIterator::operator++() -{ - *this_child() = *this + 1; - return *this_child(); -} -template -ChildClass &AbstractSeriesIterator::operator--() -{ - *this_child() = *this - 1; - return *this_child(); -} -template ChildClass AbstractSeriesIterator::operator++(int) { auto prev = *this_child(); - operator++(); + this_child()->operator++(); return prev; } template ChildClass AbstractSeriesIterator::operator--(int) { auto prev = *this_child(); - operator--(); + this_child()->operator--(); return prev; } @@ -178,27 +93,6 @@ ChildClass operator+( { return static_cast(iterator).operator+(index); } -template -ChildClass operator+( - Iteration::IterationIndex_t index, - AbstractSeriesIterator &iterator) -{ - return iterator.operator+(index); -} -template -ChildClass operator-( - Iteration::IterationIndex_t index, - AbstractSeriesIterator const &iterator) -{ - return iterator.operator-(index); -} -template -ChildClass operator-( - Iteration::IterationIndex_t index, - AbstractSeriesIterator &iterator) -{ - return iterator.operator-(index); -} /************* * overrides * @@ -212,36 +106,6 @@ auto AbstractSeriesIterator::dereference_operator() const return this_child()->operator*(); } -// arithmetic random-access -template -std::unique_ptr -AbstractSeriesIterator::plus_operator(difference_type diff) const -{ - return std::unique_ptr{ - new ChildClass(this_child()->operator+(diff))}; -} -template -std::unique_ptr -AbstractSeriesIterator::plus_operator(difference_type diff) -{ - return std::unique_ptr{ - new ChildClass(this_child()->operator+(diff))}; -} -template -std::unique_ptr -AbstractSeriesIterator::minus_operator(difference_type diff) const -{ - return std::unique_ptr{ - new ChildClass(this_child()->operator-(diff))}; -} -template -std::unique_ptr -AbstractSeriesIterator::minus_operator(difference_type diff) -{ - return std::unique_ptr{ - new ChildClass(this_child()->operator-(diff))}; -} - // increment/decrement template DynamicSeriesIterator &AbstractSeriesIterator::increment_operator() @@ -256,12 +120,6 @@ DynamicSeriesIterator &AbstractSeriesIterator::decrement_operator() // comparison template -auto AbstractSeriesIterator::difference_operator( - DynamicSeriesIterator const &other) const -> difference_type -{ - return this_child()->operator-(dynamic_cast(other)); -} -template bool AbstractSeriesIterator::equality_operator( DynamicSeriesIterator const &other) const { @@ -296,18 +154,7 @@ AbstractSeriesIterator::clone() const new ChildClass(*static_cast(this))); } -#define OPENPMD_INSTANTIATE(type) \ - template class AbstractSeriesIterator; \ - template auto operator+( \ - Iteration::IterationIndex_t, AbstractSeriesIterator const &) \ - -> type; \ - template auto operator+( \ - Iteration::IterationIndex_t, AbstractSeriesIterator &) -> type; \ - template auto operator-( \ - Iteration::IterationIndex_t, AbstractSeriesIterator const &) \ - -> type; \ - template auto operator-( \ - Iteration::IterationIndex_t, AbstractSeriesIterator &) -> type; +#define OPENPMD_INSTANTIATE(type) template class AbstractSeriesIterator; OPENPMD_INSTANTIATE(SeriesIterator) OPENPMD_INSTANTIATE(OpaqueSeriesIterator) #undef OPENPMD_INSTANTIATE From 24ed6fa05bd298d3b3d25667fccab1e16dc4ca6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 26 Jan 2024 16:58:04 +0100 Subject: [PATCH 10/93] Introduce AbstractSnapshotsContainer --- include/openPMD/RandomAccessSnapshots.hpp | 40 ++++++++++ include/openPMD/Snapshots.hpp | 38 +++++++-- src/Series.cpp | 93 +++++++++++++++++------ src/Snapshots.cpp | 10 +++ 4 files changed, 150 insertions(+), 31 deletions(-) create mode 100644 include/openPMD/RandomAccessSnapshots.hpp diff --git a/include/openPMD/RandomAccessSnapshots.hpp b/include/openPMD/RandomAccessSnapshots.hpp new file mode 100644 index 0000000000..35cc51c607 --- /dev/null +++ b/include/openPMD/RandomAccessSnapshots.hpp @@ -0,0 +1,40 @@ +/* Copyright 2021 Franz Poeschel + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ +#pragma once + +#include "openPMD/Iteration.hpp" +#include "openPMD/SeriesIterator.hpp" +namespace openPMD +{ + +// @todo const iteration? +class RandomAccessSnapshots : AbstractSeriesIterator +{ +private: + using iterator_t = Container::iterator; + iterator_t m_it; + + inline RandomAccessSnapshots(iterator_t it) : m_it(it) + {} + +public: +}; +} // namespace openPMD diff --git a/include/openPMD/Snapshots.hpp b/include/openPMD/Snapshots.hpp index cef52f8ca9..84bf0fd84b 100644 --- a/include/openPMD/Snapshots.hpp +++ b/include/openPMD/Snapshots.hpp @@ -22,6 +22,7 @@ #include "openPMD/SeriesIterator.hpp" #include +#include #include namespace openPMD @@ -31,6 +32,7 @@ class OpaqueSeriesIterator : public AbstractSeriesIterator { private: friend class Series; + friend class StatefulSnapshotsContainer; using parent_t = AbstractSeriesIterator; // no shared_ptr since copied iterators should not share state std::unique_ptr m_internal_iterator; @@ -98,28 +100,48 @@ class OpaqueSeriesIterator : public AbstractSeriesIterator } }; +class AbstractSnapshotsContainer +{ +public: + using iterator_t = OpaqueSeriesIterator; + // using const_iterator_t = ...; + virtual iterator_t begin() = 0; + virtual iterator_t end() = 0; +}; + +class StatefulSnapshotsContainer : public AbstractSnapshotsContainer +{ +private: + friend class Series; + std::function m_begin; + StatefulSnapshotsContainer(std::function begin) + : m_begin(std::move(begin)) + {} + +public: + iterator_t begin() override; + iterator_t end() override; +}; + class Snapshots { private: friend class Series; - std::function m_begin; - std::function m_end; + std::shared_ptr m_snapshots; - inline Snapshots( - std::function begin, - std::function end) - : m_begin(std::move(begin)), m_end(std::move(end)) + inline Snapshots(std::shared_ptr snapshots) + : m_snapshots(std::move(snapshots)) {} public: inline OpaqueSeriesIterator begin() { - return m_begin(); + return m_snapshots->begin(); } inline OpaqueSeriesIterator end() { - return m_end(); + return m_snapshots->end(); } }; } // namespace openPMD diff --git a/src/Series.cpp b/src/Series.cpp index c8b187ddc1..ef08500b41 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -29,6 +29,7 @@ #include "openPMD/IO/Format.hpp" #include "openPMD/IO/IOTask.hpp" #include "openPMD/IterationEncoding.hpp" +#include "openPMD/RandomAccessSnapshots.hpp" #include "openPMD/ReadIterations.hpp" #include "openPMD/SeriesIterator.hpp" #include "openPMD/Snapshots.hpp" @@ -2930,31 +2931,77 @@ ReadIterations Series::readIterations() std::move(res), IOHandler()->m_frontendAccess, get().m_parsePreference}; } -Snapshots Series::snapshots() +namespace { - // Use private constructor instead of copy constructor to avoid - // object slicing - Series copied_series; - copied_series.setData( - std::dynamic_pointer_cast(this->m_attri)); - auto begin = [s = std::move(copied_series)]() mutable { - auto &series = s.get(); - if (!series.m_sharedStatefulIterator) - { - auto parse_preference = series.m_parsePreference; - series.m_sharedStatefulIterator = std::make_unique( - std::move(s), parse_preference); - } - std::unique_ptr internal_iterator_cloned{ - new SeriesIterator(*series.m_sharedStatefulIterator)}; - return OpaqueSeriesIterator(std::move(internal_iterator_cloned)); - }; - auto end = []() mutable { - std::unique_ptr internal_iterator{ - new SeriesIterator()}; - return OpaqueSeriesIterator(std::move(internal_iterator)); + enum class IteratorKind + { + RandomAccess, + Stateful }; - return Snapshots(std::move(begin), std::move(end)); +} + +Snapshots Series::snapshots() +{ + auto &series = get(); + IteratorKind iterator_kind; + { + using IK = IteratorKind; + switch (IOHandler()->m_frontendAccess) + { + case Access::READ_LINEAR: + iterator_kind = IK::Stateful; + break; + case Access::READ_ONLY: + switch (series.m_parsePreference.value()) + { + + case internal::ParsePreference::UpFront: + iterator_kind = IK::RandomAccess; + break; + case internal::ParsePreference::PerStep: + iterator_kind = IK::Stateful; + break; + } + break; + case Access::READ_WRITE: + case Access::CREATE: + case Access::APPEND: + // @todo: consider WriteIterations + iterator_kind = IK::RandomAccess; + break; + } + } + + switch (iterator_kind) + { + + case IteratorKind::RandomAccess: { + } + case IteratorKind::Stateful: { + // Use private constructor instead of copy constructor to avoid + // object slicing + Series copied_series; + copied_series.setData( + std::dynamic_pointer_cast(this->m_attri)); + auto begin = [s = std::move(copied_series)]() mutable { + auto &series_data = s.get(); + if (!series_data.m_sharedStatefulIterator) + { + auto parse_preference = series_data.m_parsePreference; + series_data.m_sharedStatefulIterator = + std::make_unique( + std::move(s), parse_preference); + } + std::unique_ptr internal_iterator_cloned{ + new SeriesIterator(*series_data.m_sharedStatefulIterator)}; + return OpaqueSeriesIterator(std::move(internal_iterator_cloned)); + }; + + return Snapshots(std::shared_ptr( + new StatefulSnapshotsContainer(std::move(begin)))); + } + } + throw std::runtime_error("unreachable!"); } void Series::parseBase() diff --git a/src/Snapshots.cpp b/src/Snapshots.cpp index ec472d866c..82b9f59ef5 100644 --- a/src/Snapshots.cpp +++ b/src/Snapshots.cpp @@ -1,4 +1,5 @@ #include "openPMD/Snapshots.hpp" +#include "openPMD/ReadIterations.hpp" #include namespace openPMD @@ -8,4 +9,13 @@ namespace openPMD // // return OpaqueSeriesIterator(); // throw std::runtime_error("unimplemented"); // } +auto StatefulSnapshotsContainer::begin() -> iterator_t +{ + return m_begin(); +} +auto StatefulSnapshotsContainer::end() -> iterator_t +{ + return OpaqueSeriesIterator( + std::unique_ptr{new SeriesIterator()}); +} } // namespace openPMD From 6be640ad3e29095a8937618ec6d814f19a7485cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 26 Jan 2024 18:02:47 +0100 Subject: [PATCH 11/93] basic random-access iteration --- examples/10_streaming_read.cpp | 5 ++-- include/openPMD/RandomAccessSnapshots.hpp | 28 +++++++++++++++++- include/openPMD/SeriesIterator.hpp | 7 ----- include/openPMD/Snapshots.hpp | 35 +++++++++++++++++++---- src/Series.cpp | 6 ++-- src/SeriesIterator.cpp | 33 ++------------------- src/helper/list_series.cpp | 5 ++-- 7 files changed, 67 insertions(+), 52 deletions(-) diff --git a/examples/10_streaming_read.cpp b/examples/10_streaming_read.cpp index 12392b4e70..cef55fd790 100644 --- a/examples/10_streaming_read.cpp +++ b/examples/10_streaming_read.cpp @@ -34,10 +34,9 @@ int main() // in streaming setups, e.g. an iteration cannot be opened again once // it has been closed. // `Series::iterations` can be directly accessed in random-access workflows. - for (IndexedIteration iteration : series.readIterations()) + for (auto &[index, iteration] : series.snapshots()) { - std::cout << "Current iteration: " << iteration.iterationIndex - << std::endl; + std::cout << "Current iteration: " << index << std::endl; Record electronPositions = iteration.particles["e"]["position"]; std::array loadedChunks; std::array extents; diff --git a/include/openPMD/RandomAccessSnapshots.hpp b/include/openPMD/RandomAccessSnapshots.hpp index 35cc51c607..8c0a363fef 100644 --- a/include/openPMD/RandomAccessSnapshots.hpp +++ b/include/openPMD/RandomAccessSnapshots.hpp @@ -26,15 +26,41 @@ namespace openPMD { // @todo const iteration? -class RandomAccessSnapshots : AbstractSeriesIterator +class RandomAccessSnapshots + : public AbstractSeriesIterator { private: + friend class RandomAccessSnapshotsContainer; using iterator_t = Container::iterator; iterator_t m_it; inline RandomAccessSnapshots(iterator_t it) : m_it(it) {} + using parent_t = AbstractSeriesIterator; + public: + using parent_t::operator*; + inline value_type const &operator*() const + { + return *m_it; + } + + inline RandomAccessSnapshots &operator++() + { + ++m_it; + return *this; + } + + inline RandomAccessSnapshots &operator--() + { + --m_it; + return *this; + } + + inline bool operator==(RandomAccessSnapshots const &other) const + { + return m_it == other.m_it; + } }; } // namespace openPMD diff --git a/include/openPMD/SeriesIterator.hpp b/include/openPMD/SeriesIterator.hpp index aebe7657e6..1eb2fafdd6 100644 --- a/include/openPMD/SeriesIterator.hpp +++ b/include/openPMD/SeriesIterator.hpp @@ -46,7 +46,6 @@ class DynamicSeriesIterator // comparison virtual bool equality_operator(DynamicSeriesIterator const &) const = 0; - virtual bool less_than_operator(DynamicSeriesIterator const &) const = 0; virtual std::unique_ptr clone() const = 0; }; @@ -77,13 +76,8 @@ class AbstractSeriesIterator : public DynamicSeriesIterator ChildClass operator--(int); // comparison - // difference_type operator-(ChildClass const &) const = 0; // bool operator==(ChildClass const &) const = 0; bool operator!=(ChildClass const &) const; - // bool operator<(ChildClass const &) const = 0; - bool operator>(ChildClass const &) const; - bool operator<=(ChildClass const &) const; - bool operator>=(ChildClass const &) const; /************* * overrides * @@ -101,7 +95,6 @@ class AbstractSeriesIterator : public DynamicSeriesIterator // comparison bool equality_operator(DynamicSeriesIterator const &) const override; - bool less_than_operator(DynamicSeriesIterator const &) const override; std::unique_ptr clone() const override; diff --git a/include/openPMD/Snapshots.hpp b/include/openPMD/Snapshots.hpp index 84bf0fd84b..3680eec1cd 100644 --- a/include/openPMD/Snapshots.hpp +++ b/include/openPMD/Snapshots.hpp @@ -20,6 +20,8 @@ */ #pragma once +#include "openPMD/Iteration.hpp" +#include "openPMD/RandomAccessSnapshots.hpp" #include "openPMD/SeriesIterator.hpp" #include #include @@ -33,6 +35,7 @@ class OpaqueSeriesIterator : public AbstractSeriesIterator private: friend class Series; friend class StatefulSnapshotsContainer; + friend class RandomAccessSnapshotsContainer; using parent_t = AbstractSeriesIterator; // no shared_ptr since copied iterators should not share state std::unique_ptr m_internal_iterator; @@ -93,16 +96,13 @@ class OpaqueSeriesIterator : public AbstractSeriesIterator return m_internal_iterator->equality_operator( *other.m_internal_iterator); } - inline bool operator<(OpaqueSeriesIterator const &other) const - { - return m_internal_iterator->less_than_operator( - *other.m_internal_iterator); - } }; class AbstractSnapshotsContainer { public: + using value_t = Iteration; + using index_t = Iteration::IterationIndex_t; using iterator_t = OpaqueSeriesIterator; // using const_iterator_t = ...; virtual iterator_t begin() = 0; @@ -123,6 +123,31 @@ class StatefulSnapshotsContainer : public AbstractSnapshotsContainer iterator_t end() override; }; +/* + * @todo how to deal with iteration::open() iteration::close() ? + */ +class RandomAccessSnapshotsContainer : public AbstractSnapshotsContainer +{ +private: + friend class Series; + Container m_cont; + RandomAccessSnapshotsContainer(Container cont) + : m_cont(std::move(cont)) + {} + +public: + inline iterator_t begin() override + { + return OpaqueSeriesIterator(std::unique_ptr{ + new RandomAccessSnapshots(m_cont.begin())}); + } + inline iterator_t end() override + { + return OpaqueSeriesIterator(std::unique_ptr{ + new RandomAccessSnapshots(m_cont.end())}); + } +}; + class Snapshots { private: diff --git a/src/Series.cpp b/src/Series.cpp index ef08500b41..671621b6f4 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -2943,7 +2943,7 @@ namespace Snapshots Series::snapshots() { auto &series = get(); - IteratorKind iterator_kind; + IteratorKind iterator_kind{}; { using IK = IteratorKind; switch (IOHandler()->m_frontendAccess) @@ -2954,7 +2954,6 @@ Snapshots Series::snapshots() case Access::READ_ONLY: switch (series.m_parsePreference.value()) { - case internal::ParsePreference::UpFront: iterator_kind = IK::RandomAccess; break; @@ -2974,8 +2973,9 @@ Snapshots Series::snapshots() switch (iterator_kind) { - case IteratorKind::RandomAccess: { + return Snapshots(std::shared_ptr{ + new RandomAccessSnapshotsContainer(series.iterations)}); } case IteratorKind::Stateful: { // Use private constructor instead of copy constructor to avoid diff --git a/src/SeriesIterator.cpp b/src/SeriesIterator.cpp index 8650c93f3a..4669289197 100644 --- a/src/SeriesIterator.cpp +++ b/src/SeriesIterator.cpp @@ -1,4 +1,5 @@ #include "openPMD/SeriesIterator.hpp" +#include "openPMD/RandomAccessSnapshots.hpp" #include "openPMD/ReadIterations.hpp" #include "openPMD/Snapshots.hpp" #include @@ -54,24 +55,6 @@ bool AbstractSeriesIterator::operator!=( { return !this_child()->operator==(other); } -template -bool AbstractSeriesIterator::operator>( - ChildClass const &other) const -{ - return other.operator<(*this_child()); -} -template -bool AbstractSeriesIterator::operator<=( - ChildClass const &other) const -{ - return !operator>(other); -} -template -bool AbstractSeriesIterator::operator>=( - ChildClass const &other) const -{ - return !this_child()->operator<(other); -} // helpers template @@ -132,19 +115,6 @@ bool AbstractSeriesIterator::equality_operator( return false; // or throw error? } } -template -bool AbstractSeriesIterator::less_than_operator( - DynamicSeriesIterator const &other) const -{ - if (auto child = dynamic_cast(&other); child) - { - return this_child()->operator<(*child); - } - else - { - return false; // or throw error? - } -} template std::unique_ptr @@ -157,5 +127,6 @@ AbstractSeriesIterator::clone() const #define OPENPMD_INSTANTIATE(type) template class AbstractSeriesIterator; OPENPMD_INSTANTIATE(SeriesIterator) OPENPMD_INSTANTIATE(OpaqueSeriesIterator) +OPENPMD_INSTANTIATE(RandomAccessSnapshots) #undef OPENPMD_INSTANTIATE } // namespace openPMD diff --git a/src/helper/list_series.cpp b/src/helper/list_series.cpp index eeb7523bb4..89c3a45139 100644 --- a/src/helper/list_series.cpp +++ b/src/helper/list_series.cpp @@ -113,10 +113,11 @@ std::ostream &listSeries(Series &series, bool const longer, std::ostream &out) if (longer) out << " all iterations: "; - for (auto const &i : series.readIterations()) + for (auto &[index, i] : series.snapshots()) { + i.open(); if (longer) - out << i.iterationIndex << " "; + out << index << " "; // find unique record names std::transform( From 118c47f961ccae8039b0b00105c9ad0a6d47cc48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 29 Jan 2024 10:51:32 +0100 Subject: [PATCH 12/93] RandomAccessSnapshots.hpp -> snapshots/RandomAccessIterator.hpp --- include/openPMD/Snapshots.hpp | 12 ++++++------ .../RandomAccessIterator.hpp} | 15 +++++++-------- src/Series.cpp | 6 +++--- src/SeriesIterator.cpp | 4 ++-- 4 files changed, 18 insertions(+), 19 deletions(-) rename include/openPMD/{RandomAccessSnapshots.hpp => snapshots/RandomAccessIterator.hpp} (76%) diff --git a/include/openPMD/Snapshots.hpp b/include/openPMD/Snapshots.hpp index 3680eec1cd..129f410ecc 100644 --- a/include/openPMD/Snapshots.hpp +++ b/include/openPMD/Snapshots.hpp @@ -21,8 +21,8 @@ #pragma once #include "openPMD/Iteration.hpp" -#include "openPMD/RandomAccessSnapshots.hpp" #include "openPMD/SeriesIterator.hpp" +#include "openPMD/snapshots/RandomAccessIterator.hpp" #include #include #include @@ -35,7 +35,7 @@ class OpaqueSeriesIterator : public AbstractSeriesIterator private: friend class Series; friend class StatefulSnapshotsContainer; - friend class RandomAccessSnapshotsContainer; + friend class RandomAccessIteratorContainer; using parent_t = AbstractSeriesIterator; // no shared_ptr since copied iterators should not share state std::unique_ptr m_internal_iterator; @@ -126,12 +126,12 @@ class StatefulSnapshotsContainer : public AbstractSnapshotsContainer /* * @todo how to deal with iteration::open() iteration::close() ? */ -class RandomAccessSnapshotsContainer : public AbstractSnapshotsContainer +class RandomAccessIteratorContainer : public AbstractSnapshotsContainer { private: friend class Series; Container m_cont; - RandomAccessSnapshotsContainer(Container cont) + RandomAccessIteratorContainer(Container cont) : m_cont(std::move(cont)) {} @@ -139,12 +139,12 @@ class RandomAccessSnapshotsContainer : public AbstractSnapshotsContainer inline iterator_t begin() override { return OpaqueSeriesIterator(std::unique_ptr{ - new RandomAccessSnapshots(m_cont.begin())}); + new RandomAccessIterator(m_cont.begin())}); } inline iterator_t end() override { return OpaqueSeriesIterator(std::unique_ptr{ - new RandomAccessSnapshots(m_cont.end())}); + new RandomAccessIterator(m_cont.end())}); } }; diff --git a/include/openPMD/RandomAccessSnapshots.hpp b/include/openPMD/snapshots/RandomAccessIterator.hpp similarity index 76% rename from include/openPMD/RandomAccessSnapshots.hpp rename to include/openPMD/snapshots/RandomAccessIterator.hpp index 8c0a363fef..0afa756bf3 100644 --- a/include/openPMD/RandomAccessSnapshots.hpp +++ b/include/openPMD/snapshots/RandomAccessIterator.hpp @@ -26,18 +26,17 @@ namespace openPMD { // @todo const iteration? -class RandomAccessSnapshots - : public AbstractSeriesIterator +class RandomAccessIterator : public AbstractSeriesIterator { private: - friend class RandomAccessSnapshotsContainer; + friend class RandomAccessIteratorContainer; using iterator_t = Container::iterator; iterator_t m_it; - inline RandomAccessSnapshots(iterator_t it) : m_it(it) + inline RandomAccessIterator(iterator_t it) : m_it(it) {} - using parent_t = AbstractSeriesIterator; + using parent_t = AbstractSeriesIterator; public: using parent_t::operator*; @@ -46,19 +45,19 @@ class RandomAccessSnapshots return *m_it; } - inline RandomAccessSnapshots &operator++() + inline RandomAccessIterator &operator++() { ++m_it; return *this; } - inline RandomAccessSnapshots &operator--() + inline RandomAccessIterator &operator--() { --m_it; return *this; } - inline bool operator==(RandomAccessSnapshots const &other) const + inline bool operator==(RandomAccessIterator const &other) const { return m_it == other.m_it; } diff --git a/src/Series.cpp b/src/Series.cpp index 671621b6f4..a9dc5bfcc7 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -29,7 +29,6 @@ #include "openPMD/IO/Format.hpp" #include "openPMD/IO/IOTask.hpp" #include "openPMD/IterationEncoding.hpp" -#include "openPMD/RandomAccessSnapshots.hpp" #include "openPMD/ReadIterations.hpp" #include "openPMD/SeriesIterator.hpp" #include "openPMD/Snapshots.hpp" @@ -41,6 +40,7 @@ #include "openPMD/auxiliary/StringManip.hpp" #include "openPMD/auxiliary/Variant.hpp" #include "openPMD/backend/Attributable.hpp" +#include "openPMD/snapshots/RandomAccessIterator.hpp" #include "openPMD/version.hpp" #include @@ -2974,8 +2974,8 @@ Snapshots Series::snapshots() switch (iterator_kind) { case IteratorKind::RandomAccess: { - return Snapshots(std::shared_ptr{ - new RandomAccessSnapshotsContainer(series.iterations)}); + return Snapshots(std::shared_ptr{ + new RandomAccessIteratorContainer(series.iterations)}); } case IteratorKind::Stateful: { // Use private constructor instead of copy constructor to avoid diff --git a/src/SeriesIterator.cpp b/src/SeriesIterator.cpp index 4669289197..0c3f6fdafb 100644 --- a/src/SeriesIterator.cpp +++ b/src/SeriesIterator.cpp @@ -1,7 +1,7 @@ #include "openPMD/SeriesIterator.hpp" -#include "openPMD/RandomAccessSnapshots.hpp" #include "openPMD/ReadIterations.hpp" #include "openPMD/Snapshots.hpp" +#include "openPMD/snapshots/RandomAccessIterator.hpp" #include namespace openPMD @@ -127,6 +127,6 @@ AbstractSeriesIterator::clone() const #define OPENPMD_INSTANTIATE(type) template class AbstractSeriesIterator; OPENPMD_INSTANTIATE(SeriesIterator) OPENPMD_INSTANTIATE(OpaqueSeriesIterator) -OPENPMD_INSTANTIATE(RandomAccessSnapshots) +OPENPMD_INSTANTIATE(RandomAccessIterator) #undef OPENPMD_INSTANTIATE } // namespace openPMD From cbe73e9a5d64d23a9804a255dbdfae224b6caf1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 29 Jan 2024 11:07:55 +0100 Subject: [PATCH 13/93] ReadIterations.hpp -> snapshots/StatefulIterator.hpp --- CMakeLists.txt | 4 ++-- include/openPMD/Series.hpp | 18 +++++++++--------- include/openPMD/openPMD.hpp | 2 +- .../StatefulIterator.hpp} | 8 ++++---- src/Series.cpp | 6 +++--- src/SeriesIterator.cpp | 2 +- src/Snapshots.cpp | 2 +- src/binding/python/Series.cpp | 10 +++++----- .../StatefulIterator.cpp} | 8 ++++---- 9 files changed, 30 insertions(+), 30 deletions(-) rename include/openPMD/{ReadIterations.hpp => snapshots/StatefulIterator.hpp} (97%) rename src/{ReadIterations.cpp => snapshots/StatefulIterator.cpp} (99%) diff --git a/CMakeLists.txt b/CMakeLists.txt index ca8f0301f7..496eee7e1f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -459,7 +459,6 @@ set(CORE_SOURCE src/Mesh.cpp src/ParticlePatches.cpp src/ParticleSpecies.cpp - src/ReadIterations.cpp src/Record.cpp src/RecordComponent.cpp src/Series.cpp @@ -478,7 +477,8 @@ set(CORE_SOURCE src/backend/PatchRecordComponent.cpp src/backend/Writable.cpp src/benchmark/mpi/OneDimensionalBlockSlicer.cpp - src/helper/list_series.cpp) + src/helper/list_series.cpp + src/snapshots/StatefulIterator.cpp) set(IO_SOURCE src/IO/AbstractIOHandler.cpp src/IO/AbstractIOHandlerImpl.cpp diff --git a/include/openPMD/Series.hpp b/include/openPMD/Series.hpp index 708e77b63d..cd12762b60 100644 --- a/include/openPMD/Series.hpp +++ b/include/openPMD/Series.hpp @@ -59,7 +59,7 @@ namespace openPMD { -class ReadIterations; +class StatefulIterator; class SeriesIterator; class Series; class Series; @@ -252,7 +252,7 @@ class Series : public Attributable friend class Attributable; friend class Iteration; friend class Writable; - friend class ReadIterations; + friend class StatefulIterator; friend class SeriesIterator; friend class internal::SeriesData; friend class internal::AttributableData; @@ -632,7 +632,7 @@ class Series : public Attributable /** * @brief Entry point to the reading end of the streaming API. * - * Creates and returns an instance of the ReadIterations class which can + * Creates and returns an instance of the StatefulIterator class which can * be used for iterating over the openPMD iterations in a C++11-style for * loop. * `Series::readIterations()` is an intentionally restricted API that @@ -640,11 +640,11 @@ class Series : public Attributable * iteration cannot be opened again once it has been closed. * For a less restrictive API in non-streaming situations, * `Series::iterations` can be accessed directly. - * Look for the ReadIterations class for further documentation. + * Look for the StatefulIterator class for further documentation. * - * @return ReadIterations + * @return StatefulIterator */ - ReadIterations readIterations(); + StatefulIterator readIterations(); Snapshots snapshots(); @@ -819,8 +819,8 @@ OPENPMD_private * and turn them into a warning (useful when parsing a Series, since parsing * should succeed without issue). * If true, the error will always be re-thrown (useful when using - * ReadIterations since those methods should be aware when the current step - * is broken). + * StatefulIterator since those methods should be aware when the current + * step is broken). */ std::optional> readGorVBased( bool do_always_throw_errors, @@ -908,4 +908,4 @@ namespace debug // Make sure that this one is always included if Series.hpp is included, // otherwise Series::readIterations() cannot be used -#include "openPMD/ReadIterations.hpp" +#include "openPMD/snapshots/StatefulIterator.hpp" diff --git a/include/openPMD/openPMD.hpp b/include/openPMD/openPMD.hpp index 08853b4d1e..a9ec6e92b7 100644 --- a/include/openPMD/openPMD.hpp +++ b/include/openPMD/openPMD.hpp @@ -34,12 +34,12 @@ namespace openPMD #include "openPMD/Mesh.hpp" #include "openPMD/ParticlePatches.hpp" #include "openPMD/ParticleSpecies.hpp" -#include "openPMD/ReadIterations.hpp" #include "openPMD/Record.hpp" #include "openPMD/RecordComponent.hpp" #include "openPMD/Series.hpp" #include "openPMD/UnitDimension.hpp" #include "openPMD/WriteIterations.hpp" +#include "openPMD/snapshots/StatefulIterator.hpp" #include "openPMD/backend/Attributable.hpp" #include "openPMD/backend/Attribute.hpp" diff --git a/include/openPMD/ReadIterations.hpp b/include/openPMD/snapshots/StatefulIterator.hpp similarity index 97% rename from include/openPMD/ReadIterations.hpp rename to include/openPMD/snapshots/StatefulIterator.hpp index 50b9fdd25e..68d9563754 100644 --- a/include/openPMD/ReadIterations.hpp +++ b/include/openPMD/snapshots/StatefulIterator.hpp @@ -123,7 +123,7 @@ class SeriesIterator : public AbstractSeriesIterator auto &data = get(); if (data.iterationsInCurrentStep.empty()) { - std::cerr << "[ReadIterations] Encountered a step without " + std::cerr << "[StatefulIterator] Encountered a step without " "iterations. Closing the Series." << std::endl; *this = end(); @@ -175,7 +175,7 @@ class LegacyIteratorAdaptor using parent_t = SeriesIterator; private: - friend class ReadIterations; + friend class StatefulIterator; SeriesIterator m_iterator; LegacyIteratorAdaptor(SeriesIterator iterator) : m_iterator(std::move(iterator)) @@ -225,7 +225,7 @@ class LegacyIteratorAdaptor * not possible once it has been closed. * */ -class ReadIterations +class StatefulIterator { friend class Series; @@ -236,7 +236,7 @@ class ReadIterations Series m_series; std::optional m_parsePreference; - ReadIterations( + StatefulIterator( Series, Access, std::optional parsePreference); diff --git a/src/Series.cpp b/src/Series.cpp index a9dc5bfcc7..a4caa38618 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -29,7 +29,6 @@ #include "openPMD/IO/Format.hpp" #include "openPMD/IO/IOTask.hpp" #include "openPMD/IterationEncoding.hpp" -#include "openPMD/ReadIterations.hpp" #include "openPMD/SeriesIterator.hpp" #include "openPMD/Snapshots.hpp" #include "openPMD/ThrowError.hpp" @@ -41,6 +40,7 @@ #include "openPMD/auxiliary/Variant.hpp" #include "openPMD/backend/Attributable.hpp" #include "openPMD/snapshots/RandomAccessIterator.hpp" +#include "openPMD/snapshots/StatefulIterator.hpp" #include "openPMD/version.hpp" #include @@ -2921,13 +2921,13 @@ Series::operator bool() const return m_attri.operator bool(); } -ReadIterations Series::readIterations() +StatefulIterator Series::readIterations() { // Use private constructor instead of copy constructor to avoid // object slicing Series res; res.setData(std::dynamic_pointer_cast(this->m_attri)); - return ReadIterations{ + return StatefulIterator{ std::move(res), IOHandler()->m_frontendAccess, get().m_parsePreference}; } diff --git a/src/SeriesIterator.cpp b/src/SeriesIterator.cpp index 0c3f6fdafb..56c74079e2 100644 --- a/src/SeriesIterator.cpp +++ b/src/SeriesIterator.cpp @@ -1,7 +1,7 @@ #include "openPMD/SeriesIterator.hpp" -#include "openPMD/ReadIterations.hpp" #include "openPMD/Snapshots.hpp" #include "openPMD/snapshots/RandomAccessIterator.hpp" +#include "openPMD/snapshots/StatefulIterator.hpp" #include namespace openPMD diff --git a/src/Snapshots.cpp b/src/Snapshots.cpp index 82b9f59ef5..3dbc173bc1 100644 --- a/src/Snapshots.cpp +++ b/src/Snapshots.cpp @@ -1,5 +1,5 @@ #include "openPMD/Snapshots.hpp" -#include "openPMD/ReadIterations.hpp" +#include "openPMD/snapshots/StatefulIterator.hpp" #include namespace openPMD diff --git a/src/binding/python/Series.cpp b/src/binding/python/Series.cpp index a7e53a415d..87ae2eaba8 100644 --- a/src/binding/python/Series.cpp +++ b/src/binding/python/Series.cpp @@ -21,10 +21,10 @@ #include "openPMD/Series.hpp" #include "openPMD/IO/Access.hpp" #include "openPMD/IterationEncoding.hpp" -#include "openPMD/ReadIterations.hpp" #include "openPMD/auxiliary/JSON.hpp" #include "openPMD/binding/python/Pickle.hpp" #include "openPMD/config.hpp" +#include "openPMD/snapshots/StatefulIterator.hpp" #include "openPMD/binding/python/Common.hpp" @@ -125,7 +125,7 @@ not possible once it has been closed. ); - py::class_(m, "ReadIterations", R"END( + py::class_(m, "StatefulIterator", R"END( Reading side of the streaming API. Create instance via Series.readIterations(). @@ -141,7 +141,7 @@ not possible once it has been closed. )END") .def( "__iter__", - [](ReadIterations &readIterations) { + [](StatefulIterator &readIterations) { // Simple iterator implementation: // But we need to release the GIL inside // SeriesIterator::operator++, so manually it is @@ -354,7 +354,7 @@ this method. R"END( Entry point to the reading end of the streaming API. -Creates and returns an instance of the ReadIterations class which can +Creates and returns an instance of the StatefulIterator class which can be used for iterating over the openPMD iterations in a C++11-style for loop. `Series.read_iterations()` is an intentionally restricted API that @@ -362,7 +362,7 @@ ensures a workflow which also works in streaming setups, e.g. an iteration cannot be opened again once it has been closed. For a less restrictive API in non-streaming situations, `Series.iterations` can be accessed directly. -Look for the ReadIterations class for further documentation. +Look for the StatefulIterator class for further documentation. )END") .def( "parse_base", diff --git a/src/ReadIterations.cpp b/src/snapshots/StatefulIterator.cpp similarity index 99% rename from src/ReadIterations.cpp rename to src/snapshots/StatefulIterator.cpp index 02442a78fd..33001878a3 100644 --- a/src/ReadIterations.cpp +++ b/src/snapshots/StatefulIterator.cpp @@ -19,7 +19,7 @@ * If not, see . */ -#include "openPMD/ReadIterations.hpp" +#include "openPMD/snapshots/StatefulIterator.hpp" #include "openPMD/Error.hpp" #include "openPMD/Series.hpp" @@ -621,7 +621,7 @@ SeriesIterator SeriesIterator::end() return SeriesIterator{}; } -ReadIterations::ReadIterations( +StatefulIterator::StatefulIterator( Series series, Access access, std::optional parsePreference) @@ -636,7 +636,7 @@ ReadIterations::ReadIterations( } } -ReadIterations::iterator_t ReadIterations::begin() +StatefulIterator::iterator_t StatefulIterator::begin() { auto &series = m_series.get(); if (!series.m_sharedStatefulIterator) @@ -647,7 +647,7 @@ ReadIterations::iterator_t ReadIterations::begin() return *series.m_sharedStatefulIterator; } -ReadIterations::iterator_t ReadIterations::end() +StatefulIterator::iterator_t StatefulIterator::end() { return iterator_t::end(); } From 03884661d001c66882cdf2b838ecc996a9e685b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 29 Jan 2024 11:19:19 +0100 Subject: [PATCH 14/93] SeriesIterator.hpp -> snapshots/IteratorTraits.hpp --- CMakeLists.txt | 2 +- include/openPMD/Snapshots.hpp | 2 +- .../{SeriesIterator.hpp => snapshots/IteratorTraits.hpp} | 0 include/openPMD/snapshots/RandomAccessIterator.hpp | 2 +- include/openPMD/snapshots/StatefulIterator.hpp | 2 +- src/Series.cpp | 2 +- src/{SeriesIterator.cpp => snapshots/IteratorTraits.cpp} | 2 +- 7 files changed, 6 insertions(+), 6 deletions(-) rename include/openPMD/{SeriesIterator.hpp => snapshots/IteratorTraits.hpp} (100%) rename src/{SeriesIterator.cpp => snapshots/IteratorTraits.cpp} (98%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 496eee7e1f..0c2caf817a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -462,7 +462,6 @@ set(CORE_SOURCE src/Record.cpp src/RecordComponent.cpp src/Series.cpp - src/SeriesIterator.cpp src/Snapshots.cpp src/version.cpp src/WriteIterations.cpp @@ -478,6 +477,7 @@ set(CORE_SOURCE src/backend/Writable.cpp src/benchmark/mpi/OneDimensionalBlockSlicer.cpp src/helper/list_series.cpp + src/snapshots/IteratorTraits.cpp src/snapshots/StatefulIterator.cpp) set(IO_SOURCE src/IO/AbstractIOHandler.cpp diff --git a/include/openPMD/Snapshots.hpp b/include/openPMD/Snapshots.hpp index 129f410ecc..8773dbd1c6 100644 --- a/include/openPMD/Snapshots.hpp +++ b/include/openPMD/Snapshots.hpp @@ -21,7 +21,7 @@ #pragma once #include "openPMD/Iteration.hpp" -#include "openPMD/SeriesIterator.hpp" +#include "openPMD/snapshots/IteratorTraits.hpp" #include "openPMD/snapshots/RandomAccessIterator.hpp" #include #include diff --git a/include/openPMD/SeriesIterator.hpp b/include/openPMD/snapshots/IteratorTraits.hpp similarity index 100% rename from include/openPMD/SeriesIterator.hpp rename to include/openPMD/snapshots/IteratorTraits.hpp diff --git a/include/openPMD/snapshots/RandomAccessIterator.hpp b/include/openPMD/snapshots/RandomAccessIterator.hpp index 0afa756bf3..439e78d151 100644 --- a/include/openPMD/snapshots/RandomAccessIterator.hpp +++ b/include/openPMD/snapshots/RandomAccessIterator.hpp @@ -21,7 +21,7 @@ #pragma once #include "openPMD/Iteration.hpp" -#include "openPMD/SeriesIterator.hpp" +#include "openPMD/snapshots/IteratorTraits.hpp" namespace openPMD { diff --git a/include/openPMD/snapshots/StatefulIterator.hpp b/include/openPMD/snapshots/StatefulIterator.hpp index 68d9563754..e230e9380b 100644 --- a/include/openPMD/snapshots/StatefulIterator.hpp +++ b/include/openPMD/snapshots/StatefulIterator.hpp @@ -23,8 +23,8 @@ #include "openPMD/Error.hpp" #include "openPMD/Iteration.hpp" #include "openPMD/Series.hpp" -#include "openPMD/SeriesIterator.hpp" #include "openPMD/backend/ParsePreference.hpp" +#include "openPMD/snapshots/IteratorTraits.hpp" #include #include diff --git a/src/Series.cpp b/src/Series.cpp index a4caa38618..80ecc52a34 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -29,7 +29,6 @@ #include "openPMD/IO/Format.hpp" #include "openPMD/IO/IOTask.hpp" #include "openPMD/IterationEncoding.hpp" -#include "openPMD/SeriesIterator.hpp" #include "openPMD/Snapshots.hpp" #include "openPMD/ThrowError.hpp" #include "openPMD/auxiliary/Date.hpp" @@ -39,6 +38,7 @@ #include "openPMD/auxiliary/StringManip.hpp" #include "openPMD/auxiliary/Variant.hpp" #include "openPMD/backend/Attributable.hpp" +#include "openPMD/snapshots/IteratorTraits.hpp" #include "openPMD/snapshots/RandomAccessIterator.hpp" #include "openPMD/snapshots/StatefulIterator.hpp" #include "openPMD/version.hpp" diff --git a/src/SeriesIterator.cpp b/src/snapshots/IteratorTraits.cpp similarity index 98% rename from src/SeriesIterator.cpp rename to src/snapshots/IteratorTraits.cpp index 56c74079e2..be40ac9e18 100644 --- a/src/SeriesIterator.cpp +++ b/src/snapshots/IteratorTraits.cpp @@ -1,4 +1,4 @@ -#include "openPMD/SeriesIterator.hpp" +#include "openPMD/snapshots/IteratorTraits.hpp" #include "openPMD/Snapshots.hpp" #include "openPMD/snapshots/RandomAccessIterator.hpp" #include "openPMD/snapshots/StatefulIterator.hpp" From 9bf0a2e38e000bf4c71949b0f1d2e59e10787feb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 29 Jan 2024 11:22:27 +0100 Subject: [PATCH 15/93] Snapshots.hpp -> snapshots/Snapshots.hpp --- CMakeLists.txt | 2 +- include/openPMD/Series.hpp | 2 +- include/openPMD/{ => snapshots}/Snapshots.hpp | 0 src/Series.cpp | 2 +- src/snapshots/IteratorTraits.cpp | 2 +- src/{ => snapshots}/Snapshots.cpp | 2 +- 6 files changed, 5 insertions(+), 5 deletions(-) rename include/openPMD/{ => snapshots}/Snapshots.hpp (100%) rename src/{ => snapshots}/Snapshots.cpp (92%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0c2caf817a..6d17c0f62e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -462,7 +462,6 @@ set(CORE_SOURCE src/Record.cpp src/RecordComponent.cpp src/Series.cpp - src/Snapshots.cpp src/version.cpp src/WriteIterations.cpp src/auxiliary/Date.cpp @@ -478,6 +477,7 @@ set(CORE_SOURCE src/benchmark/mpi/OneDimensionalBlockSlicer.cpp src/helper/list_series.cpp src/snapshots/IteratorTraits.cpp + src/snapshots/Snapshots.cpp src/snapshots/StatefulIterator.cpp) set(IO_SOURCE src/IO/AbstractIOHandler.cpp diff --git a/include/openPMD/Series.hpp b/include/openPMD/Series.hpp index cd12762b60..30a1530d47 100644 --- a/include/openPMD/Series.hpp +++ b/include/openPMD/Series.hpp @@ -26,7 +26,6 @@ #include "openPMD/IO/Format.hpp" #include "openPMD/Iteration.hpp" #include "openPMD/IterationEncoding.hpp" -#include "openPMD/Snapshots.hpp" #include "openPMD/Streaming.hpp" #include "openPMD/WriteIterations.hpp" #include "openPMD/auxiliary/Variant.hpp" @@ -34,6 +33,7 @@ #include "openPMD/backend/Container.hpp" #include "openPMD/backend/ParsePreference.hpp" #include "openPMD/config.hpp" +#include "openPMD/snapshots/Snapshots.hpp" #include "openPMD/version.hpp" #if openPMD_HAVE_MPI diff --git a/include/openPMD/Snapshots.hpp b/include/openPMD/snapshots/Snapshots.hpp similarity index 100% rename from include/openPMD/Snapshots.hpp rename to include/openPMD/snapshots/Snapshots.hpp diff --git a/src/Series.cpp b/src/Series.cpp index 80ecc52a34..92a5c4a1f0 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -29,7 +29,6 @@ #include "openPMD/IO/Format.hpp" #include "openPMD/IO/IOTask.hpp" #include "openPMD/IterationEncoding.hpp" -#include "openPMD/Snapshots.hpp" #include "openPMD/ThrowError.hpp" #include "openPMD/auxiliary/Date.hpp" #include "openPMD/auxiliary/Filesystem.hpp" @@ -40,6 +39,7 @@ #include "openPMD/backend/Attributable.hpp" #include "openPMD/snapshots/IteratorTraits.hpp" #include "openPMD/snapshots/RandomAccessIterator.hpp" +#include "openPMD/snapshots/Snapshots.hpp" #include "openPMD/snapshots/StatefulIterator.hpp" #include "openPMD/version.hpp" diff --git a/src/snapshots/IteratorTraits.cpp b/src/snapshots/IteratorTraits.cpp index be40ac9e18..1ab623216f 100644 --- a/src/snapshots/IteratorTraits.cpp +++ b/src/snapshots/IteratorTraits.cpp @@ -1,6 +1,6 @@ #include "openPMD/snapshots/IteratorTraits.hpp" -#include "openPMD/Snapshots.hpp" #include "openPMD/snapshots/RandomAccessIterator.hpp" +#include "openPMD/snapshots/Snapshots.hpp" #include "openPMD/snapshots/StatefulIterator.hpp" #include diff --git a/src/Snapshots.cpp b/src/snapshots/Snapshots.cpp similarity index 92% rename from src/Snapshots.cpp rename to src/snapshots/Snapshots.cpp index 3dbc173bc1..c32e20440c 100644 --- a/src/Snapshots.cpp +++ b/src/snapshots/Snapshots.cpp @@ -1,4 +1,4 @@ -#include "openPMD/Snapshots.hpp" +#include "openPMD/snapshots/Snapshots.hpp" #include "openPMD/snapshots/StatefulIterator.hpp" #include From 033e9a79852a0d11fd33b0cf0027b9c94db83e1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 29 Jan 2024 11:29:48 +0100 Subject: [PATCH 16/93] Move AbstractSnapshotsContainer to ContainerTraits.hpp --- include/openPMD/snapshots/ContainerTraits.hpp | 85 +++++++++++++++++++ include/openPMD/snapshots/Snapshots.hpp | 82 +----------------- 2 files changed, 86 insertions(+), 81 deletions(-) create mode 100644 include/openPMD/snapshots/ContainerTraits.hpp diff --git a/include/openPMD/snapshots/ContainerTraits.hpp b/include/openPMD/snapshots/ContainerTraits.hpp new file mode 100644 index 0000000000..bd12305ef7 --- /dev/null +++ b/include/openPMD/snapshots/ContainerTraits.hpp @@ -0,0 +1,85 @@ +#pragma once + +#include "openPMD/Iteration.hpp" +#include "openPMD/snapshots/IteratorTraits.hpp" +namespace openPMD +{ +class OpaqueSeriesIterator : public AbstractSeriesIterator +{ +private: + friend class Series; + friend class StatefulSnapshotsContainer; + friend class RandomAccessIteratorContainer; + using parent_t = AbstractSeriesIterator; + // no shared_ptr since copied iterators should not share state + std::unique_ptr m_internal_iterator; + + OpaqueSeriesIterator( + std::unique_ptr internal_iterator) + : m_internal_iterator(std::move(internal_iterator)) + {} + +public: + OpaqueSeriesIterator(OpaqueSeriesIterator const &other) + : m_internal_iterator(other.m_internal_iterator->clone()) + {} + OpaqueSeriesIterator(OpaqueSeriesIterator &&other) = default; + OpaqueSeriesIterator &operator=(OpaqueSeriesIterator const &other) + { + m_internal_iterator = other.m_internal_iterator->clone(); + return *this; + } + OpaqueSeriesIterator &operator=(OpaqueSeriesIterator &&other) = default; + + ~OpaqueSeriesIterator() = default; + + // dereference + using parent_t::operator*; + inline value_type const &operator*() const + { + return m_internal_iterator->dereference_operator(); + } + + // increment/decrement + OpaqueSeriesIterator &operator++() + { + m_internal_iterator->increment_operator(); + return *this; + } + OpaqueSeriesIterator &operator--() + { + m_internal_iterator->decrement_operator(); + return *this; + } + OpaqueSeriesIterator operator++(int) + { + auto prev = *this; + ++(*this); + return prev; + } + OpaqueSeriesIterator operator--(int) + { + auto prev = *this; + --(*this); + return prev; + } + + // comparison + inline bool operator==(OpaqueSeriesIterator const &other) const + { + return m_internal_iterator->equality_operator( + *other.m_internal_iterator); + } +}; + +class AbstractSnapshotsContainer +{ +public: + using value_t = Iteration; + using index_t = Iteration::IterationIndex_t; + using iterator_t = OpaqueSeriesIterator; + // using const_iterator_t = ...; + virtual iterator_t begin() = 0; + virtual iterator_t end() = 0; +}; +} // namespace openPMD diff --git a/include/openPMD/snapshots/Snapshots.hpp b/include/openPMD/snapshots/Snapshots.hpp index 8773dbd1c6..a64ab75ccc 100644 --- a/include/openPMD/snapshots/Snapshots.hpp +++ b/include/openPMD/snapshots/Snapshots.hpp @@ -21,7 +21,7 @@ #pragma once #include "openPMD/Iteration.hpp" -#include "openPMD/snapshots/IteratorTraits.hpp" +#include "openPMD/snapshots/ContainerTraits.hpp" #include "openPMD/snapshots/RandomAccessIterator.hpp" #include #include @@ -29,86 +29,6 @@ namespace openPMD { - -class OpaqueSeriesIterator : public AbstractSeriesIterator -{ -private: - friend class Series; - friend class StatefulSnapshotsContainer; - friend class RandomAccessIteratorContainer; - using parent_t = AbstractSeriesIterator; - // no shared_ptr since copied iterators should not share state - std::unique_ptr m_internal_iterator; - - OpaqueSeriesIterator( - std::unique_ptr internal_iterator) - : m_internal_iterator(std::move(internal_iterator)) - {} - -public: - OpaqueSeriesIterator(OpaqueSeriesIterator const &other) - : m_internal_iterator(other.m_internal_iterator->clone()) - {} - OpaqueSeriesIterator(OpaqueSeriesIterator &&other) = default; - OpaqueSeriesIterator &operator=(OpaqueSeriesIterator const &other) - { - m_internal_iterator = other.m_internal_iterator->clone(); - return *this; - } - OpaqueSeriesIterator &operator=(OpaqueSeriesIterator &&other) = default; - - ~OpaqueSeriesIterator() = default; - - // dereference - using parent_t::operator*; - inline value_type const &operator*() const - { - return m_internal_iterator->dereference_operator(); - } - - // increment/decrement - OpaqueSeriesIterator &operator++() - { - m_internal_iterator->increment_operator(); - return *this; - } - OpaqueSeriesIterator &operator--() - { - m_internal_iterator->decrement_operator(); - return *this; - } - OpaqueSeriesIterator operator++(int) - { - auto prev = *this; - ++(*this); - return prev; - } - OpaqueSeriesIterator operator--(int) - { - auto prev = *this; - --(*this); - return prev; - } - - // comparison - inline bool operator==(OpaqueSeriesIterator const &other) const - { - return m_internal_iterator->equality_operator( - *other.m_internal_iterator); - } -}; - -class AbstractSnapshotsContainer -{ -public: - using value_t = Iteration; - using index_t = Iteration::IterationIndex_t; - using iterator_t = OpaqueSeriesIterator; - // using const_iterator_t = ...; - virtual iterator_t begin() = 0; - virtual iterator_t end() = 0; -}; - class StatefulSnapshotsContainer : public AbstractSnapshotsContainer { private: From ed689f795e45390f59ee100b3b9d404aa14f3d28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 29 Jan 2024 11:41:14 +0100 Subject: [PATCH 17/93] Move Container implementations to ContainerImpls.(h|c)pp --- CMakeLists.txt | 2 +- include/openPMD/snapshots/ContainerImpls.hpp | 46 +++++++++++++++++++ include/openPMD/snapshots/Snapshots.hpp | 39 ---------------- src/Series.cpp | 1 + .../{Snapshots.cpp => ContainerImpls.cpp} | 7 +-- 5 files changed, 49 insertions(+), 46 deletions(-) create mode 100644 include/openPMD/snapshots/ContainerImpls.hpp rename src/snapshots/{Snapshots.cpp => ContainerImpls.cpp} (65%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6d17c0f62e..1349724ad6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -476,8 +476,8 @@ set(CORE_SOURCE src/backend/Writable.cpp src/benchmark/mpi/OneDimensionalBlockSlicer.cpp src/helper/list_series.cpp + src/snapshots/ContainerImpls.cpp src/snapshots/IteratorTraits.cpp - src/snapshots/Snapshots.cpp src/snapshots/StatefulIterator.cpp) set(IO_SOURCE src/IO/AbstractIOHandler.cpp diff --git a/include/openPMD/snapshots/ContainerImpls.hpp b/include/openPMD/snapshots/ContainerImpls.hpp new file mode 100644 index 0000000000..5ff3aa03fe --- /dev/null +++ b/include/openPMD/snapshots/ContainerImpls.hpp @@ -0,0 +1,46 @@ +#pragma once + +#include "openPMD/snapshots/ContainerTraits.hpp" +#include "openPMD/snapshots/RandomAccessIterator.hpp" + +namespace openPMD +{ +class StatefulSnapshotsContainer : public AbstractSnapshotsContainer +{ +private: + friend class Series; + std::function m_begin; + StatefulSnapshotsContainer(std::function begin) + : m_begin(std::move(begin)) + {} + +public: + iterator_t begin() override; + iterator_t end() override; +}; + +/* + * @todo how to deal with iteration::open() iteration::close() ? + */ +class RandomAccessIteratorContainer : public AbstractSnapshotsContainer +{ +private: + friend class Series; + Container m_cont; + RandomAccessIteratorContainer(Container cont) + : m_cont(std::move(cont)) + {} + +public: + inline iterator_t begin() override + { + return OpaqueSeriesIterator(std::unique_ptr{ + new RandomAccessIterator(m_cont.begin())}); + } + inline iterator_t end() override + { + return OpaqueSeriesIterator(std::unique_ptr{ + new RandomAccessIterator(m_cont.end())}); + } +}; +} // namespace openPMD diff --git a/include/openPMD/snapshots/Snapshots.hpp b/include/openPMD/snapshots/Snapshots.hpp index a64ab75ccc..45dfad8dd4 100644 --- a/include/openPMD/snapshots/Snapshots.hpp +++ b/include/openPMD/snapshots/Snapshots.hpp @@ -29,45 +29,6 @@ namespace openPMD { -class StatefulSnapshotsContainer : public AbstractSnapshotsContainer -{ -private: - friend class Series; - std::function m_begin; - StatefulSnapshotsContainer(std::function begin) - : m_begin(std::move(begin)) - {} - -public: - iterator_t begin() override; - iterator_t end() override; -}; - -/* - * @todo how to deal with iteration::open() iteration::close() ? - */ -class RandomAccessIteratorContainer : public AbstractSnapshotsContainer -{ -private: - friend class Series; - Container m_cont; - RandomAccessIteratorContainer(Container cont) - : m_cont(std::move(cont)) - {} - -public: - inline iterator_t begin() override - { - return OpaqueSeriesIterator(std::unique_ptr{ - new RandomAccessIterator(m_cont.begin())}); - } - inline iterator_t end() override - { - return OpaqueSeriesIterator(std::unique_ptr{ - new RandomAccessIterator(m_cont.end())}); - } -}; - class Snapshots { private: diff --git a/src/Series.cpp b/src/Series.cpp index 92a5c4a1f0..9bc9cc37fa 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -37,6 +37,7 @@ #include "openPMD/auxiliary/StringManip.hpp" #include "openPMD/auxiliary/Variant.hpp" #include "openPMD/backend/Attributable.hpp" +#include "openPMD/snapshots/ContainerImpls.hpp" #include "openPMD/snapshots/IteratorTraits.hpp" #include "openPMD/snapshots/RandomAccessIterator.hpp" #include "openPMD/snapshots/Snapshots.hpp" diff --git a/src/snapshots/Snapshots.cpp b/src/snapshots/ContainerImpls.cpp similarity index 65% rename from src/snapshots/Snapshots.cpp rename to src/snapshots/ContainerImpls.cpp index c32e20440c..a7f8290951 100644 --- a/src/snapshots/Snapshots.cpp +++ b/src/snapshots/ContainerImpls.cpp @@ -1,14 +1,9 @@ -#include "openPMD/snapshots/Snapshots.hpp" +#include "openPMD/snapshots/ContainerImpls.hpp" #include "openPMD/snapshots/StatefulIterator.hpp" #include namespace openPMD { -// OpaqueSeriesIterator Snapshots::begin() -// { -// // return OpaqueSeriesIterator(); -// throw std::runtime_error("unimplemented"); -// } auto StatefulSnapshotsContainer::begin() -> iterator_t { return m_begin(); From bf32dfe41195cf5c87ad3e225d14ea0376c75306 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 29 Jan 2024 11:56:47 +0100 Subject: [PATCH 18/93] Fix: parsePreference is not set in file-based iteratione encoding --- src/Series.cpp | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/src/Series.cpp b/src/Series.cpp index 9bc9cc37fa..7e58931d92 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -2953,15 +2953,29 @@ Snapshots Series::snapshots() iterator_kind = IK::Stateful; break; case Access::READ_ONLY: - switch (series.m_parsePreference.value()) + if (series.m_parsePreference.has_value()) + { + switch (series.m_parsePreference.value()) + { + case internal::ParsePreference::UpFront: + iterator_kind = IK::RandomAccess; + break; + case internal::ParsePreference::PerStep: + iterator_kind = IK::Stateful; + break; + } + } + else if (iterationEncoding() != IterationEncoding::fileBased) + { + throw error::Internal( + "READ_ONLY mode and non-fileBased iteration encoding, but " + "the backend did not set a parse preference."); + } + else { - case internal::ParsePreference::UpFront: iterator_kind = IK::RandomAccess; - break; - case internal::ParsePreference::PerStep: - iterator_kind = IK::Stateful; - break; } + break; case Access::READ_WRITE: case Access::CREATE: From c61954515aae0033c35d69091bcc0ae83e09775c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 29 Jan 2024 13:13:29 +0100 Subject: [PATCH 19/93] Temporarily fix test --- test/ParallelIOTest.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/ParallelIOTest.cpp b/test/ParallelIOTest.cpp index c6d90d773e..e0275eabeb 100644 --- a/test/ParallelIOTest.cpp +++ b/test/ParallelIOTest.cpp @@ -1830,9 +1830,9 @@ void append_mode( ++counter; } REQUIRE(counter == 8); - // listSeries will not see any iterations since they have already - // been read - helper::listSeries(read); + // @todo this does not work (yet), but should at the end of the + // current project + // helper::listSeries(read); } } #endif From 2b054251b96a73bfeab274005af6d543a3f54807 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 29 Jan 2024 15:33:14 +0100 Subject: [PATCH 20/93] Const iteration --- include/openPMD/snapshots/ContainerImpls.hpp | 41 ++++++--- include/openPMD/snapshots/ContainerTraits.hpp | 31 ++++--- include/openPMD/snapshots/IteratorTraits.hpp | 23 +++-- .../snapshots/RandomAccessIterator.hpp | 20 ++++- include/openPMD/snapshots/Snapshots.hpp | 19 ++++- .../openPMD/snapshots/StatefulIterator.hpp | 8 +- src/Series.cpp | 6 +- src/snapshots/ContainerImpls.cpp | 18 +++- src/snapshots/IteratorTraits.cpp | 85 +++++++++++-------- 9 files changed, 171 insertions(+), 80 deletions(-) diff --git a/include/openPMD/snapshots/ContainerImpls.hpp b/include/openPMD/snapshots/ContainerImpls.hpp index 5ff3aa03fe..d4f5a31c33 100644 --- a/include/openPMD/snapshots/ContainerImpls.hpp +++ b/include/openPMD/snapshots/ContainerImpls.hpp @@ -9,14 +9,17 @@ class StatefulSnapshotsContainer : public AbstractSnapshotsContainer { private: friend class Series; - std::function m_begin; - StatefulSnapshotsContainer(std::function begin) + std::function()> m_begin; + StatefulSnapshotsContainer( + std::function()> begin) : m_begin(std::move(begin)) {} public: - iterator_t begin() override; - iterator_t end() override; + iterator begin() override; + iterator end() override; + const_iterator begin() const override; + const_iterator end() const override; }; /* @@ -26,21 +29,35 @@ class RandomAccessIteratorContainer : public AbstractSnapshotsContainer { private: friend class Series; - Container m_cont; - RandomAccessIteratorContainer(Container cont) + Container m_cont; + RandomAccessIteratorContainer(Container cont) : m_cont(std::move(cont)) {} public: - inline iterator_t begin() override + inline iterator begin() override { - return OpaqueSeriesIterator(std::unique_ptr{ - new RandomAccessIterator(m_cont.begin())}); + return OpaqueSeriesIterator( + std::unique_ptr>{ + new RandomAccessIterator(m_cont.begin())}); } - inline iterator_t end() override + inline iterator end() override { - return OpaqueSeriesIterator(std::unique_ptr{ - new RandomAccessIterator(m_cont.end())}); + return OpaqueSeriesIterator( + std::unique_ptr>{ + new RandomAccessIterator(m_cont.end())}); + } + inline const_iterator begin() const override + { + return OpaqueSeriesIterator( + std::unique_ptr>{ + new RandomAccessIterator(m_cont.begin())}); + } + inline const_iterator end() const override + { + return OpaqueSeriesIterator( + std::unique_ptr>{ + new RandomAccessIterator(m_cont.end())}); } }; } // namespace openPMD diff --git a/include/openPMD/snapshots/ContainerTraits.hpp b/include/openPMD/snapshots/ContainerTraits.hpp index bd12305ef7..d5719f5da8 100644 --- a/include/openPMD/snapshots/ContainerTraits.hpp +++ b/include/openPMD/snapshots/ContainerTraits.hpp @@ -4,7 +4,11 @@ #include "openPMD/snapshots/IteratorTraits.hpp" namespace openPMD { -class OpaqueSeriesIterator : public AbstractSeriesIterator +template +class OpaqueSeriesIterator + : public AbstractSeriesIterator< + OpaqueSeriesIterator, + value_type_in> { private: friend class Series; @@ -12,10 +16,10 @@ class OpaqueSeriesIterator : public AbstractSeriesIterator friend class RandomAccessIteratorContainer; using parent_t = AbstractSeriesIterator; // no shared_ptr since copied iterators should not share state - std::unique_ptr m_internal_iterator; + std::unique_ptr> m_internal_iterator; OpaqueSeriesIterator( - std::unique_ptr internal_iterator) + std::unique_ptr> internal_iterator) : m_internal_iterator(std::move(internal_iterator)) {} @@ -23,18 +27,21 @@ class OpaqueSeriesIterator : public AbstractSeriesIterator OpaqueSeriesIterator(OpaqueSeriesIterator const &other) : m_internal_iterator(other.m_internal_iterator->clone()) {} - OpaqueSeriesIterator(OpaqueSeriesIterator &&other) = default; + OpaqueSeriesIterator(OpaqueSeriesIterator &&other) noexcept = default; OpaqueSeriesIterator &operator=(OpaqueSeriesIterator const &other) { m_internal_iterator = other.m_internal_iterator->clone(); return *this; } - OpaqueSeriesIterator &operator=(OpaqueSeriesIterator &&other) = default; + OpaqueSeriesIterator & + operator=(OpaqueSeriesIterator &&other) noexcept = default; ~OpaqueSeriesIterator() = default; // dereference using parent_t::operator*; + using value_type = value_type_in; + inline value_type const &operator*() const { return m_internal_iterator->dereference_operator(); @@ -75,11 +82,13 @@ class OpaqueSeriesIterator : public AbstractSeriesIterator class AbstractSnapshotsContainer { public: - using value_t = Iteration; - using index_t = Iteration::IterationIndex_t; - using iterator_t = OpaqueSeriesIterator; - // using const_iterator_t = ...; - virtual iterator_t begin() = 0; - virtual iterator_t end() = 0; + using key_type = Iteration::IterationIndex_t; + using value_type = Container::value_type; + using iterator = OpaqueSeriesIterator; + using const_iterator = OpaqueSeriesIterator; + virtual iterator begin() = 0; + virtual const_iterator begin() const = 0; + virtual iterator end() = 0; + virtual const_iterator end() const = 0; }; } // namespace openPMD diff --git a/include/openPMD/snapshots/IteratorTraits.hpp b/include/openPMD/snapshots/IteratorTraits.hpp index 1eb2fafdd6..0980ac9c65 100644 --- a/include/openPMD/snapshots/IteratorTraits.hpp +++ b/include/openPMD/snapshots/IteratorTraits.hpp @@ -27,13 +27,16 @@ namespace openPMD { // Abstract class that can be used as an abstract interface to an opaque // iterator implementation +template < + typename value_type = + Container::value_type> class DynamicSeriesIterator { public: using difference_type = Iteration::IterationIndex_t; - using value_type = Container::value_type; protected: + template friend class OpaqueSeriesIterator; // dereference @@ -56,12 +59,14 @@ class DynamicSeriesIterator // Implement as `class MyIterator : public AbstractSeriesIterator` // Use `using AbstractSeriesIterator::operator-` to pull default // implementations. -template -class AbstractSeriesIterator : public DynamicSeriesIterator +template < + typename ChildClass, + typename value_type_in = typename ChildClass::value_type> +class AbstractSeriesIterator : public DynamicSeriesIterator { public: using difference_type = Iteration::IterationIndex_t; - using value_type = Container::value_type; + using value_type = value_type_in; // dereference // value_type const &operator*() const = 0; @@ -84,19 +89,19 @@ class AbstractSeriesIterator : public DynamicSeriesIterator *************/ protected: - using parent_t = DynamicSeriesIterator; + using parent_t = DynamicSeriesIterator; // dereference using parent_t::dereference_operator; value_type const &dereference_operator() const override; // increment/decrement - DynamicSeriesIterator &increment_operator() override; - DynamicSeriesIterator &decrement_operator() override; + parent_t &increment_operator() override; + parent_t &decrement_operator() override; // comparison - bool equality_operator(DynamicSeriesIterator const &) const override; + bool equality_operator(parent_t const &) const override; - std::unique_ptr clone() const override; + std::unique_ptr clone() const override; private: ChildClass *this_child(); diff --git a/include/openPMD/snapshots/RandomAccessIterator.hpp b/include/openPMD/snapshots/RandomAccessIterator.hpp index 439e78d151..c31638935e 100644 --- a/include/openPMD/snapshots/RandomAccessIterator.hpp +++ b/include/openPMD/snapshots/RandomAccessIterator.hpp @@ -22,24 +22,36 @@ #include "openPMD/Iteration.hpp" #include "openPMD/snapshots/IteratorTraits.hpp" +#include namespace openPMD { +namespace detail +{ + template + using iterator_to_value_type = + // do NOT remove const from type + std::remove_reference_t())>; +} -// @todo const iteration? -class RandomAccessIterator : public AbstractSeriesIterator +template +class RandomAccessIterator + : public AbstractSeriesIterator< + RandomAccessIterator, + detail::iterator_to_value_type> { private: friend class RandomAccessIteratorContainer; - using iterator_t = Container::iterator; iterator_t m_it; inline RandomAccessIterator(iterator_t it) : m_it(it) {} - using parent_t = AbstractSeriesIterator; + using parent_t = AbstractSeriesIterator>; public: using parent_t::operator*; + using typename parent_t::value_type; + inline value_type const &operator*() const { return *m_it; diff --git a/include/openPMD/snapshots/Snapshots.hpp b/include/openPMD/snapshots/Snapshots.hpp index 45dfad8dd4..bb50e76736 100644 --- a/include/openPMD/snapshots/Snapshots.hpp +++ b/include/openPMD/snapshots/Snapshots.hpp @@ -41,13 +41,28 @@ class Snapshots {} public: - inline OpaqueSeriesIterator begin() + using key_type = AbstractSnapshotsContainer::key_type; + using value_type = AbstractSnapshotsContainer::value_type; + using iterator = AbstractSnapshotsContainer::iterator; + using const_iterator = AbstractSnapshotsContainer::const_iterator; + + inline iterator begin() { return m_snapshots->begin(); } - inline OpaqueSeriesIterator end() + inline iterator end() { return m_snapshots->end(); } + inline const_iterator begin() const + { + return static_cast(*m_snapshots) + .begin(); + } + inline const_iterator end() const + { + return static_cast(*m_snapshots) + .end(); + } }; } // namespace openPMD diff --git a/include/openPMD/snapshots/StatefulIterator.hpp b/include/openPMD/snapshots/StatefulIterator.hpp index e230e9380b..e73a69f627 100644 --- a/include/openPMD/snapshots/StatefulIterator.hpp +++ b/include/openPMD/snapshots/StatefulIterator.hpp @@ -33,7 +33,10 @@ namespace openPMD { -class SeriesIterator : public AbstractSeriesIterator +class SeriesIterator + : public AbstractSeriesIterator< + SeriesIterator, + Container::value_type> { using iteration_index_t = IndexedIteration::index_t; @@ -77,6 +80,9 @@ class SeriesIterator : public AbstractSeriesIterator using parent_t = AbstractSeriesIterator; public: + using value_type = + typename Container::value_type; + using typename parent_t ::difference_type; //! construct the end() iterator explicit SeriesIterator(); diff --git a/src/Series.cpp b/src/Series.cpp index 7e58931d92..f26efbff32 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -3007,8 +3007,10 @@ Snapshots Series::snapshots() std::make_unique( std::move(s), parse_preference); } - std::unique_ptr internal_iterator_cloned{ - new SeriesIterator(*series_data.m_sharedStatefulIterator)}; + std::unique_ptr::value_type>> + internal_iterator_cloned{ + new SeriesIterator(*series_data.m_sharedStatefulIterator)}; return OpaqueSeriesIterator(std::move(internal_iterator_cloned)); }; diff --git a/src/snapshots/ContainerImpls.cpp b/src/snapshots/ContainerImpls.cpp index a7f8290951..db63043b4b 100644 --- a/src/snapshots/ContainerImpls.cpp +++ b/src/snapshots/ContainerImpls.cpp @@ -1,16 +1,28 @@ #include "openPMD/snapshots/ContainerImpls.hpp" +#include "openPMD/Error.hpp" #include "openPMD/snapshots/StatefulIterator.hpp" #include namespace openPMD { -auto StatefulSnapshotsContainer::begin() -> iterator_t +auto StatefulSnapshotsContainer::begin() -> iterator { return m_begin(); } -auto StatefulSnapshotsContainer::end() -> iterator_t +auto StatefulSnapshotsContainer::end() -> iterator { return OpaqueSeriesIterator( - std::unique_ptr{new SeriesIterator()}); + std::unique_ptr>{ + new SeriesIterator()}); +} +auto StatefulSnapshotsContainer::begin() const -> const_iterator +{ + throw error::WrongAPIUsage( + "Const iteration not possible on a stateful container/iterator."); +} +auto StatefulSnapshotsContainer::end() const -> const_iterator +{ + throw error::WrongAPIUsage( + "Const iteration not possible on a stateful container/iterator."); } } // namespace openPMD diff --git a/src/snapshots/IteratorTraits.cpp b/src/snapshots/IteratorTraits.cpp index 1ab623216f..d12931bdb9 100644 --- a/src/snapshots/IteratorTraits.cpp +++ b/src/snapshots/IteratorTraits.cpp @@ -6,7 +6,8 @@ namespace openPMD { -auto DynamicSeriesIterator::dereference_operator() -> value_type & +template +auto DynamicSeriesIterator::dereference_operator() -> value_type & { return const_cast( static_cast(this) @@ -14,34 +15,35 @@ auto DynamicSeriesIterator::dereference_operator() -> value_type & } // dereference -template -auto AbstractSeriesIterator::operator*() -> value_type & +template +auto AbstractSeriesIterator::operator*() -> value_type & { return const_cast( static_cast(this)->operator*()); } -template -auto AbstractSeriesIterator::operator->() const +template +auto AbstractSeriesIterator::operator->() const -> value_type const * { return &this_child()->operator*(); } -template -auto AbstractSeriesIterator::operator->() -> value_type * +template +auto AbstractSeriesIterator::operator->() + -> value_type * { return &this_child()->operator*(); } // increment/decrement -template -ChildClass AbstractSeriesIterator::operator++(int) +template +ChildClass AbstractSeriesIterator::operator++(int) { auto prev = *this_child(); this_child()->operator++(); return prev; } -template -ChildClass AbstractSeriesIterator::operator--(int) +template +ChildClass AbstractSeriesIterator::operator--(int) { auto prev = *this_child(); this_child()->operator--(); @@ -49,30 +51,31 @@ ChildClass AbstractSeriesIterator::operator--(int) } // comparison -template -bool AbstractSeriesIterator::operator!=( +template +bool AbstractSeriesIterator::operator!=( ChildClass const &other) const { return !this_child()->operator==(other); } // helpers -template -ChildClass *AbstractSeriesIterator::this_child() +template +ChildClass *AbstractSeriesIterator::this_child() { return static_cast(this); } -template -ChildClass const *AbstractSeriesIterator::this_child() const +template +ChildClass const * +AbstractSeriesIterator::this_child() const { return static_cast(this); } // out-of-line arithmetic operators -template +template ChildClass operator+( Iteration::IterationIndex_t index, - AbstractSeriesIterator const &iterator) + AbstractSeriesIterator const &iterator) { return static_cast(iterator).operator+(index); } @@ -82,29 +85,31 @@ ChildClass operator+( *************/ // dereference -template -auto AbstractSeriesIterator::dereference_operator() const - -> value_type const & +template +auto AbstractSeriesIterator::dereference_operator() + const -> value_type const & { return this_child()->operator*(); } // increment/decrement -template -DynamicSeriesIterator &AbstractSeriesIterator::increment_operator() +template +auto AbstractSeriesIterator::increment_operator() + -> parent_t & { return ++*this_child(); } -template -DynamicSeriesIterator &AbstractSeriesIterator::decrement_operator() +template +auto AbstractSeriesIterator::decrement_operator() + -> parent_t & { return --*this_child(); } // comparison -template -bool AbstractSeriesIterator::equality_operator( - DynamicSeriesIterator const &other) const +template +bool AbstractSeriesIterator::equality_operator( + parent_t const &other) const { if (auto child = dynamic_cast(&other); child) { @@ -116,17 +121,25 @@ bool AbstractSeriesIterator::equality_operator( } } -template -std::unique_ptr -AbstractSeriesIterator::clone() const +template +auto AbstractSeriesIterator::clone() const + -> std::unique_ptr { - return std::unique_ptr( + return std::unique_ptr( new ChildClass(*static_cast(this))); } -#define OPENPMD_INSTANTIATE(type) template class AbstractSeriesIterator; +using iterations_container_t = + Container; + +#define OPENPMD_INSTANTIATE(type) \ + template class AbstractSeriesIterator; OPENPMD_INSTANTIATE(SeriesIterator) -OPENPMD_INSTANTIATE(OpaqueSeriesIterator) -OPENPMD_INSTANTIATE(RandomAccessIterator) +OPENPMD_INSTANTIATE(OpaqueSeriesIterator) +OPENPMD_INSTANTIATE( + OpaqueSeriesIterator) +OPENPMD_INSTANTIATE(RandomAccessIterator) +OPENPMD_INSTANTIATE( + RandomAccessIterator) #undef OPENPMD_INSTANTIATE } // namespace openPMD From 2e26222eaa23bedc7b58bf7ce16fda80544cd794 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 29 Jan 2024 16:29:15 +0100 Subject: [PATCH 21/93] Extract stuff to .cpp --- CMakeLists.txt | 3 + include/openPMD/snapshots/ContainerImpls.hpp | 36 ++------ include/openPMD/snapshots/ContainerTraits.hpp | 60 +++---------- .../snapshots/RandomAccessIterator.hpp | 32 ++----- include/openPMD/snapshots/Snapshots.hpp | 26 ++---- src/snapshots/ContainerImpls.cpp | 33 +++++++ src/snapshots/ContainerTraits.cpp | 86 +++++++++++++++++++ src/snapshots/RandomAccessIterator.cpp | 41 +++++++++ 8 files changed, 194 insertions(+), 123 deletions(-) create mode 100644 src/snapshots/ContainerTraits.cpp create mode 100644 src/snapshots/RandomAccessIterator.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 1349724ad6..50af59f825 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -477,7 +477,10 @@ set(CORE_SOURCE src/benchmark/mpi/OneDimensionalBlockSlicer.cpp src/helper/list_series.cpp src/snapshots/ContainerImpls.cpp + src/snapshots/ContainerTraits.cpp src/snapshots/IteratorTraits.cpp + src/snapshots/RandomAccessIterator.cpp + src/snapshots/Snapshots.cpp src/snapshots/StatefulIterator.cpp) set(IO_SOURCE src/IO/AbstractIOHandler.cpp diff --git a/include/openPMD/snapshots/ContainerImpls.hpp b/include/openPMD/snapshots/ContainerImpls.hpp index d4f5a31c33..97d133a370 100644 --- a/include/openPMD/snapshots/ContainerImpls.hpp +++ b/include/openPMD/snapshots/ContainerImpls.hpp @@ -11,9 +11,7 @@ class StatefulSnapshotsContainer : public AbstractSnapshotsContainer friend class Series; std::function()> m_begin; StatefulSnapshotsContainer( - std::function()> begin) - : m_begin(std::move(begin)) - {} + std::function()> begin); public: iterator begin() override; @@ -30,34 +28,12 @@ class RandomAccessIteratorContainer : public AbstractSnapshotsContainer private: friend class Series; Container m_cont; - RandomAccessIteratorContainer(Container cont) - : m_cont(std::move(cont)) - {} + RandomAccessIteratorContainer(Container cont); public: - inline iterator begin() override - { - return OpaqueSeriesIterator( - std::unique_ptr>{ - new RandomAccessIterator(m_cont.begin())}); - } - inline iterator end() override - { - return OpaqueSeriesIterator( - std::unique_ptr>{ - new RandomAccessIterator(m_cont.end())}); - } - inline const_iterator begin() const override - { - return OpaqueSeriesIterator( - std::unique_ptr>{ - new RandomAccessIterator(m_cont.begin())}); - } - inline const_iterator end() const override - { - return OpaqueSeriesIterator( - std::unique_ptr>{ - new RandomAccessIterator(m_cont.end())}); - } + iterator begin() override; + iterator end() override; + const_iterator begin() const override; + const_iterator end() const override; }; } // namespace openPMD diff --git a/include/openPMD/snapshots/ContainerTraits.hpp b/include/openPMD/snapshots/ContainerTraits.hpp index d5719f5da8..8fd962d820 100644 --- a/include/openPMD/snapshots/ContainerTraits.hpp +++ b/include/openPMD/snapshots/ContainerTraits.hpp @@ -18,65 +18,31 @@ class OpaqueSeriesIterator // no shared_ptr since copied iterators should not share state std::unique_ptr> m_internal_iterator; - OpaqueSeriesIterator( - std::unique_ptr> internal_iterator) - : m_internal_iterator(std::move(internal_iterator)) - {} + OpaqueSeriesIterator(std::unique_ptr> + internal_iterator); public: - OpaqueSeriesIterator(OpaqueSeriesIterator const &other) - : m_internal_iterator(other.m_internal_iterator->clone()) - {} - OpaqueSeriesIterator(OpaqueSeriesIterator &&other) noexcept = default; - OpaqueSeriesIterator &operator=(OpaqueSeriesIterator const &other) - { - m_internal_iterator = other.m_internal_iterator->clone(); - return *this; - } - OpaqueSeriesIterator & - operator=(OpaqueSeriesIterator &&other) noexcept = default; + OpaqueSeriesIterator(OpaqueSeriesIterator const &other); + OpaqueSeriesIterator(OpaqueSeriesIterator &&other) noexcept; + OpaqueSeriesIterator &operator=(OpaqueSeriesIterator const &other); + OpaqueSeriesIterator &operator=(OpaqueSeriesIterator &&other) noexcept; - ~OpaqueSeriesIterator() = default; + ~OpaqueSeriesIterator(); // dereference using parent_t::operator*; using value_type = value_type_in; - inline value_type const &operator*() const - { - return m_internal_iterator->dereference_operator(); - } + value_type const &operator*() const; // increment/decrement - OpaqueSeriesIterator &operator++() - { - m_internal_iterator->increment_operator(); - return *this; - } - OpaqueSeriesIterator &operator--() - { - m_internal_iterator->decrement_operator(); - return *this; - } - OpaqueSeriesIterator operator++(int) - { - auto prev = *this; - ++(*this); - return prev; - } - OpaqueSeriesIterator operator--(int) - { - auto prev = *this; - --(*this); - return prev; - } + OpaqueSeriesIterator &operator++(); + OpaqueSeriesIterator &operator--(); + OpaqueSeriesIterator operator++(int); + OpaqueSeriesIterator operator--(int); // comparison - inline bool operator==(OpaqueSeriesIterator const &other) const - { - return m_internal_iterator->equality_operator( - *other.m_internal_iterator); - } + bool operator==(OpaqueSeriesIterator const &other) const; }; class AbstractSnapshotsContainer diff --git a/include/openPMD/snapshots/RandomAccessIterator.hpp b/include/openPMD/snapshots/RandomAccessIterator.hpp index c31638935e..7b6253d3a6 100644 --- a/include/openPMD/snapshots/RandomAccessIterator.hpp +++ b/include/openPMD/snapshots/RandomAccessIterator.hpp @@ -41,37 +41,19 @@ class RandomAccessIterator { private: friend class RandomAccessIteratorContainer; - iterator_t m_it; + using parent_t = AbstractSeriesIterator>; - inline RandomAccessIterator(iterator_t it) : m_it(it) - {} + RandomAccessIterator(iterator_t it); - using parent_t = AbstractSeriesIterator>; + iterator_t m_it; public: using parent_t::operator*; using typename parent_t::value_type; - inline value_type const &operator*() const - { - return *m_it; - } - - inline RandomAccessIterator &operator++() - { - ++m_it; - return *this; - } - - inline RandomAccessIterator &operator--() - { - --m_it; - return *this; - } - - inline bool operator==(RandomAccessIterator const &other) const - { - return m_it == other.m_it; - } + value_type const &operator*() const; + RandomAccessIterator &operator++(); + RandomAccessIterator &operator--(); + bool operator==(RandomAccessIterator const &other) const; }; } // namespace openPMD diff --git a/include/openPMD/snapshots/Snapshots.hpp b/include/openPMD/snapshots/Snapshots.hpp index bb50e76736..f553f88ee7 100644 --- a/include/openPMD/snapshots/Snapshots.hpp +++ b/include/openPMD/snapshots/Snapshots.hpp @@ -36,9 +36,7 @@ class Snapshots std::shared_ptr m_snapshots; - inline Snapshots(std::shared_ptr snapshots) - : m_snapshots(std::move(snapshots)) - {} + Snapshots(std::shared_ptr snapshots); public: using key_type = AbstractSnapshotsContainer::key_type; @@ -46,23 +44,9 @@ class Snapshots using iterator = AbstractSnapshotsContainer::iterator; using const_iterator = AbstractSnapshotsContainer::const_iterator; - inline iterator begin() - { - return m_snapshots->begin(); - } - inline iterator end() - { - return m_snapshots->end(); - } - inline const_iterator begin() const - { - return static_cast(*m_snapshots) - .begin(); - } - inline const_iterator end() const - { - return static_cast(*m_snapshots) - .end(); - } + iterator begin(); + iterator end(); + const_iterator begin() const; + const_iterator end() const; }; } // namespace openPMD diff --git a/src/snapshots/ContainerImpls.cpp b/src/snapshots/ContainerImpls.cpp index db63043b4b..6134211631 100644 --- a/src/snapshots/ContainerImpls.cpp +++ b/src/snapshots/ContainerImpls.cpp @@ -5,6 +5,10 @@ namespace openPMD { +StatefulSnapshotsContainer::StatefulSnapshotsContainer( + std::function()> begin) + : m_begin(std::move(begin)) +{} auto StatefulSnapshotsContainer::begin() -> iterator { return m_begin(); @@ -25,4 +29,33 @@ auto StatefulSnapshotsContainer::end() const -> const_iterator throw error::WrongAPIUsage( "Const iteration not possible on a stateful container/iterator."); } + +RandomAccessIteratorContainer::RandomAccessIteratorContainer( + Container cont) + : m_cont(std::move(cont)) +{} +auto RandomAccessIteratorContainer::begin() -> iterator +{ + return OpaqueSeriesIterator( + std::unique_ptr>{ + new RandomAccessIterator(m_cont.begin())}); +} +auto RandomAccessIteratorContainer::end() -> iterator +{ + return OpaqueSeriesIterator( + std::unique_ptr>{ + new RandomAccessIterator(m_cont.end())}); +} +auto RandomAccessIteratorContainer::begin() const -> const_iterator +{ + return OpaqueSeriesIterator( + std::unique_ptr>{ + new RandomAccessIterator(m_cont.begin())}); +} +auto RandomAccessIteratorContainer::end() const -> const_iterator +{ + return OpaqueSeriesIterator( + std::unique_ptr>{ + new RandomAccessIterator(m_cont.end())}); +} } // namespace openPMD diff --git a/src/snapshots/ContainerTraits.cpp b/src/snapshots/ContainerTraits.cpp new file mode 100644 index 0000000000..c3cfdf6393 --- /dev/null +++ b/src/snapshots/ContainerTraits.cpp @@ -0,0 +1,86 @@ +#include "openPMD/snapshots/ContainerTraits.hpp" +#include "openPMD/Iteration.hpp" + +namespace openPMD +{ +// constructors +template +OpaqueSeriesIterator::OpaqueSeriesIterator( + std::unique_ptr> internal_iterator) + : m_internal_iterator(std::move(internal_iterator)) +{} + +// copy/move constructor +template +OpaqueSeriesIterator::OpaqueSeriesIterator( + OpaqueSeriesIterator const &other) + : m_internal_iterator(other.m_internal_iterator->clone()) +{} +template +OpaqueSeriesIterator::OpaqueSeriesIterator( + OpaqueSeriesIterator &&other) noexcept = default; +template + +// copy/move assignment +OpaqueSeriesIterator & +OpaqueSeriesIterator::operator=(OpaqueSeriesIterator const &other) +{ + m_internal_iterator = other.m_internal_iterator->clone(); + return *this; +} +template +OpaqueSeriesIterator &OpaqueSeriesIterator::operator=( + OpaqueSeriesIterator &&other) noexcept = default; + +// destructor +template +OpaqueSeriesIterator::~OpaqueSeriesIterator() = default; + +// dereference +template +auto OpaqueSeriesIterator::operator*() const -> value_type const & +{ + return m_internal_iterator->dereference_operator(); +} + +// increment/decrement +template +auto OpaqueSeriesIterator::operator++() -> OpaqueSeriesIterator & +{ + m_internal_iterator->increment_operator(); + return *this; +} +template +auto OpaqueSeriesIterator::operator--() -> OpaqueSeriesIterator & +{ + m_internal_iterator->decrement_operator(); + return *this; +} +template +auto OpaqueSeriesIterator::operator++(int) -> OpaqueSeriesIterator +{ + auto prev = *this; + ++(*this); + return prev; +} +template +auto OpaqueSeriesIterator::operator--(int) -> OpaqueSeriesIterator +{ + auto prev = *this; + --(*this); + return prev; +} + +// comparison +template +auto OpaqueSeriesIterator::operator==( + OpaqueSeriesIterator const &other) const -> bool +{ + return m_internal_iterator->equality_operator(*other.m_internal_iterator); +} + +using value_type = + Container::value_type; +template class OpaqueSeriesIterator; +template class OpaqueSeriesIterator; +} // namespace openPMD diff --git a/src/snapshots/RandomAccessIterator.cpp b/src/snapshots/RandomAccessIterator.cpp new file mode 100644 index 0000000000..760e8d57e8 --- /dev/null +++ b/src/snapshots/RandomAccessIterator.cpp @@ -0,0 +1,41 @@ +#include "openPMD/snapshots/RandomAccessIterator.hpp" +namespace openPMD +{ +template +inline RandomAccessIterator::RandomAccessIterator(iterator_t it) + : m_it(it) +{} + +template +auto RandomAccessIterator::operator*() const -> value_type const & +{ + return *m_it; +} + +template +auto RandomAccessIterator::operator++() -> RandomAccessIterator & +{ + ++m_it; + return *this; +} + +template +auto RandomAccessIterator::operator--() -> RandomAccessIterator & +{ + --m_it; + return *this; +} + +template +auto RandomAccessIterator::operator==( + RandomAccessIterator const &other) const -> bool +{ + return m_it == other.m_it; +} + +using iterator = Container::iterator; +using const_iterator = + Container::const_iterator; +template class RandomAccessIterator; +template class RandomAccessIterator; +} // namespace openPMD From 890dc8971a237f0e2c96cd262d0058eef1ad9561 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 29 Jan 2024 17:17:48 +0100 Subject: [PATCH 22/93] Reverse iteration --- include/openPMD/snapshots/ContainerImpls.hpp | 8 +++ include/openPMD/snapshots/ContainerTraits.hpp | 9 ++++ include/openPMD/snapshots/Snapshots.hpp | 9 ++++ src/snapshots/ContainerImpls.cpp | 50 +++++++++++++++++++ src/snapshots/IteratorTraits.cpp | 4 ++ src/snapshots/RandomAccessIterator.cpp | 6 +++ 6 files changed, 86 insertions(+) diff --git a/include/openPMD/snapshots/ContainerImpls.hpp b/include/openPMD/snapshots/ContainerImpls.hpp index 97d133a370..4b97843ac6 100644 --- a/include/openPMD/snapshots/ContainerImpls.hpp +++ b/include/openPMD/snapshots/ContainerImpls.hpp @@ -18,6 +18,10 @@ class StatefulSnapshotsContainer : public AbstractSnapshotsContainer iterator end() override; const_iterator begin() const override; const_iterator end() const override; + iterator rbegin() override; + iterator rend() override; + const_iterator rbegin() const override; + const_iterator rend() const override; }; /* @@ -35,5 +39,9 @@ class RandomAccessIteratorContainer : public AbstractSnapshotsContainer iterator end() override; const_iterator begin() const override; const_iterator end() const override; + iterator rbegin() override; + iterator rend() override; + const_iterator rbegin() const override; + const_iterator rend() const override; }; } // namespace openPMD diff --git a/include/openPMD/snapshots/ContainerTraits.hpp b/include/openPMD/snapshots/ContainerTraits.hpp index 8fd962d820..022ed66456 100644 --- a/include/openPMD/snapshots/ContainerTraits.hpp +++ b/include/openPMD/snapshots/ContainerTraits.hpp @@ -52,9 +52,18 @@ class AbstractSnapshotsContainer using value_type = Container::value_type; using iterator = OpaqueSeriesIterator; using const_iterator = OpaqueSeriesIterator; + // since AbstractSnapshotsContainer abstracts away the specific mode of + // iteration, these are the same type + using reverse_iterator = OpaqueSeriesIterator; + using const_reverse_iterator = OpaqueSeriesIterator; + virtual iterator begin() = 0; virtual const_iterator begin() const = 0; virtual iterator end() = 0; virtual const_iterator end() const = 0; + virtual reverse_iterator rbegin() = 0; + virtual const_reverse_iterator rbegin() const = 0; + virtual reverse_iterator rend() = 0; + virtual const_reverse_iterator rend() const = 0; }; } // namespace openPMD diff --git a/include/openPMD/snapshots/Snapshots.hpp b/include/openPMD/snapshots/Snapshots.hpp index f553f88ee7..dcfa6121ac 100644 --- a/include/openPMD/snapshots/Snapshots.hpp +++ b/include/openPMD/snapshots/Snapshots.hpp @@ -43,10 +43,19 @@ class Snapshots using value_type = AbstractSnapshotsContainer::value_type; using iterator = AbstractSnapshotsContainer::iterator; using const_iterator = AbstractSnapshotsContainer::const_iterator; + // since AbstractSnapshotsContainer abstracts away the specific mode of + // iteration, these are the same type + using reverse_iterator = AbstractSnapshotsContainer::reverse_iterator; + using const_reverse_iterator = + AbstractSnapshotsContainer::const_reverse_iterator; iterator begin(); iterator end(); const_iterator begin() const; const_iterator end() const; + reverse_iterator rbegin(); + reverse_iterator rend(); + const_reverse_iterator rbegin() const; + const_reverse_iterator rend() const; }; } // namespace openPMD diff --git a/src/snapshots/ContainerImpls.cpp b/src/snapshots/ContainerImpls.cpp index 6134211631..b8263c5be6 100644 --- a/src/snapshots/ContainerImpls.cpp +++ b/src/snapshots/ContainerImpls.cpp @@ -29,6 +29,32 @@ auto StatefulSnapshotsContainer::end() const -> const_iterator throw error::WrongAPIUsage( "Const iteration not possible on a stateful container/iterator."); } +auto StatefulSnapshotsContainer::rbegin() -> iterator +{ + /* + * @todo maybe can adapt std::reverse_iterator as soon as the stateful + * iterator is powerful enough for this + */ + throw std::runtime_error( + "Reverse iteration not (yet) implemented on a stateful " + "container/iterator."); +} +auto StatefulSnapshotsContainer::rend() -> iterator +{ + throw std::runtime_error( + "Reverse iteration not (yet) implemented on a stateful " + "container/iterator."); +} +auto StatefulSnapshotsContainer::rbegin() const -> const_iterator +{ + throw error::WrongAPIUsage( + "Const iteration not possible on a stateful container/iterator."); +} +auto StatefulSnapshotsContainer::rend() const -> const_iterator +{ + throw error::WrongAPIUsage( + "Const iteration not possible on a stateful container/iterator."); +} RandomAccessIteratorContainer::RandomAccessIteratorContainer( Container cont) @@ -58,4 +84,28 @@ auto RandomAccessIteratorContainer::end() const -> const_iterator std::unique_ptr>{ new RandomAccessIterator(m_cont.end())}); } +auto RandomAccessIteratorContainer::rbegin() -> reverse_iterator +{ + return OpaqueSeriesIterator( + std::unique_ptr>{ + new RandomAccessIterator(m_cont.rbegin())}); +} +auto RandomAccessIteratorContainer::rend() -> reverse_iterator +{ + return OpaqueSeriesIterator( + std::unique_ptr>{ + new RandomAccessIterator(m_cont.end())}); +} +auto RandomAccessIteratorContainer::rbegin() const -> const_reverse_iterator +{ + return OpaqueSeriesIterator( + std::unique_ptr>{ + new RandomAccessIterator(m_cont.rbegin())}); +} +auto RandomAccessIteratorContainer::rend() const -> const_reverse_iterator +{ + return OpaqueSeriesIterator( + std::unique_ptr>{ + new RandomAccessIterator(m_cont.rend())}); +} } // namespace openPMD diff --git a/src/snapshots/IteratorTraits.cpp b/src/snapshots/IteratorTraits.cpp index d12931bdb9..d553ff31d5 100644 --- a/src/snapshots/IteratorTraits.cpp +++ b/src/snapshots/IteratorTraits.cpp @@ -141,5 +141,9 @@ OPENPMD_INSTANTIATE( OPENPMD_INSTANTIATE(RandomAccessIterator) OPENPMD_INSTANTIATE( RandomAccessIterator) +OPENPMD_INSTANTIATE( + RandomAccessIterator) +OPENPMD_INSTANTIATE( + RandomAccessIterator) #undef OPENPMD_INSTANTIATE } // namespace openPMD diff --git a/src/snapshots/RandomAccessIterator.cpp b/src/snapshots/RandomAccessIterator.cpp index 760e8d57e8..816fb13487 100644 --- a/src/snapshots/RandomAccessIterator.cpp +++ b/src/snapshots/RandomAccessIterator.cpp @@ -36,6 +36,12 @@ auto RandomAccessIterator::operator==( using iterator = Container::iterator; using const_iterator = Container::const_iterator; +using reverse_iterator = + Container::reverse_iterator; +using const_reverse_iterator = + Container::const_reverse_iterator; template class RandomAccessIterator; template class RandomAccessIterator; +template class RandomAccessIterator; +template class RandomAccessIterator; } // namespace openPMD From 4fc208397911de81fa54f637a00198cb4cb94a33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 29 Jan 2024 17:18:04 +0100 Subject: [PATCH 23/93] Commit missing Snapshots.cpp file --- src/snapshots/Snapshots.cpp | 41 +++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 src/snapshots/Snapshots.cpp diff --git a/src/snapshots/Snapshots.cpp b/src/snapshots/Snapshots.cpp new file mode 100644 index 0000000000..e958dd310e --- /dev/null +++ b/src/snapshots/Snapshots.cpp @@ -0,0 +1,41 @@ +#include "openPMD/snapshots/Snapshots.hpp" +namespace openPMD +{ +Snapshots::Snapshots(std::shared_ptr snapshots) + : m_snapshots(std::move(snapshots)) +{} +auto Snapshots::begin() -> iterator +{ + return m_snapshots->begin(); +} +auto Snapshots::end() -> iterator +{ + return m_snapshots->end(); +} +auto Snapshots::begin() const -> const_iterator +{ + return static_cast(*m_snapshots) + .begin(); +} +auto Snapshots::end() const -> const_iterator +{ + return static_cast(*m_snapshots).end(); +} +auto Snapshots::rbegin() -> reverse_iterator +{ + return m_snapshots->begin(); +} +auto Snapshots::rend() -> reverse_iterator +{ + return m_snapshots->end(); +} +auto Snapshots::rbegin() const -> const_reverse_iterator +{ + return static_cast(*m_snapshots) + .begin(); +} +auto Snapshots::rend() const -> const_reverse_iterator +{ + return static_cast(*m_snapshots).end(); +} +} // namespace openPMD From 8aefaff7c5476c1ec2694c1b5f5bbb0b4492a42d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 29 Jan 2024 17:55:22 +0100 Subject: [PATCH 24/93] empty() --- include/openPMD/snapshots/ContainerImpls.hpp | 10 +++++-- include/openPMD/snapshots/ContainerTraits.hpp | 4 ++- include/openPMD/snapshots/Snapshots.hpp | 2 ++ .../openPMD/snapshots/StatefulIterator.hpp | 2 ++ src/Series.cpp | 6 +--- src/snapshots/ContainerImpls.cpp | 30 +++++++++++++++++-- src/snapshots/Snapshots.cpp | 5 ++++ src/snapshots/StatefulIterator.cpp | 5 ++++ 8 files changed, 53 insertions(+), 11 deletions(-) diff --git a/include/openPMD/snapshots/ContainerImpls.hpp b/include/openPMD/snapshots/ContainerImpls.hpp index 4b97843ac6..5bb0d494dd 100644 --- a/include/openPMD/snapshots/ContainerImpls.hpp +++ b/include/openPMD/snapshots/ContainerImpls.hpp @@ -2,6 +2,7 @@ #include "openPMD/snapshots/ContainerTraits.hpp" #include "openPMD/snapshots/RandomAccessIterator.hpp" +#include "openPMD/snapshots/StatefulIterator.hpp" namespace openPMD { @@ -9,9 +10,8 @@ class StatefulSnapshotsContainer : public AbstractSnapshotsContainer { private: friend class Series; - std::function()> m_begin; - StatefulSnapshotsContainer( - std::function()> begin); + std::function m_begin; + StatefulSnapshotsContainer(std::function begin); public: iterator begin() override; @@ -22,6 +22,8 @@ class StatefulSnapshotsContainer : public AbstractSnapshotsContainer iterator rend() override; const_iterator rbegin() const override; const_iterator rend() const override; + + bool empty() const override; }; /* @@ -43,5 +45,7 @@ class RandomAccessIteratorContainer : public AbstractSnapshotsContainer iterator rend() override; const_iterator rbegin() const override; const_iterator rend() const override; + + bool empty() const override; }; } // namespace openPMD diff --git a/include/openPMD/snapshots/ContainerTraits.hpp b/include/openPMD/snapshots/ContainerTraits.hpp index 022ed66456..434918a981 100644 --- a/include/openPMD/snapshots/ContainerTraits.hpp +++ b/include/openPMD/snapshots/ContainerTraits.hpp @@ -18,10 +18,10 @@ class OpaqueSeriesIterator // no shared_ptr since copied iterators should not share state std::unique_ptr> m_internal_iterator; +public: OpaqueSeriesIterator(std::unique_ptr> internal_iterator); -public: OpaqueSeriesIterator(OpaqueSeriesIterator const &other); OpaqueSeriesIterator(OpaqueSeriesIterator &&other) noexcept; OpaqueSeriesIterator &operator=(OpaqueSeriesIterator const &other); @@ -65,5 +65,7 @@ class AbstractSnapshotsContainer virtual const_reverse_iterator rbegin() const = 0; virtual reverse_iterator rend() = 0; virtual const_reverse_iterator rend() const = 0; + + virtual bool empty() const = 0; }; } // namespace openPMD diff --git a/include/openPMD/snapshots/Snapshots.hpp b/include/openPMD/snapshots/Snapshots.hpp index dcfa6121ac..b9aa263beb 100644 --- a/include/openPMD/snapshots/Snapshots.hpp +++ b/include/openPMD/snapshots/Snapshots.hpp @@ -57,5 +57,7 @@ class Snapshots reverse_iterator rend(); const_reverse_iterator rbegin() const; const_reverse_iterator rend() const; + + bool empty() const; }; } // namespace openPMD diff --git a/include/openPMD/snapshots/StatefulIterator.hpp b/include/openPMD/snapshots/StatefulIterator.hpp index e73a69f627..c11219a51e 100644 --- a/include/openPMD/snapshots/StatefulIterator.hpp +++ b/include/openPMD/snapshots/StatefulIterator.hpp @@ -123,6 +123,8 @@ class SeriesIterator static SeriesIterator end(); + operator bool(); + private: inline bool setCurrentIteration() { diff --git a/src/Series.cpp b/src/Series.cpp index f26efbff32..3ef1c75f39 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -3007,11 +3007,7 @@ Snapshots Series::snapshots() std::make_unique( std::move(s), parse_preference); } - std::unique_ptr::value_type>> - internal_iterator_cloned{ - new SeriesIterator(*series_data.m_sharedStatefulIterator)}; - return OpaqueSeriesIterator(std::move(internal_iterator_cloned)); + return series_data.m_sharedStatefulIterator.get(); }; return Snapshots(std::shared_ptr( diff --git a/src/snapshots/ContainerImpls.cpp b/src/snapshots/ContainerImpls.cpp index b8263c5be6..a05a72bbd7 100644 --- a/src/snapshots/ContainerImpls.cpp +++ b/src/snapshots/ContainerImpls.cpp @@ -1,17 +1,33 @@ #include "openPMD/snapshots/ContainerImpls.hpp" #include "openPMD/Error.hpp" +#include "openPMD/snapshots/ContainerTraits.hpp" #include "openPMD/snapshots/StatefulIterator.hpp" +#include #include namespace openPMD { +namespace +{ + using value_type = + Container::value_type; + auto stateful_to_opaque(SeriesIterator const &it) + -> OpaqueSeriesIterator + { + std::unique_ptr> + internal_iterator_cloned{new SeriesIterator(it)}; + return OpaqueSeriesIterator( + std::move(internal_iterator_cloned)); + } +} // namespace + StatefulSnapshotsContainer::StatefulSnapshotsContainer( - std::function()> begin) + std::function begin) : m_begin(std::move(begin)) {} auto StatefulSnapshotsContainer::begin() -> iterator { - return m_begin(); + return stateful_to_opaque(*m_begin()); } auto StatefulSnapshotsContainer::end() -> iterator { @@ -56,6 +72,11 @@ auto StatefulSnapshotsContainer::rend() const -> const_iterator "Const iteration not possible on a stateful container/iterator."); } +bool StatefulSnapshotsContainer::empty() const +{ + return m_begin()->operator bool(); +} + RandomAccessIteratorContainer::RandomAccessIteratorContainer( Container cont) : m_cont(std::move(cont)) @@ -108,4 +129,9 @@ auto RandomAccessIteratorContainer::rend() const -> const_reverse_iterator std::unique_ptr>{ new RandomAccessIterator(m_cont.rend())}); } + +bool RandomAccessIteratorContainer::empty() const +{ + return m_cont.empty(); +} } // namespace openPMD diff --git a/src/snapshots/Snapshots.cpp b/src/snapshots/Snapshots.cpp index e958dd310e..eb63f24359 100644 --- a/src/snapshots/Snapshots.cpp +++ b/src/snapshots/Snapshots.cpp @@ -38,4 +38,9 @@ auto Snapshots::rend() const -> const_reverse_iterator { return static_cast(*m_snapshots).end(); } + +bool Snapshots::empty() const +{ + return m_snapshots->empty(); +} } // namespace openPMD diff --git a/src/snapshots/StatefulIterator.cpp b/src/snapshots/StatefulIterator.cpp index 33001878a3..3dce1b6b7b 100644 --- a/src/snapshots/StatefulIterator.cpp +++ b/src/snapshots/StatefulIterator.cpp @@ -621,6 +621,11 @@ SeriesIterator SeriesIterator::end() return SeriesIterator{}; } +SeriesIterator::operator bool() +{ + return m_data->has_value(); +} + StatefulIterator::StatefulIterator( Series series, Access access, From 0bbdb33422d5287493f1a17db78b75afbdd4f0b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 29 Jan 2024 17:59:05 +0100 Subject: [PATCH 25/93] Revert wrong renaming ReadIterations/StatefulIterator --- include/openPMD/Series.hpp | 16 ++++++++-------- include/openPMD/snapshots/StatefulIterator.hpp | 8 ++++---- src/Series.cpp | 10 +++++----- src/binding/python/Series.cpp | 8 ++++---- src/snapshots/StatefulIterator.cpp | 18 +++++++++--------- 5 files changed, 30 insertions(+), 30 deletions(-) diff --git a/include/openPMD/Series.hpp b/include/openPMD/Series.hpp index 30a1530d47..75ea982fea 100644 --- a/include/openPMD/Series.hpp +++ b/include/openPMD/Series.hpp @@ -59,7 +59,7 @@ namespace openPMD { -class StatefulIterator; +class ReadIterations; class SeriesIterator; class Series; class Series; @@ -117,7 +117,7 @@ namespace internal * Due to include order, this member needs to be a pointer instead of * an optional. */ - std::unique_ptr m_sharedStatefulIterator; + std::unique_ptr m_sharedReadIterations; /** * For writing: Remember which iterations have been written in the * currently active output step. Use this later when writing the @@ -252,7 +252,7 @@ class Series : public Attributable friend class Attributable; friend class Iteration; friend class Writable; - friend class StatefulIterator; + friend class ReadIterations; friend class SeriesIterator; friend class internal::SeriesData; friend class internal::AttributableData; @@ -632,7 +632,7 @@ class Series : public Attributable /** * @brief Entry point to the reading end of the streaming API. * - * Creates and returns an instance of the StatefulIterator class which can + * Creates and returns an instance of the ReadIterations class which can * be used for iterating over the openPMD iterations in a C++11-style for * loop. * `Series::readIterations()` is an intentionally restricted API that @@ -640,11 +640,11 @@ class Series : public Attributable * iteration cannot be opened again once it has been closed. * For a less restrictive API in non-streaming situations, * `Series::iterations` can be accessed directly. - * Look for the StatefulIterator class for further documentation. + * Look for the ReadIterations class for further documentation. * - * @return StatefulIterator + * @return ReadIterations */ - StatefulIterator readIterations(); + ReadIterations readIterations(); Snapshots snapshots(); @@ -819,7 +819,7 @@ OPENPMD_private * and turn them into a warning (useful when parsing a Series, since parsing * should succeed without issue). * If true, the error will always be re-thrown (useful when using - * StatefulIterator since those methods should be aware when the current + * ReadIterations since those methods should be aware when the current * step is broken). */ std::optional> readGorVBased( diff --git a/include/openPMD/snapshots/StatefulIterator.hpp b/include/openPMD/snapshots/StatefulIterator.hpp index c11219a51e..364e0fa062 100644 --- a/include/openPMD/snapshots/StatefulIterator.hpp +++ b/include/openPMD/snapshots/StatefulIterator.hpp @@ -131,7 +131,7 @@ class SeriesIterator auto &data = get(); if (data.iterationsInCurrentStep.empty()) { - std::cerr << "[StatefulIterator] Encountered a step without " + std::cerr << "[ReadIterations] Encountered a step without " "iterations. Closing the Series." << std::endl; *this = end(); @@ -183,7 +183,7 @@ class LegacyIteratorAdaptor using parent_t = SeriesIterator; private: - friend class StatefulIterator; + friend class ReadIterations; SeriesIterator m_iterator; LegacyIteratorAdaptor(SeriesIterator iterator) : m_iterator(std::move(iterator)) @@ -233,7 +233,7 @@ class LegacyIteratorAdaptor * not possible once it has been closed. * */ -class StatefulIterator +class ReadIterations { friend class Series; @@ -244,7 +244,7 @@ class StatefulIterator Series m_series; std::optional m_parsePreference; - StatefulIterator( + ReadIterations( Series, Access, std::optional parsePreference); diff --git a/src/Series.cpp b/src/Series.cpp index 3ef1c75f39..964526afab 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -2922,13 +2922,13 @@ Series::operator bool() const return m_attri.operator bool(); } -StatefulIterator Series::readIterations() +ReadIterations Series::readIterations() { // Use private constructor instead of copy constructor to avoid // object slicing Series res; res.setData(std::dynamic_pointer_cast(this->m_attri)); - return StatefulIterator{ + return ReadIterations{ std::move(res), IOHandler()->m_frontendAccess, get().m_parsePreference}; } @@ -3000,14 +3000,14 @@ Snapshots Series::snapshots() std::dynamic_pointer_cast(this->m_attri)); auto begin = [s = std::move(copied_series)]() mutable { auto &series_data = s.get(); - if (!series_data.m_sharedStatefulIterator) + if (!series_data.m_sharedReadIterations) { auto parse_preference = series_data.m_parsePreference; - series_data.m_sharedStatefulIterator = + series_data.m_sharedReadIterations = std::make_unique( std::move(s), parse_preference); } - return series_data.m_sharedStatefulIterator.get(); + return series_data.m_sharedReadIterations.get(); }; return Snapshots(std::shared_ptr( diff --git a/src/binding/python/Series.cpp b/src/binding/python/Series.cpp index 87ae2eaba8..e700a19f32 100644 --- a/src/binding/python/Series.cpp +++ b/src/binding/python/Series.cpp @@ -125,7 +125,7 @@ not possible once it has been closed. ); - py::class_(m, "StatefulIterator", R"END( + py::class_(m, "ReadIterations", R"END( Reading side of the streaming API. Create instance via Series.readIterations(). @@ -141,7 +141,7 @@ not possible once it has been closed. )END") .def( "__iter__", - [](StatefulIterator &readIterations) { + [](ReadIterations &readIterations) { // Simple iterator implementation: // But we need to release the GIL inside // SeriesIterator::operator++, so manually it is @@ -354,7 +354,7 @@ this method. R"END( Entry point to the reading end of the streaming API. -Creates and returns an instance of the StatefulIterator class which can +Creates and returns an instance of the ReadIterations class which can be used for iterating over the openPMD iterations in a C++11-style for loop. `Series.read_iterations()` is an intentionally restricted API that @@ -362,7 +362,7 @@ ensures a workflow which also works in streaming setups, e.g. an iteration cannot be opened again once it has been closed. For a less restrictive API in non-streaming situations, `Series.iterations` can be accessed directly. -Look for the StatefulIterator class for further documentation. +Look for the ReadIterations class for further documentation. )END") .def( "parse_base", diff --git a/src/snapshots/StatefulIterator.cpp b/src/snapshots/StatefulIterator.cpp index 3dce1b6b7b..0d2696c50b 100644 --- a/src/snapshots/StatefulIterator.cpp +++ b/src/snapshots/StatefulIterator.cpp @@ -130,7 +130,7 @@ SeriesIterator::SeriesIterator( data.parsePreference = parsePreference; /* * Since the iterator is stored in - * internal::SeriesData::m_sharedStatefulIterator, + * internal::SeriesData::m_sharedReadIterations, * we need to use a non-owning Series instance here for tie-breaking * purposes. * This is ok due to the usual C++ iterator invalidation workflows @@ -626,33 +626,33 @@ SeriesIterator::operator bool() return m_data->has_value(); } -StatefulIterator::StatefulIterator( +ReadIterations::ReadIterations( Series series, Access access, std::optional parsePreference) : m_series(std::move(series)), m_parsePreference(parsePreference) { auto &data = m_series.get(); - if (access == Access::READ_LINEAR && !data.m_sharedStatefulIterator) + if (access == Access::READ_LINEAR && !data.m_sharedReadIterations) { // Open the iterator now already, so that metadata may already be read - data.m_sharedStatefulIterator = + data.m_sharedReadIterations = std::make_unique(m_series, m_parsePreference); } } -StatefulIterator::iterator_t StatefulIterator::begin() +ReadIterations::iterator_t ReadIterations::begin() { auto &series = m_series.get(); - if (!series.m_sharedStatefulIterator) + if (!series.m_sharedReadIterations) { - series.m_sharedStatefulIterator = + series.m_sharedReadIterations = std::make_unique(m_series, m_parsePreference); } - return *series.m_sharedStatefulIterator; + return *series.m_sharedReadIterations; } -StatefulIterator::iterator_t StatefulIterator::end() +ReadIterations::iterator_t ReadIterations::end() { return iterator_t::end(); } From ab631bc45cec26560210cc1d541399ac90692632 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 29 Jan 2024 18:03:15 +0100 Subject: [PATCH 26/93] Rename SeriesIterator -> StatefulIterator --- include/openPMD/Iteration.hpp | 4 +- include/openPMD/Series.hpp | 6 +-- include/openPMD/WriteIterations.hpp | 2 +- include/openPMD/backend/Container.hpp | 2 +- include/openPMD/snapshots/ContainerImpls.hpp | 4 +- .../openPMD/snapshots/StatefulIterator.hpp | 40 ++++++++-------- src/Series.cpp | 2 +- src/binding/python/Series.cpp | 12 ++--- src/snapshots/ContainerImpls.cpp | 8 ++-- src/snapshots/IteratorTraits.cpp | 2 +- src/snapshots/StatefulIterator.cpp | 48 ++++++++++--------- 11 files changed, 66 insertions(+), 64 deletions(-) diff --git a/include/openPMD/Iteration.hpp b/include/openPMD/Iteration.hpp index 02166892f3..7f137807ad 100644 --- a/include/openPMD/Iteration.hpp +++ b/include/openPMD/Iteration.hpp @@ -129,10 +129,10 @@ class Iteration : public Attributable friend class Container; friend class Series; friend class WriteIterations; - friend class SeriesIterator; friend class internal::AttributableData; template friend T &internal::makeOwning(T &self, Series); + friend class StatefulIterator; public: Iteration(Iteration const &) = default; @@ -435,7 +435,7 @@ inline T Iteration::dt() const */ class IndexedIteration : public Iteration { - friend class SeriesIterator; + friend class StatefulIterator; friend class WriteIterations; friend class LegacyIteratorAdaptor; diff --git a/include/openPMD/Series.hpp b/include/openPMD/Series.hpp index 75ea982fea..00c41e161d 100644 --- a/include/openPMD/Series.hpp +++ b/include/openPMD/Series.hpp @@ -60,7 +60,7 @@ namespace openPMD { class ReadIterations; -class SeriesIterator; +class StatefulIterator; class Series; class Series; @@ -117,7 +117,7 @@ namespace internal * Due to include order, this member needs to be a pointer instead of * an optional. */ - std::unique_ptr m_sharedReadIterations; + std::unique_ptr m_sharedReadIterations; /** * For writing: Remember which iterations have been written in the * currently active output step. Use this later when writing the @@ -253,7 +253,7 @@ class Series : public Attributable friend class Iteration; friend class Writable; friend class ReadIterations; - friend class SeriesIterator; + friend class StatefulIterator; friend class internal::SeriesData; friend class internal::AttributableData; friend class WriteIterations; diff --git a/include/openPMD/WriteIterations.hpp b/include/openPMD/WriteIterations.hpp index fcf4a8fbfa..41967711aa 100644 --- a/include/openPMD/WriteIterations.hpp +++ b/include/openPMD/WriteIterations.hpp @@ -37,7 +37,7 @@ class Series; * streaming alike. Calling Iteration::close() manually before opening * the next iteration is encouraged and will implicitly flush all * deferred IO actions. Otherwise, Iteration::close() will be implicitly - * called upon SeriesIterator::operator++(), i.e. upon going to the next + * called upon StatefulIterator::operator++(), i.e. upon going to the next * iteration in the foreach loop. * * Since this is designed for streaming mode, reopening an iteration is diff --git a/include/openPMD/backend/Container.hpp b/include/openPMD/backend/Container.hpp index 58b07bd48a..974b106549 100644 --- a/include/openPMD/backend/Container.hpp +++ b/include/openPMD/backend/Container.hpp @@ -113,7 +113,7 @@ class Container : virtual public Attributable friend class Series; template friend class internal::EraseStaleEntries; - friend class SeriesIterator; + friend class StatefulIterator; protected: using ContainerData = internal::ContainerData; diff --git a/include/openPMD/snapshots/ContainerImpls.hpp b/include/openPMD/snapshots/ContainerImpls.hpp index 5bb0d494dd..2a60507590 100644 --- a/include/openPMD/snapshots/ContainerImpls.hpp +++ b/include/openPMD/snapshots/ContainerImpls.hpp @@ -10,8 +10,8 @@ class StatefulSnapshotsContainer : public AbstractSnapshotsContainer { private: friend class Series; - std::function m_begin; - StatefulSnapshotsContainer(std::function begin); + std::function m_begin; + StatefulSnapshotsContainer(std::function begin); public: iterator begin() override; diff --git a/include/openPMD/snapshots/StatefulIterator.hpp b/include/openPMD/snapshots/StatefulIterator.hpp index 364e0fa062..a5fe789fc9 100644 --- a/include/openPMD/snapshots/StatefulIterator.hpp +++ b/include/openPMD/snapshots/StatefulIterator.hpp @@ -33,9 +33,9 @@ namespace openPMD { -class SeriesIterator +class StatefulIterator : public AbstractSeriesIterator< - SeriesIterator, + StatefulIterator, Container::value_type> { using iteration_index_t = IndexedIteration::index_t; @@ -77,16 +77,16 @@ class SeriesIterator return m_data->value(); } - using parent_t = AbstractSeriesIterator; + using parent_t = AbstractSeriesIterator; public: using value_type = typename Container::value_type; using typename parent_t ::difference_type; //! construct the end() iterator - explicit SeriesIterator(); + explicit StatefulIterator(); - SeriesIterator( + StatefulIterator( Series const &, std::optional const &parsePreference); @@ -95,33 +95,33 @@ class SeriesIterator value_type const &operator*() const; // increment/decrement - SeriesIterator &operator++(); + StatefulIterator &operator++(); using parent_t::operator--; - inline SeriesIterator &operator--() + inline StatefulIterator &operator--() { throw error::WrongAPIUsage( "Global stateful iterator supports no decrement (yet)."); } - SeriesIterator operator++(int) + StatefulIterator operator++(int) { throw error::WrongAPIUsage( "Global stateful iterator supports no post-increment."); } // comparison - difference_type operator-(SeriesIterator const &) const + difference_type operator-(StatefulIterator const &) const { throw error::WrongAPIUsage( "Global stateful iterator supports no relative comparison."); } - bool operator==(SeriesIterator const &other) const; - inline bool operator<(SeriesIterator const &) const + bool operator==(StatefulIterator const &other) const; + inline bool operator<(StatefulIterator const &) const { throw error::WrongAPIUsage( "Global stateful iterator supports no relative comparison."); } - static SeriesIterator end(); + static StatefulIterator end(); operator bool(); @@ -154,7 +154,7 @@ class SeriesIterator } } - std::optional nextIterationInStep(); + std::optional nextIterationInStep(); /* * When a step cannot successfully be opened, the method nextStep() calls @@ -166,9 +166,9 @@ class SeriesIterator * the /data/snapshot attribute, this helps figuring out which iteration * is now active. Hence, recursion_depth. */ - std::optional nextStep(size_t recursion_depth); + std::optional nextStep(size_t recursion_depth); - std::optional loopBody(); + std::optional loopBody(); void deactivateDeadIteration(iteration_index_t); @@ -180,12 +180,12 @@ class SeriesIterator class LegacyIteratorAdaptor { using value_type = IndexedIteration; - using parent_t = SeriesIterator; + using parent_t = StatefulIterator; private: friend class ReadIterations; - SeriesIterator m_iterator; - LegacyIteratorAdaptor(SeriesIterator iterator) + StatefulIterator m_iterator; + LegacyIteratorAdaptor(StatefulIterator iterator) : m_iterator(std::move(iterator)) {} @@ -203,7 +203,7 @@ class LegacyIteratorAdaptor static LegacyIteratorAdaptor end() { - return SeriesIterator::end(); + return StatefulIterator::end(); } inline bool operator==(LegacyIteratorAdaptor const &other) const @@ -227,7 +227,7 @@ class LegacyIteratorAdaptor * Calling Iteration::close() manually before opening the next iteration is * encouraged and will implicitly flush all deferred IO actions. * Otherwise, Iteration::close() will be implicitly called upon - * SeriesIterator::operator++(), i.e. upon going to the next iteration in + * StatefulIterator::operator++(), i.e. upon going to the next iteration in * the foreach loop. * Since this is designed for streaming mode, reopening an iteration is * not possible once it has been closed. diff --git a/src/Series.cpp b/src/Series.cpp index 964526afab..425ba1f44e 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -3004,7 +3004,7 @@ Snapshots Series::snapshots() { auto parse_preference = series_data.m_parsePreference; series_data.m_sharedReadIterations = - std::make_unique( + std::make_unique( std::move(s), parse_preference); } return series_data.m_sharedReadIterations.get(); diff --git a/src/binding/python/Series.cpp b/src/binding/python/Series.cpp index e700a19f32..882ffbe500 100644 --- a/src/binding/python/Series.cpp +++ b/src/binding/python/Series.cpp @@ -38,9 +38,9 @@ #include #include -struct SeriesIteratorPythonAdaptor : LegacyIteratorAdaptor +struct StatefulIteratorPythonAdaptor : LegacyIteratorAdaptor { - SeriesIteratorPythonAdaptor(LegacyIteratorAdaptor it) + StatefulIteratorPythonAdaptor(LegacyIteratorAdaptor it) : LegacyIteratorAdaptor(std::move(it)) {} @@ -91,10 +91,10 @@ not possible once it has been closed. "Return the iteration that is currently being written to, if it " "exists."); - py::class_(m, "SeriesIterator") + py::class_(m, "StatefulIterator") .def( "__next__", - [](SeriesIteratorPythonAdaptor &iterator) { + [](StatefulIteratorPythonAdaptor &iterator) { if (iterator == LegacyIteratorAdaptor::end()) { throw py::stop_iteration(); @@ -144,10 +144,10 @@ not possible once it has been closed. [](ReadIterations &readIterations) { // Simple iterator implementation: // But we need to release the GIL inside - // SeriesIterator::operator++, so manually it is + // StatefulIterator::operator++, so manually it is // return py::make_iterator( // readIterations.begin(), readIterations.end()); - return SeriesIteratorPythonAdaptor(readIterations.begin()); + return StatefulIteratorPythonAdaptor(readIterations.begin()); }, // keep handle alive while iterator exists py::keep_alive<0, 1>()); diff --git a/src/snapshots/ContainerImpls.cpp b/src/snapshots/ContainerImpls.cpp index a05a72bbd7..bda96c86b4 100644 --- a/src/snapshots/ContainerImpls.cpp +++ b/src/snapshots/ContainerImpls.cpp @@ -11,18 +11,18 @@ namespace { using value_type = Container::value_type; - auto stateful_to_opaque(SeriesIterator const &it) + auto stateful_to_opaque(StatefulIterator const &it) -> OpaqueSeriesIterator { std::unique_ptr> - internal_iterator_cloned{new SeriesIterator(it)}; + internal_iterator_cloned{new StatefulIterator(it)}; return OpaqueSeriesIterator( std::move(internal_iterator_cloned)); } } // namespace StatefulSnapshotsContainer::StatefulSnapshotsContainer( - std::function begin) + std::function begin) : m_begin(std::move(begin)) {} auto StatefulSnapshotsContainer::begin() -> iterator @@ -33,7 +33,7 @@ auto StatefulSnapshotsContainer::end() -> iterator { return OpaqueSeriesIterator( std::unique_ptr>{ - new SeriesIterator()}); + new StatefulIterator()}); } auto StatefulSnapshotsContainer::begin() const -> const_iterator { diff --git a/src/snapshots/IteratorTraits.cpp b/src/snapshots/IteratorTraits.cpp index d553ff31d5..ef18257a4a 100644 --- a/src/snapshots/IteratorTraits.cpp +++ b/src/snapshots/IteratorTraits.cpp @@ -134,7 +134,7 @@ using iterations_container_t = #define OPENPMD_INSTANTIATE(type) \ template class AbstractSeriesIterator; -OPENPMD_INSTANTIATE(SeriesIterator) +OPENPMD_INSTANTIATE(StatefulIterator) OPENPMD_INSTANTIATE(OpaqueSeriesIterator) OPENPMD_INSTANTIATE( OpaqueSeriesIterator) diff --git a/src/snapshots/StatefulIterator.cpp b/src/snapshots/StatefulIterator.cpp index 0d2696c50b..6a433dd5a3 100644 --- a/src/snapshots/StatefulIterator.cpp +++ b/src/snapshots/StatefulIterator.cpp @@ -55,9 +55,9 @@ namespace } } // namespace -SeriesIterator::SeriesIterator() = default; +StatefulIterator::StatefulIterator() = default; -void SeriesIterator::initSeriesInLinearReadMode() +void StatefulIterator::initSeriesInLinearReadMode() { auto &data = get(); auto &series = *data.series; @@ -116,12 +116,12 @@ void SeriesIterator::initSeriesInLinearReadMode() series.IOHandler()->m_seriesStatus = internal::SeriesStatus::Default; } -void SeriesIterator::close() +void StatefulIterator::close() { *m_data = std::nullopt; // turn this into end iterator } -SeriesIterator::SeriesIterator( +StatefulIterator::StatefulIterator( Series const &series_in, std::optional const &parsePreference) : m_data{std::make_shared>(std::in_place)} @@ -253,10 +253,10 @@ SeriesIterator::SeriesIterator( } } -std::optional SeriesIterator::nextIterationInStep() +std::optional StatefulIterator::nextIterationInStep() { auto &data = get(); - using ret_t = std::optional; + using ret_t = std::optional; if (data.iterationsInCurrentStep.empty()) { @@ -314,9 +314,10 @@ std::optional SeriesIterator::nextIterationInStep() } catch (error::ReadError const &err) { - std::cerr << "[SeriesIterator] Cannot read iteration due to error " - "below, will skip it.\n" - << err.what() << std::endl; + std::cerr + << "[StatefulIterator] Cannot read iteration due to error " + "below, will skip it.\n" + << err.what() << std::endl; return nextIterationInStep(); } @@ -325,7 +326,8 @@ std::optional SeriesIterator::nextIterationInStep() throw std::runtime_error("Unreachable!"); } -std::optional SeriesIterator::nextStep(size_t recursion_depth) +std::optional +StatefulIterator::nextStep(size_t recursion_depth) { auto &data = get(); // since we are in group-based iteration layout, it does not @@ -342,7 +344,7 @@ std::optional SeriesIterator::nextStep(size_t recursion_depth) } catch (error::ReadError const &err) { - std::cerr << "[SeriesIterator] Cannot read iteration due to error " + std::cerr << "[StatefulIterator] Cannot read iteration due to error " "below, will skip it.\n" << err.what() << std::endl; data.series->advance(AdvanceMode::ENDSTEP); @@ -426,7 +428,7 @@ std::optional SeriesIterator::nextStep(size_t recursion_depth) return {this}; } -std::optional SeriesIterator::loopBody() +std::optional StatefulIterator::loopBody() { auto &data = get(); Series &series = data.series.value(); @@ -446,7 +448,7 @@ std::optional SeriesIterator::loopBody() auto guardReturn = [&series, &iterations]( - auto const &option) -> std::optional { + auto const &option) -> std::optional { if (!option.has_value() || *option.value() == end()) { return option; @@ -522,7 +524,7 @@ std::optional SeriesIterator::loopBody() return guardReturn(option); } -void SeriesIterator::deactivateDeadIteration(iteration_index_t index) +void StatefulIterator::deactivateDeadIteration(iteration_index_t index) { auto &data = get(); switch (data.series->iterationEncoding()) @@ -547,7 +549,7 @@ void SeriesIterator::deactivateDeadIteration(iteration_index_t index) data.series->iterations.container().erase(index); } -SeriesIterator &SeriesIterator::operator++() +StatefulIterator &StatefulIterator::operator++() { auto &data = get(); if (!data.series.has_value()) @@ -556,7 +558,7 @@ SeriesIterator &SeriesIterator::operator++() return *this; } auto oldIterationIndex = data.currentIteration; - std::optional res; + std::optional res; /* * loopBody() might return an empty option to indicate a skipped iteration. * Loop until it returns something real for us. @@ -597,7 +599,7 @@ SeriesIterator &SeriesIterator::operator++() return *resvalue; } -auto SeriesIterator::operator*() const -> value_type const & +auto StatefulIterator::operator*() const -> value_type const & { auto &data = get(); auto iterator = static_cast( @@ -606,7 +608,7 @@ auto SeriesIterator::operator*() const -> value_type const & return iterator.operator*(); } -bool SeriesIterator::operator==(SeriesIterator const &other) const +bool StatefulIterator::operator==(StatefulIterator const &other) const { return // either both iterators are filled @@ -616,12 +618,12 @@ bool SeriesIterator::operator==(SeriesIterator const &other) const (!this->m_data->has_value() && !other.m_data->has_value()); } -SeriesIterator SeriesIterator::end() +StatefulIterator StatefulIterator::end() { - return SeriesIterator{}; + return StatefulIterator{}; } -SeriesIterator::operator bool() +StatefulIterator::operator bool() { return m_data->has_value(); } @@ -637,7 +639,7 @@ ReadIterations::ReadIterations( { // Open the iterator now already, so that metadata may already be read data.m_sharedReadIterations = - std::make_unique(m_series, m_parsePreference); + std::make_unique(m_series, m_parsePreference); } } @@ -647,7 +649,7 @@ ReadIterations::iterator_t ReadIterations::begin() if (!series.m_sharedReadIterations) { series.m_sharedReadIterations = - std::make_unique(m_series, m_parsePreference); + std::make_unique(m_series, m_parsePreference); } return *series.m_sharedReadIterations; } From 80e8aa081fef6cf450972bbe3a2ed846369c265b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 30 Jan 2024 13:47:42 +0100 Subject: [PATCH 27/93] Add ::at, operator[] --- include/openPMD/snapshots/ContainerImpls.hpp | 51 +++++++++++-------- include/openPMD/snapshots/ContainerTraits.hpp | 24 +++++---- include/openPMD/snapshots/Snapshots.hpp | 6 +++ src/snapshots/ContainerImpls.cpp | 33 ++++++++++++ src/snapshots/ContainerTraits.cpp | 6 +++ src/snapshots/Snapshots.cpp | 13 +++++ 6 files changed, 104 insertions(+), 29 deletions(-) diff --git a/include/openPMD/snapshots/ContainerImpls.hpp b/include/openPMD/snapshots/ContainerImpls.hpp index 2a60507590..41f4e83b45 100644 --- a/include/openPMD/snapshots/ContainerImpls.hpp +++ b/include/openPMD/snapshots/ContainerImpls.hpp @@ -14,20 +14,27 @@ class StatefulSnapshotsContainer : public AbstractSnapshotsContainer StatefulSnapshotsContainer(std::function begin); public: - iterator begin() override; - iterator end() override; - const_iterator begin() const override; - const_iterator end() const override; - iterator rbegin() override; - iterator rend() override; - const_iterator rbegin() const override; - const_iterator rend() const override; - - bool empty() const override; + auto begin() -> iterator override; + auto end() -> iterator override; + auto begin() const -> const_iterator override; + auto end() const -> const_iterator override; + auto rbegin() -> iterator override; + auto rend() -> iterator override; + auto rbegin() const -> const_iterator override; + auto rend() const -> const_iterator override; + + auto empty() const -> bool override; + + auto at(key_type const &key) const -> mapped_type const & override; + auto at(key_type const &key) -> mapped_type & override; + + auto operator[](key_type const &key) -> mapped_type & override; }; /* * @todo how to deal with iteration::open() iteration::close() ? + * -> have it guaranteed with READ_LINEAR, not with READ_RANDOM_ACCESS, + * but need to see how to deal with write modes */ class RandomAccessIteratorContainer : public AbstractSnapshotsContainer { @@ -37,15 +44,19 @@ class RandomAccessIteratorContainer : public AbstractSnapshotsContainer RandomAccessIteratorContainer(Container cont); public: - iterator begin() override; - iterator end() override; - const_iterator begin() const override; - const_iterator end() const override; - iterator rbegin() override; - iterator rend() override; - const_iterator rbegin() const override; - const_iterator rend() const override; - - bool empty() const override; + auto begin() -> iterator override; + auto end() -> iterator override; + auto begin() const -> const_iterator override; + auto end() const -> const_iterator override; + auto rbegin() -> iterator override; + auto rend() -> iterator override; + auto rbegin() const -> const_iterator override; + auto rend() const -> const_iterator override; + + auto empty() const -> bool override; + + using AbstractSnapshotsContainer::at; + auto at(key_type const &key) const -> mapped_type const & override; + auto operator[](key_type const &key) -> mapped_type & override; }; } // namespace openPMD diff --git a/include/openPMD/snapshots/ContainerTraits.hpp b/include/openPMD/snapshots/ContainerTraits.hpp index 434918a981..3972c3f7f8 100644 --- a/include/openPMD/snapshots/ContainerTraits.hpp +++ b/include/openPMD/snapshots/ContainerTraits.hpp @@ -50,6 +50,7 @@ class AbstractSnapshotsContainer public: using key_type = Iteration::IterationIndex_t; using value_type = Container::value_type; + using mapped_type = Iteration; using iterator = OpaqueSeriesIterator; using const_iterator = OpaqueSeriesIterator; // since AbstractSnapshotsContainer abstracts away the specific mode of @@ -57,15 +58,20 @@ class AbstractSnapshotsContainer using reverse_iterator = OpaqueSeriesIterator; using const_reverse_iterator = OpaqueSeriesIterator; - virtual iterator begin() = 0; - virtual const_iterator begin() const = 0; - virtual iterator end() = 0; - virtual const_iterator end() const = 0; - virtual reverse_iterator rbegin() = 0; - virtual const_reverse_iterator rbegin() const = 0; - virtual reverse_iterator rend() = 0; - virtual const_reverse_iterator rend() const = 0; + virtual auto begin() -> iterator = 0; + virtual auto begin() const -> const_iterator = 0; + virtual auto end() -> iterator = 0; + virtual auto end() const -> const_iterator = 0; + virtual auto rbegin() -> reverse_iterator = 0; + virtual auto rbegin() const -> const_reverse_iterator = 0; + virtual auto rend() -> reverse_iterator = 0; + virtual auto rend() const -> const_reverse_iterator = 0; - virtual bool empty() const = 0; + virtual auto empty() const -> bool = 0; + + virtual auto at(key_type const &key) const -> mapped_type const & = 0; + virtual auto at(key_type const &key) -> mapped_type &; + + virtual auto operator[](key_type const &key) -> mapped_type & = 0; }; } // namespace openPMD diff --git a/include/openPMD/snapshots/Snapshots.hpp b/include/openPMD/snapshots/Snapshots.hpp index b9aa263beb..94711a7e2e 100644 --- a/include/openPMD/snapshots/Snapshots.hpp +++ b/include/openPMD/snapshots/Snapshots.hpp @@ -41,6 +41,7 @@ class Snapshots public: using key_type = AbstractSnapshotsContainer::key_type; using value_type = AbstractSnapshotsContainer::value_type; + using mapped_type = AbstractSnapshotsContainer::mapped_type; using iterator = AbstractSnapshotsContainer::iterator; using const_iterator = AbstractSnapshotsContainer::const_iterator; // since AbstractSnapshotsContainer abstracts away the specific mode of @@ -59,5 +60,10 @@ class Snapshots const_reverse_iterator rend() const; bool empty() const; + + mapped_type const &at(key_type const &key) const; + mapped_type &at(key_type const &key); + + mapped_type &operator[](key_type const &key); }; } // namespace openPMD diff --git a/src/snapshots/ContainerImpls.cpp b/src/snapshots/ContainerImpls.cpp index bda96c86b4..3240efde7c 100644 --- a/src/snapshots/ContainerImpls.cpp +++ b/src/snapshots/ContainerImpls.cpp @@ -77,6 +77,27 @@ bool StatefulSnapshotsContainer::empty() const return m_begin()->operator bool(); } +auto StatefulSnapshotsContainer::at(key_type const &) const + -> mapped_type const & +{ + throw std::runtime_error( + "Item access not (yet) implemented on a stateful " + "container/iterator."); +} +auto StatefulSnapshotsContainer::at(key_type const &) -> mapped_type & +{ + throw std::runtime_error( + "Item access not (yet) implemented on a stateful " + "container/iterator."); +} + +auto StatefulSnapshotsContainer::operator[](key_type const &) -> mapped_type & +{ + throw std::runtime_error( + "Item access not (yet) implemented on a stateful " + "container/iterator."); +} + RandomAccessIteratorContainer::RandomAccessIteratorContainer( Container cont) : m_cont(std::move(cont)) @@ -134,4 +155,16 @@ bool RandomAccessIteratorContainer::empty() const { return m_cont.empty(); } + +auto RandomAccessIteratorContainer::at(key_type const &key) const + -> mapped_type const & +{ + return m_cont.at(key); +} + +auto RandomAccessIteratorContainer::operator[](key_type const &key) + -> mapped_type & +{ + return m_cont[key]; +} } // namespace openPMD diff --git a/src/snapshots/ContainerTraits.cpp b/src/snapshots/ContainerTraits.cpp index c3cfdf6393..8392ea9f0d 100644 --- a/src/snapshots/ContainerTraits.cpp +++ b/src/snapshots/ContainerTraits.cpp @@ -83,4 +83,10 @@ using value_type = Container::value_type; template class OpaqueSeriesIterator; template class OpaqueSeriesIterator; + +auto AbstractSnapshotsContainer::at(key_type const &key) -> mapped_type & +{ + return const_cast( + static_cast(this)->at(key)); +} } // namespace openPMD diff --git a/src/snapshots/Snapshots.cpp b/src/snapshots/Snapshots.cpp index eb63f24359..ce9001d2d1 100644 --- a/src/snapshots/Snapshots.cpp +++ b/src/snapshots/Snapshots.cpp @@ -43,4 +43,17 @@ bool Snapshots::empty() const { return m_snapshots->empty(); } + +auto Snapshots::at(key_type const &key) const -> mapped_type const & +{ + return m_snapshots->at(key); +} +auto Snapshots::at(key_type const &key) -> mapped_type & +{ + return m_snapshots->at(key); +} +auto Snapshots::operator[](key_type const &key) -> mapped_type & +{ + return m_snapshots->operator[](key); +} } // namespace openPMD From aef260e0501c495ad697c85477ed715a03b3fcbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 30 Jan 2024 15:39:37 +0100 Subject: [PATCH 28/93] beginStep(): always return relevant iteration indices --- include/openPMD/Iteration.hpp | 2 +- include/openPMD/Series.hpp | 2 +- include/openPMD/snapshots/ContainerImpls.hpp | 5 + .../openPMD/snapshots/StatefulIterator.hpp | 5 +- src/Iteration.cpp | 28 +++++ src/Series.cpp | 18 ++-- src/snapshots/ContainerImpls.cpp | 16 ++- src/snapshots/StatefulIterator.cpp | 101 +++++++++--------- 8 files changed, 112 insertions(+), 65 deletions(-) diff --git a/include/openPMD/Iteration.hpp b/include/openPMD/Iteration.hpp index 7f137807ad..e15895db94 100644 --- a/include/openPMD/Iteration.hpp +++ b/include/openPMD/Iteration.hpp @@ -315,7 +315,7 @@ class Iteration : public Attributable */ struct BeginStepStatus { - using AvailableIterations_t = std::optional >; + using AvailableIterations_t = std::deque; AdvanceStatus stepStatus{}; /* diff --git a/include/openPMD/Series.hpp b/include/openPMD/Series.hpp index 00c41e161d..de47f6ff0f 100644 --- a/include/openPMD/Series.hpp +++ b/include/openPMD/Series.hpp @@ -822,7 +822,7 @@ OPENPMD_private * ReadIterations since those methods should be aware when the current * step is broken). */ - std::optional> readGorVBased( + std::deque readGorVBased( bool do_always_throw_errors, bool init, std::set const &ignoreIterations = {}); diff --git a/include/openPMD/snapshots/ContainerImpls.hpp b/include/openPMD/snapshots/ContainerImpls.hpp index 41f4e83b45..913fb53d3a 100644 --- a/include/openPMD/snapshots/ContainerImpls.hpp +++ b/include/openPMD/snapshots/ContainerImpls.hpp @@ -13,6 +13,11 @@ class StatefulSnapshotsContainer : public AbstractSnapshotsContainer std::function m_begin; StatefulSnapshotsContainer(std::function begin); + std::optional m_bufferedIterator = std::nullopt; + + auto get() -> StatefulIterator *; + auto get() const -> StatefulIterator const *; + public: auto begin() -> iterator override; auto end() -> iterator override; diff --git a/include/openPMD/snapshots/StatefulIterator.hpp b/include/openPMD/snapshots/StatefulIterator.hpp index a5fe789fc9..640c61326a 100644 --- a/include/openPMD/snapshots/StatefulIterator.hpp +++ b/include/openPMD/snapshots/StatefulIterator.hpp @@ -52,7 +52,8 @@ class StatefulIterator maybe_series_t series; std::deque iterationsInCurrentStep; - uint64_t currentIteration{}; + // nullopt <-> currently out of step + std::optional currentIteration{}; std::optional parsePreference; /* * Necessary because in the old ADIOS2 schema, old iterations' metadata @@ -123,7 +124,7 @@ class StatefulIterator static StatefulIterator end(); - operator bool(); + operator bool() const; private: inline bool setCurrentIteration() diff --git a/src/Iteration.cpp b/src/Iteration.cpp index 93318fdeb0..8052b77efa 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -21,6 +21,7 @@ #include "openPMD/Iteration.hpp" #include "openPMD/Dataset.hpp" #include "openPMD/Datatype.hpp" +#include "openPMD/Error.hpp" #include "openPMD/IO/AbstractIOHandler.hpp" #include "openPMD/IO/IOTask.hpp" #include "openPMD/Series.hpp" @@ -30,8 +31,11 @@ #include "openPMD/backend/Attributable.hpp" #include "openPMD/backend/Writable.hpp" +#include #include #include +#include +#include #include namespace openPMD @@ -777,6 +781,30 @@ auto Iteration::beginStep( series.iterations.setWritten( previous, Attributable::EnqueueAsynchronously::Yes); } + else if (thisObject.has_value()) + { + IterationIndex_t idx = series.indexOf(*thisObject)->first; + res.iterationsInOpenedStep = {idx}; + } + else + { + switch (status) + { + + case AdvanceStatus::OK: + throw error::Internal( + "Control flow error: Opening a new step requires reparsing."); + case AdvanceStatus::RANDOMACCESS: + std::transform( + series.iterations.begin(), + series.iterations.end(), + std::back_inserter(res.iterationsInOpenedStep), + [](auto const &pair) { return pair.first; }); + break; + case AdvanceStatus::OVER: + break; + } + } res.stepStatus = status; return res; diff --git a/src/Series.cpp b/src/Series.cpp index 425ba1f44e..e12c2d8975 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -1855,9 +1855,8 @@ namespace * otherwise. Use only where an empty subtract vector is the * common case. */ - template - void - vectorDifference(std::vector &baseVector, std::vector const &subtract) + template + void vectorDifference(V1 &baseVector, V2 const &subtract) { for (auto const &elem : subtract) { @@ -1877,7 +1876,7 @@ auto Series::readGorVBased( bool do_always_throw_errors, bool do_init, std::set const &ignoreIterations) - -> std::optional> + -> std::deque { auto &series = get(); Parameter fOpen; @@ -2075,7 +2074,8 @@ creating new iterations. * Sic! This happens when a file-based Series is opened in group-based mode. */ case IterationEncoding::fileBased: { - std::vector unreadableIterations; + std::deque unreadableIterations; + std::deque readableIterations; for (auto const &it : *pList.paths) { IterationIndex_t index = std::stoull(it); @@ -2099,6 +2099,10 @@ creating new iterations. } unreadableIterations.push_back(index); } + else + { + readableIterations.push_back(index); + } } if (currentSteps.has_value()) { @@ -2108,7 +2112,7 @@ creating new iterations. } else { - return std::optional>(); + return readableIterations; } } case IterationEncoding::variableBased: { @@ -2981,7 +2985,7 @@ Snapshots Series::snapshots() case Access::CREATE: case Access::APPEND: // @todo: consider WriteIterations - iterator_kind = IK::RandomAccess; + iterator_kind = IK::Stateful; break; } } diff --git a/src/snapshots/ContainerImpls.cpp b/src/snapshots/ContainerImpls.cpp index 3240efde7c..f029266ad3 100644 --- a/src/snapshots/ContainerImpls.cpp +++ b/src/snapshots/ContainerImpls.cpp @@ -25,9 +25,21 @@ StatefulSnapshotsContainer::StatefulSnapshotsContainer( std::function begin) : m_begin(std::move(begin)) {} +auto StatefulSnapshotsContainer::get() -> StatefulIterator * +{ + if (!m_bufferedIterator.has_value()) + { + m_bufferedIterator = m_begin(); + } + return *m_bufferedIterator; +} +auto StatefulSnapshotsContainer::get() const -> StatefulIterator const * +{ + return m_bufferedIterator.value_or(nullptr); +} auto StatefulSnapshotsContainer::begin() -> iterator { - return stateful_to_opaque(*m_begin()); + return stateful_to_opaque(*get()); } auto StatefulSnapshotsContainer::end() -> iterator { @@ -74,7 +86,7 @@ auto StatefulSnapshotsContainer::rend() const -> const_iterator bool StatefulSnapshotsContainer::empty() const { - return m_begin()->operator bool(); + return get()->operator bool(); } auto StatefulSnapshotsContainer::at(key_type const &) const diff --git a/src/snapshots/StatefulIterator.cpp b/src/snapshots/StatefulIterator.cpp index 6a433dd5a3..3f53ac6671 100644 --- a/src/snapshots/StatefulIterator.cpp +++ b/src/snapshots/StatefulIterator.cpp @@ -26,6 +26,7 @@ #include #include +#include namespace openPMD { @@ -209,32 +210,12 @@ StatefulIterator::StatefulIterator( * one by one in ascending order (fallback implementation in the * second if branch). */ - if (availableIterations.has_value() && - status != AdvanceStatus::RANDOMACCESS) + data.iterationsInCurrentStep = availableIterations; + if (!data.iterationsInCurrentStep.empty()) { - data.iterationsInCurrentStep = availableIterations.value(); - if (!data.iterationsInCurrentStep.empty()) - { - openIteration(series.iterations.at( - data.iterationsInCurrentStep.at(0))); - } - } - else if (!series.iterations.empty()) - { - /* - * Fallback implementation: Assume that each step corresponds - * with an iteration in ascending order. - */ - data.iterationsInCurrentStep = { - series.iterations.begin()->first}; - openIteration(series.iterations.begin()->second); + openIteration( + series.iterations.at(data.iterationsInCurrentStep.at(0))); } - else - { - // this is a no-op, but let's keep it explicit - data.iterationsInCurrentStep = {}; - } - break; } } @@ -275,22 +256,24 @@ std::optional StatefulIterator::nextIterationInStep() { case IterationEncoding::groupBased: case IterationEncoding::variableBased: { - auto begin = series.iterations.find(oldIterationIndex); - auto end = begin; - ++end; - series.flush_impl( - begin, - end, - {FlushLevel::UserFlush}, - /* flushIOHandler = */ true); - + if (oldIterationIndex.has_value()) + { + auto begin = series.iterations.find(*oldIterationIndex); + auto end = begin; + ++end; + series.flush_impl( + begin, + end, + {FlushLevel::UserFlush}, + /* flushIOHandler = */ true); + } try { - series.iterations[data.currentIteration].open(); + series.iterations[*data.currentIteration].open(); } catch (error::ReadError const &err) { - std::cerr << "Cannot read iteration '" << data.currentIteration + std::cerr << "Cannot read iteration '" << *data.currentIteration << "' and will skip it due to read error:\n" << err.what() << std::endl; return nextIterationInStep(); @@ -304,12 +287,12 @@ std::optional StatefulIterator::nextIterationInStep() /* * Errors in here might appear due to deferred iteration parsing. */ - series.iterations[data.currentIteration].open(); + series.iterations[*data.currentIteration].open(); /* * Errors in here might appear due to reparsing after opening a * new step. */ - series.iterations[data.currentIteration].beginStep( + series.iterations[*data.currentIteration].beginStep( /* reread = */ true); } catch (error::ReadError const &err) @@ -351,10 +334,9 @@ StatefulIterator::nextStep(size_t recursion_depth) return nextStep(recursion_depth + 1); } - if (availableIterations.has_value() && - status != AdvanceStatus::RANDOMACCESS) + if (status != AdvanceStatus::RANDOMACCESS) { - data.iterationsInCurrentStep = availableIterations.value(); + data.iterationsInCurrentStep = availableIterations; } else { @@ -362,8 +344,13 @@ StatefulIterator::nextStep(size_t recursion_depth) * Fallback implementation: Assume that each step corresponds * with an iteration in ascending order. */ + if (!data.currentIteration.has_value()) + { + throw std::runtime_error( + "CATASTROPHE. need to think how to resolve this one."); + } auto &series = data.series.value(); - auto it = series.iterations.find(data.currentIteration); + auto it = series.iterations.find(*data.currentIteration); auto itEnd = series.iterations.end(); if (it == itEnd) { @@ -437,9 +424,10 @@ std::optional StatefulIterator::loopBody() /* * Might not be present because parsing might have failed in previous step */ - if (iterations.contains(data.currentIteration)) + if (data.currentIteration.has_value() && + iterations.contains(*data.currentIteration)) { - auto ¤tIteration = iterations[data.currentIteration]; + auto ¤tIteration = iterations[*data.currentIteration]; if (!currentIteration.closed()) { currentIteration.close(); @@ -576,10 +564,11 @@ StatefulIterator &StatefulIterator::operator++() { auto &series = data.series.value(); auto index = data.currentIteration; - auto &iteration = series.iterations[index]; + auto &iteration = series.iterations[index.value()]; iteration.setStepStatus(StepStatus::DuringStep); - if (series.IOHandler()->m_frontendAccess == Access::READ_LINEAR) + if (series.IOHandler()->m_frontendAccess == Access::READ_LINEAR && + oldIterationIndex.has_value()) { /* * Linear read mode: Any data outside the current iteration is @@ -592,8 +581,8 @@ StatefulIterator &StatefulIterator::operator++() * @todo Also delete data in the backends upon doing this. */ auto &container = series.iterations.container(); - container.erase(oldIterationIndex); - data.ignoreIterations.emplace(oldIterationIndex); + container.erase(*oldIterationIndex); + data.ignoreIterations.emplace(*oldIterationIndex); } } return *resvalue; @@ -602,10 +591,18 @@ StatefulIterator &StatefulIterator::operator++() auto StatefulIterator::operator*() const -> value_type const & { auto &data = get(); - auto iterator = static_cast( - data.series.value().iterations) - .find(data.currentIteration); - return iterator.operator*(); + if (data.currentIteration.has_value()) + { + + auto iterator = static_cast( + data.series.value().iterations) + .find(*data.currentIteration); + return iterator.operator*(); + } + else + { + throw std::runtime_error("No iteration currently active."); + } } bool StatefulIterator::operator==(StatefulIterator const &other) const @@ -623,7 +620,7 @@ StatefulIterator StatefulIterator::end() return StatefulIterator{}; } -StatefulIterator::operator bool() +StatefulIterator::operator bool() const { return m_data->has_value(); } From ba05ee2ee5c802261fdc12ebf42562df866dacd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 30 Jan 2024 17:02:01 +0100 Subject: [PATCH 29/93] Basically working example for snapshots() in write access --- examples/10_streaming_write.cpp | 2 +- include/openPMD/Iteration.hpp | 1 + include/openPMD/Series.hpp | 1 + include/openPMD/backend/Attributable.hpp | 1 + include/openPMD/snapshots/IteratorTraits.hpp | 1 + .../openPMD/snapshots/StatefulIterator.hpp | 46 +++----- src/Series.cpp | 38 ++++-- src/snapshots/ContainerImpls.cpp | 59 +++++++++- src/snapshots/StatefulIterator.cpp | 108 +++++++++++++----- 9 files changed, 187 insertions(+), 70 deletions(-) diff --git a/examples/10_streaming_write.cpp b/examples/10_streaming_write.cpp index 2eb825ae4a..5704638403 100644 --- a/examples/10_streaming_write.cpp +++ b/examples/10_streaming_write.cpp @@ -43,7 +43,7 @@ int main() // in streaming setups, e.g. an iteration cannot be opened again once // it has been closed. // `Series::iterations` can be directly accessed in random-access workflows. - WriteIterations iterations = series.writeIterations(); + auto iterations = series.snapshots(); for (size_t i = 0; i < 100; ++i) { Iteration iteration = iterations[i]; diff --git a/include/openPMD/Iteration.hpp b/include/openPMD/Iteration.hpp index e15895db94..7fe7ecaf48 100644 --- a/include/openPMD/Iteration.hpp +++ b/include/openPMD/Iteration.hpp @@ -133,6 +133,7 @@ class Iteration : public Attributable template friend T &internal::makeOwning(T &self, Series); friend class StatefulIterator; + friend class StatefulSnapshotsContainer; public: Iteration(Iteration const &) = default; diff --git a/include/openPMD/Series.hpp b/include/openPMD/Series.hpp index de47f6ff0f..a649da2726 100644 --- a/include/openPMD/Series.hpp +++ b/include/openPMD/Series.hpp @@ -257,6 +257,7 @@ class Series : public Attributable friend class internal::SeriesData; friend class internal::AttributableData; friend class WriteIterations; + friend class StatefulSnapshotsContainer; public: explicit Series(); diff --git a/include/openPMD/backend/Attributable.hpp b/include/openPMD/backend/Attributable.hpp index 05b140de32..36f2e290b4 100644 --- a/include/openPMD/backend/Attributable.hpp +++ b/include/openPMD/backend/Attributable.hpp @@ -209,6 +209,7 @@ class Attributable friend void debug::printDirty(Series const &); template friend T &internal::makeOwning(T &self, Series); + friend class StatefulSnapshotsContainer; protected: // tag for internal constructor diff --git a/include/openPMD/snapshots/IteratorTraits.hpp b/include/openPMD/snapshots/IteratorTraits.hpp index 0980ac9c65..01ae319d3a 100644 --- a/include/openPMD/snapshots/IteratorTraits.hpp +++ b/include/openPMD/snapshots/IteratorTraits.hpp @@ -21,6 +21,7 @@ #pragma once #include "openPMD/Iteration.hpp" +#include "openPMD/backend/Writable.hpp" #include namespace openPMD diff --git a/include/openPMD/snapshots/StatefulIterator.hpp b/include/openPMD/snapshots/StatefulIterator.hpp index 640c61326a..2d1829ec91 100644 --- a/include/openPMD/snapshots/StatefulIterator.hpp +++ b/include/openPMD/snapshots/StatefulIterator.hpp @@ -38,6 +38,8 @@ class StatefulIterator StatefulIterator, Container::value_type> { + friend class StatefulSnapshotsContainer; + using iteration_index_t = IndexedIteration::index_t; using maybe_series_t = std::optional; @@ -50,7 +52,7 @@ class StatefulIterator SharedData &operator=(SharedData const &) = delete; SharedData &operator=(SharedData &&) = delete; - maybe_series_t series; + Series series; std::deque iterationsInCurrentStep; // nullopt <-> currently out of step std::optional currentIteration{}; @@ -87,10 +89,20 @@ class StatefulIterator //! construct the end() iterator explicit StatefulIterator(); + class tag_write_t + {}; + static constexpr tag_write_t const tag_write{}; + class tag_read_t + {}; + static constexpr tag_read_t const tag_read{}; + StatefulIterator( + tag_read_t, Series const &, std::optional const &parsePreference); + StatefulIterator(tag_write_t, Series const &); + // dereference using parent_t::operator*; value_type const &operator*() const; @@ -127,34 +139,6 @@ class StatefulIterator operator bool() const; private: - inline bool setCurrentIteration() - { - auto &data = get(); - if (data.iterationsInCurrentStep.empty()) - { - std::cerr << "[ReadIterations] Encountered a step without " - "iterations. Closing the Series." - << std::endl; - *this = end(); - return false; - } - data.currentIteration = *data.iterationsInCurrentStep.begin(); - return true; - } - - inline std::optional peekCurrentIteration() - { - auto &data = get(); - if (data.iterationsInCurrentStep.empty()) - { - return std::nullopt; - } - else - { - return {*data.iterationsInCurrentStep.begin()}; - } - } - std::optional nextIterationInStep(); /* @@ -176,6 +160,10 @@ class StatefulIterator void initSeriesInLinearReadMode(); void close(); + + auto setCurrentIteration() -> bool; + auto peekCurrentIteration() -> std::optional; + auto peekCurrentlyOpenIteration() -> std::optional; }; class LegacyIteratorAdaptor diff --git a/src/Series.cpp b/src/Series.cpp index e12c2d8975..ec918c67aa 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -2984,7 +2984,7 @@ Snapshots Series::snapshots() case Access::READ_WRITE: case Access::CREATE: case Access::APPEND: - // @todo: consider WriteIterations + // @todo: properly distinguish here iterator_kind = IK::Stateful; break; } @@ -3002,17 +3002,37 @@ Snapshots Series::snapshots() Series copied_series; copied_series.setData( std::dynamic_pointer_cast(this->m_attri)); - auto begin = [s = std::move(copied_series)]() mutable { - auto &series_data = s.get(); - if (!series_data.m_sharedReadIterations) + std::function begin; + + // @todo: distinguish read/write access + if (access::write(IOHandler()->m_frontendAccess)) + { + if (!series.m_sharedReadIterations) { - auto parse_preference = series_data.m_parsePreference; - series_data.m_sharedReadIterations = + series.m_sharedReadIterations = std::make_unique( - std::move(s), parse_preference); + StatefulIterator::tag_write, std::move(copied_series)); } - return series_data.m_sharedReadIterations.get(); - }; + begin = [ptr = series.m_sharedReadIterations.get()]() { + return ptr; + }; + } + else + { + begin = [s = std::move(copied_series)]() mutable { + auto &series_data = s.get(); + if (!series_data.m_sharedReadIterations) + { + auto parse_preference = series_data.m_parsePreference; + series_data.m_sharedReadIterations = + std::make_unique( + StatefulIterator::tag_read, + std::move(s), + parse_preference); + } + return series_data.m_sharedReadIterations.get(); + }; + } return Snapshots(std::shared_ptr( new StatefulSnapshotsContainer(std::move(begin)))); diff --git a/src/snapshots/ContainerImpls.cpp b/src/snapshots/ContainerImpls.cpp index f029266ad3..66621946a1 100644 --- a/src/snapshots/ContainerImpls.cpp +++ b/src/snapshots/ContainerImpls.cpp @@ -1,5 +1,6 @@ #include "openPMD/snapshots/ContainerImpls.hpp" #include "openPMD/Error.hpp" +#include "openPMD/IO/Access.hpp" #include "openPMD/snapshots/ContainerTraits.hpp" #include "openPMD/snapshots/StatefulIterator.hpp" #include @@ -103,11 +104,61 @@ auto StatefulSnapshotsContainer::at(key_type const &) -> mapped_type & "container/iterator."); } -auto StatefulSnapshotsContainer::operator[](key_type const &) -> mapped_type & +auto StatefulSnapshotsContainer::operator[](key_type const &key) + -> mapped_type & { - throw std::runtime_error( - "Item access not (yet) implemented on a stateful " - "container/iterator."); + auto it = get(); + auto &shared = it->m_data; + if (!shared || !shared->has_value()) + { + throw error::WrongAPIUsage( + "[WriteIterations] Trying to access after closing Series."); + } + auto &s = shared->value(); + auto access = s.series.IOHandler()->m_frontendAccess; + + // @todo distinguish read_write + if (access::write(access)) + { + auto lastIteration = it->peekCurrentlyOpenIteration(); + if (lastIteration.has_value()) + { + auto lastIteration_v = lastIteration.value(); + if (lastIteration_v.iterationIndex == key) + { + return s.series.iterations.at(key); + } + else + { + lastIteration_v.close(); // continue below + } + } + s.currentIteration = key; + auto &res = s.series.iterations[key]; + if (res.getStepStatus() == StepStatus::NoStep) + { + try + { + res.beginStep(/* reread = */ false); + } + catch (error::OperationUnsupportedInBackend const &) + { + s.series.iterations.retrieveSeries() + .get() + .m_currentlyActiveIterations.clear(); + throw; + } + res.setStepStatus(StepStatus::DuringStep); + } + return res; + } + else if (access::read(access)) + { + throw std::runtime_error( + "Jumping to existing iteration not implemented in stateful " + "container/iterator."); + } + throw error::Internal("Control flow error: This should be unreachable."); } RandomAccessIteratorContainer::RandomAccessIteratorContainer( diff --git a/src/snapshots/StatefulIterator.cpp b/src/snapshots/StatefulIterator.cpp index 3f53ac6671..d0cf74045e 100644 --- a/src/snapshots/StatefulIterator.cpp +++ b/src/snapshots/StatefulIterator.cpp @@ -61,7 +61,7 @@ StatefulIterator::StatefulIterator() = default; void StatefulIterator::initSeriesInLinearReadMode() { auto &data = get(); - auto &series = *data.series; + auto &series = data.series; series.IOHandler()->m_seriesStatus = internal::SeriesStatus::Parsing; try { @@ -122,7 +122,66 @@ void StatefulIterator::close() *m_data = std::nullopt; // turn this into end iterator } +auto StatefulIterator::setCurrentIteration() -> bool +{ + auto &data = get(); + if (data.iterationsInCurrentStep.empty()) + { + std::cerr << "[ReadIterations] Encountered a step without " + "iterations. Closing the Series." + << std::endl; + *this = end(); + return false; + } + data.currentIteration = *data.iterationsInCurrentStep.begin(); + return true; +} + +auto StatefulIterator::peekCurrentIteration() -> std::optional +{ + if (!m_data || !m_data->has_value()) + { + return std::nullopt; + } + auto &data = m_data->value(); + if (data.iterationsInCurrentStep.empty()) + { + return std::nullopt; + } + else + { + return {*data.iterationsInCurrentStep.begin()}; + } +} +auto StatefulIterator::peekCurrentlyOpenIteration() + -> std::optional +{ + if (!m_data || !m_data->has_value()) + { + return std::nullopt; + } + auto &s = m_data->value(); + if (!s.currentIteration.has_value()) + { + return std::nullopt; + } + Iteration ¤tIteration = s.series.iterations.at(*s.currentIteration); + if (currentIteration.closed()) + { + return std::nullopt; + } + return std::make_optional( + IndexedIteration(currentIteration, *s.currentIteration)); +} + +StatefulIterator::StatefulIterator(tag_write_t, Series const &series_in) + : m_data{std::make_shared>(std::in_place)} +{ + m_data->value().series = series_in; +} + StatefulIterator::StatefulIterator( + tag_read_t, Series const &series_in, std::optional const &parsePreference) : m_data{std::make_shared>(std::in_place)} @@ -138,9 +197,9 @@ StatefulIterator::StatefulIterator( * (deleting the original container invalidates the iterator). */ data.series = Series(); - data.series->setData(std::shared_ptr( + data.series.setData(std::shared_ptr( series_in.m_series.get(), [](auto const *) {})); - auto &series = data.series.value(); + auto &series = data.series; if (series.IOHandler()->m_frontendAccess == Access::READ_LINEAR && series.iterations.empty()) { @@ -250,7 +309,7 @@ std::optional StatefulIterator::nextIterationInStep() } auto oldIterationIndex = data.currentIteration; data.currentIteration = *data.iterationsInCurrentStep.begin(); - auto &series = data.series.value(); + auto &series = data.series; switch (series.iterationEncoding()) { @@ -321,7 +380,7 @@ StatefulIterator::nextStep(size_t recursion_depth) { std::tie(status, availableIterations) = Iteration::beginStep( {}, - *data.series, + data.series, /* reread = */ reread(data.parsePreference), data.ignoreIterations); } @@ -330,7 +389,7 @@ StatefulIterator::nextStep(size_t recursion_depth) std::cerr << "[StatefulIterator] Cannot read iteration due to error " "below, will skip it.\n" << err.what() << std::endl; - data.series->advance(AdvanceMode::ENDSTEP); + data.series.advance(AdvanceMode::ENDSTEP); return nextStep(recursion_depth + 1); } @@ -349,7 +408,7 @@ StatefulIterator::nextStep(size_t recursion_depth) throw std::runtime_error( "CATASTROPHE. need to think how to resolve this one."); } - auto &series = data.series.value(); + auto &series = data.series; auto it = series.iterations.find(*data.currentIteration); auto itEnd = series.iterations.end(); if (it == itEnd) @@ -418,7 +477,7 @@ StatefulIterator::nextStep(size_t recursion_depth) std::optional StatefulIterator::loopBody() { auto &data = get(); - Series &series = data.series.value(); + Series &series = data.series; auto &iterations = series.iterations; /* @@ -515,36 +574,31 @@ std::optional StatefulIterator::loopBody() void StatefulIterator::deactivateDeadIteration(iteration_index_t index) { auto &data = get(); - switch (data.series->iterationEncoding()) + switch (data.series.iterationEncoding()) { case IterationEncoding::fileBased: { Parameter param; - data.series->IOHandler()->enqueue( - IOTask(&data.series->iterations[index], std::move(param))); - data.series->IOHandler()->flush({FlushLevel::UserFlush}); + data.series.IOHandler()->enqueue( + IOTask(&data.series.iterations[index], std::move(param))); + data.series.IOHandler()->flush({FlushLevel::UserFlush}); } break; case IterationEncoding::variableBased: case IterationEncoding::groupBased: { Parameter param; param.mode = AdvanceMode::ENDSTEP; - data.series->IOHandler()->enqueue( - IOTask(&data.series->iterations[index], std::move(param))); - data.series->IOHandler()->flush({FlushLevel::UserFlush}); + data.series.IOHandler()->enqueue( + IOTask(&data.series.iterations[index], std::move(param))); + data.series.IOHandler()->flush({FlushLevel::UserFlush}); } break; } - data.series->iterations.container().erase(index); + data.series.iterations.container().erase(index); } StatefulIterator &StatefulIterator::operator++() { auto &data = get(); - if (!data.series.has_value()) - { - this->close(); - return *this; - } auto oldIterationIndex = data.currentIteration; std::optional res; /* @@ -562,7 +616,7 @@ StatefulIterator &StatefulIterator::operator++() auto resvalue = res.value(); if (*resvalue != end()) { - auto &series = data.series.value(); + auto &series = data.series; auto index = data.currentIteration; auto &iteration = series.iterations[index.value()]; iteration.setStepStatus(StepStatus::DuringStep); @@ -595,7 +649,7 @@ auto StatefulIterator::operator*() const -> value_type const & { auto iterator = static_cast( - data.series.value().iterations) + data.series.iterations) .find(*data.currentIteration); return iterator.operator*(); } @@ -635,8 +689,8 @@ ReadIterations::ReadIterations( if (access == Access::READ_LINEAR && !data.m_sharedReadIterations) { // Open the iterator now already, so that metadata may already be read - data.m_sharedReadIterations = - std::make_unique(m_series, m_parsePreference); + data.m_sharedReadIterations = std::make_unique( + StatefulIterator::tag_read, m_series, m_parsePreference); } } @@ -645,8 +699,8 @@ ReadIterations::iterator_t ReadIterations::begin() auto &series = m_series.get(); if (!series.m_sharedReadIterations) { - series.m_sharedReadIterations = - std::make_unique(m_series, m_parsePreference); + series.m_sharedReadIterations = std::make_unique( + StatefulIterator::tag_read, m_series, m_parsePreference); } return *series.m_sharedReadIterations; } From 0c7c04baebbb31f15905ce683beddefeb2474c77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 31 Jan 2024 11:19:24 +0100 Subject: [PATCH 30/93] Extract some methods to .cpp --- .../openPMD/snapshots/StatefulIterator.hpp | 79 ++++--------------- src/snapshots/StatefulIterator.cpp | 69 ++++++++++++++++ 2 files changed, 86 insertions(+), 62 deletions(-) diff --git a/include/openPMD/snapshots/StatefulIterator.hpp b/include/openPMD/snapshots/StatefulIterator.hpp index 2d1829ec91..1f70918b08 100644 --- a/include/openPMD/snapshots/StatefulIterator.hpp +++ b/include/openPMD/snapshots/StatefulIterator.hpp @@ -71,14 +71,8 @@ class StatefulIterator std::shared_ptr> m_data = std::make_shared>(std::nullopt); - SharedData &get() - { - return m_data->value(); - } - SharedData const &get() const - { - return m_data->value(); - } + auto get() -> SharedData &; + auto get() const -> SharedData const &; using parent_t = AbstractSeriesIterator; @@ -108,33 +102,17 @@ class StatefulIterator value_type const &operator*() const; // increment/decrement - StatefulIterator &operator++(); - using parent_t::operator--; - inline StatefulIterator &operator--() - { - throw error::WrongAPIUsage( - "Global stateful iterator supports no decrement (yet)."); - } - StatefulIterator operator++(int) - { - throw error::WrongAPIUsage( - "Global stateful iterator supports no post-increment."); - } + auto operator++() -> StatefulIterator &; + auto operator--() -> StatefulIterator &; + auto operator--(int) -> StatefulIterator; + auto operator++(int) -> StatefulIterator; // comparison - difference_type operator-(StatefulIterator const &) const - { - throw error::WrongAPIUsage( - "Global stateful iterator supports no relative comparison."); - } + auto operator-(StatefulIterator const &) const -> difference_type; bool operator==(StatefulIterator const &other) const; - inline bool operator<(StatefulIterator const &) const - { - throw error::WrongAPIUsage( - "Global stateful iterator supports no relative comparison."); - } + auto operator<(StatefulIterator const &) const -> bool; - static StatefulIterator end(); + static auto end() -> StatefulIterator; operator bool() const; @@ -174,36 +152,14 @@ class LegacyIteratorAdaptor private: friend class ReadIterations; StatefulIterator m_iterator; - LegacyIteratorAdaptor(StatefulIterator iterator) - : m_iterator(std::move(iterator)) - {} + LegacyIteratorAdaptor(StatefulIterator iterator); public: - inline value_type operator*() const - { - return m_iterator.operator*(); - } - - inline LegacyIteratorAdaptor &operator++() - { - ++m_iterator; - return *this; - } - - static LegacyIteratorAdaptor end() - { - return StatefulIterator::end(); - } - - inline bool operator==(LegacyIteratorAdaptor const &other) const - { - return m_iterator == other.m_iterator; - }; - - inline bool operator!=(LegacyIteratorAdaptor const &other) const - { - return m_iterator != other.m_iterator; - }; + value_type operator*() const; + LegacyIteratorAdaptor &operator++(); + static LegacyIteratorAdaptor end(); + bool operator==(LegacyIteratorAdaptor const &other) const; + bool operator!=(LegacyIteratorAdaptor const &other) const; }; /** @@ -239,8 +195,7 @@ class ReadIterations std::optional parsePreference); public: - iterator_t begin(); - - iterator_t end(); + auto begin() -> iterator_t; + auto end() -> iterator_t; }; } // namespace openPMD diff --git a/src/snapshots/StatefulIterator.cpp b/src/snapshots/StatefulIterator.cpp index d0cf74045e..8714cb3f49 100644 --- a/src/snapshots/StatefulIterator.cpp +++ b/src/snapshots/StatefulIterator.cpp @@ -58,6 +58,15 @@ namespace StatefulIterator::StatefulIterator() = default; +auto StatefulIterator::get() -> SharedData & +{ + return m_data->value(); +} +auto StatefulIterator::get() const -> SharedData const & +{ + return m_data->value(); +} + void StatefulIterator::initSeriesInLinearReadMode() { auto &data = get(); @@ -659,6 +668,29 @@ auto StatefulIterator::operator*() const -> value_type const & } } +auto StatefulIterator::operator--() -> StatefulIterator & +{ + throw error::WrongAPIUsage( + "Global stateful iterator supports no decrement (yet)."); +} +auto StatefulIterator::operator--(int) -> StatefulIterator +{ + throw error::WrongAPIUsage( + "Global stateful iterator supports no post-decrement."); +} +auto StatefulIterator::operator++(int) -> StatefulIterator +{ + throw error::WrongAPIUsage( + "Global stateful iterator supports no post-increment."); +} + +// comparison +auto StatefulIterator::operator-(StatefulIterator const &) const + -> difference_type +{ + throw error::WrongAPIUsage( + "Global stateful iterator supports no relative comparison."); +} bool StatefulIterator::operator==(StatefulIterator const &other) const { return @@ -668,6 +700,11 @@ bool StatefulIterator::operator==(StatefulIterator const &other) const // or both are empty (!this->m_data->has_value() && !other.m_data->has_value()); } +auto StatefulIterator::operator<(StatefulIterator const &) const -> bool +{ + throw error::WrongAPIUsage( + "Global stateful iterator supports no relative comparison."); +} StatefulIterator StatefulIterator::end() { @@ -679,6 +716,38 @@ StatefulIterator::operator bool() const return m_data->has_value(); } +LegacyIteratorAdaptor::LegacyIteratorAdaptor(StatefulIterator iterator) + : m_iterator(std::move(iterator)) +{} + +auto LegacyIteratorAdaptor::operator*() const -> value_type +{ + return m_iterator.operator*(); +} + +auto LegacyIteratorAdaptor::operator++() -> LegacyIteratorAdaptor & +{ + ++m_iterator; + return *this; +} + +auto LegacyIteratorAdaptor::end() -> LegacyIteratorAdaptor +{ + return StatefulIterator::end(); +} + +auto LegacyIteratorAdaptor::operator==(LegacyIteratorAdaptor const &other) const + -> bool +{ + return m_iterator == other.m_iterator; +}; + +auto LegacyIteratorAdaptor::operator!=(LegacyIteratorAdaptor const &other) const + -> bool +{ + return m_iterator != other.m_iterator; +}; + ReadIterations::ReadIterations( Series series, Access access, From 559658b0655156eadddd317683ad73946386df59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 31 Jan 2024 13:05:31 +0100 Subject: [PATCH 31/93] Fully replace WriteIterations class with the new one --- CMakeLists.txt | 1 - include/openPMD/Iteration.hpp | 4 +- include/openPMD/Series.hpp | 10 -- include/openPMD/WriteIterations.hpp | 109 --------------- include/openPMD/backend/Attributable.hpp | 1 - include/openPMD/openPMD.hpp | 1 - include/openPMD/snapshots/ContainerImpls.hpp | 4 + include/openPMD/snapshots/ContainerTraits.hpp | 3 + include/openPMD/snapshots/Snapshots.hpp | 9 ++ .../openPMD/snapshots/StatefulIterator.hpp | 12 +- src/Series.cpp | 78 ++++++----- src/WriteIterations.cpp | 124 ------------------ src/binding/python/Series.cpp | 22 +++- src/snapshots/ContainerImpls.cpp | 10 +- src/snapshots/ContainerTraits.cpp | 21 +++ src/snapshots/Snapshots.cpp | 34 +++-- src/snapshots/StatefulIterator.cpp | 57 +++++++- 17 files changed, 193 insertions(+), 307 deletions(-) delete mode 100644 include/openPMD/WriteIterations.hpp delete mode 100644 src/WriteIterations.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 50af59f825..7c4cf15ee0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -463,7 +463,6 @@ set(CORE_SOURCE src/RecordComponent.cpp src/Series.cpp src/version.cpp - src/WriteIterations.cpp src/auxiliary/Date.cpp src/auxiliary/Filesystem.cpp src/auxiliary/JSON.cpp diff --git a/include/openPMD/Iteration.hpp b/include/openPMD/Iteration.hpp index 7fe7ecaf48..c594860a92 100644 --- a/include/openPMD/Iteration.hpp +++ b/include/openPMD/Iteration.hpp @@ -128,7 +128,6 @@ class Iteration : public Attributable template friend class Container; friend class Series; - friend class WriteIterations; friend class internal::AttributableData; template friend T &internal::makeOwning(T &self, Series); @@ -437,18 +436,17 @@ inline T Iteration::dt() const class IndexedIteration : public Iteration { friend class StatefulIterator; - friend class WriteIterations; friend class LegacyIteratorAdaptor; public: using index_t = Iteration::IterationIndex_t; index_t const iterationIndex; -private: inline IndexedIteration(std::pair pair) : Iteration(std::move(pair.second)), iterationIndex(pair.first) {} +private: template IndexedIteration(Iteration_t &&it, index_t index) : Iteration(std::forward(it)), iterationIndex(index) diff --git a/include/openPMD/Series.hpp b/include/openPMD/Series.hpp index a649da2726..e36c610bec 100644 --- a/include/openPMD/Series.hpp +++ b/include/openPMD/Series.hpp @@ -27,7 +27,6 @@ #include "openPMD/Iteration.hpp" #include "openPMD/IterationEncoding.hpp" #include "openPMD/Streaming.hpp" -#include "openPMD/WriteIterations.hpp" #include "openPMD/auxiliary/Variant.hpp" #include "openPMD/backend/Attributable.hpp" #include "openPMD/backend/Container.hpp" @@ -94,14 +93,6 @@ namespace internal using IterationsContainer_t = Container; IterationsContainer_t iterations{}; - /** - * For each instance of Series, there is only one instance - * of WriteIterations, stored in this Option. - * This ensures that Series::writeIteration() always returns - * the same instance. - */ - std::optional m_writeIterations; - /** * Series::readIterations() returns an iterator type that modifies the * state of the Series (by proceeding through IO steps). @@ -256,7 +247,6 @@ class Series : public Attributable friend class StatefulIterator; friend class internal::SeriesData; friend class internal::AttributableData; - friend class WriteIterations; friend class StatefulSnapshotsContainer; public: diff --git a/include/openPMD/WriteIterations.hpp b/include/openPMD/WriteIterations.hpp deleted file mode 100644 index 41967711aa..0000000000 --- a/include/openPMD/WriteIterations.hpp +++ /dev/null @@ -1,109 +0,0 @@ -/* Copyright 2021 Franz Poeschel - * - * This file is part of openPMD-api. - * - * openPMD-api is free software: you can redistribute it and/or modify - * it under the terms of of either the GNU General Public License or - * the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * openPMD-api is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License and the GNU Lesser General Public License - * for more details. - * - * You should have received a copy of the GNU General Public License - * and the GNU Lesser General Public License along with openPMD-api. - * If not, see . - */ -#pragma once - -#include "openPMD/Iteration.hpp" -#include "openPMD/backend/Container.hpp" - -#include - -namespace openPMD -{ -class Series; - -/** Writing side of the streaming API. - * - * Create instance via Series::writeIterations(). - * For use via WriteIterations::operator[](). - * Designed to allow reading any kind of Series, streaming and non- - * streaming alike. Calling Iteration::close() manually before opening - * the next iteration is encouraged and will implicitly flush all - * deferred IO actions. Otherwise, Iteration::close() will be implicitly - * called upon StatefulIterator::operator++(), i.e. upon going to the next - * iteration in the foreach loop. - * - * Since this is designed for streaming mode, reopening an iteration is - * not possible once it has been closed. - * - */ - -namespace internal -{ - class SeriesData; -} - -/** - * @brief Writing side of the streaming API. - * - * Create instance via Series::writeIterations(). - * Restricted Container of Iterations, designed to allow reading any kind - * of Series, streaming and non-streaming alike. - * Calling Iteration::close() manually before opening the next iteration is - * encouraged and will implicitly flush all deferred IO actions. - * Otherwise, Iteration::close() will be implicitly called upon - * opening the next iteration or upon destruction. - * Since this is designed for streaming mode, reopening an iteration is - * not possible once it has been closed. - */ -class WriteIterations -{ - friend class Series; - friend class internal::SeriesData; - -private: - using IterationsContainer_t = - Container; - -public: - using key_type = IterationsContainer_t::key_type; - using mapped_type = IterationsContainer_t::mapped_type; - using value_type = IterationsContainer_t::value_type; - using reference = IterationsContainer_t::reference; - -private: - struct SharedResources - { - IterationsContainer_t iterations; - //! Index of the last opened iteration - std::optional currentlyOpen; - - SharedResources(IterationsContainer_t); - ~SharedResources(); - }; - - WriteIterations(IterationsContainer_t); - explicit WriteIterations() = default; - // std::optional so that a single instance is able to close this without - // needing to wait for all instances to deallocate - std::shared_ptr> shared; - - void close(); - -public: - mapped_type &operator[](key_type const &key); - mapped_type &operator[](key_type &&key); - - /** - * Return the iteration that is currently being written to, if it exists. - */ - std::optional currentIteration(); -}; -} // namespace openPMD diff --git a/include/openPMD/backend/Attributable.hpp b/include/openPMD/backend/Attributable.hpp index 36f2e290b4..8117081128 100644 --- a/include/openPMD/backend/Attributable.hpp +++ b/include/openPMD/backend/Attributable.hpp @@ -204,7 +204,6 @@ class Attributable friend class Iteration; friend class Series; friend class Writable; - friend class WriteIterations; friend class internal::RecordComponentData; friend void debug::printDirty(Series const &); template diff --git a/include/openPMD/openPMD.hpp b/include/openPMD/openPMD.hpp index a9ec6e92b7..d29bcdd818 100644 --- a/include/openPMD/openPMD.hpp +++ b/include/openPMD/openPMD.hpp @@ -38,7 +38,6 @@ namespace openPMD #include "openPMD/RecordComponent.hpp" #include "openPMD/Series.hpp" #include "openPMD/UnitDimension.hpp" -#include "openPMD/WriteIterations.hpp" #include "openPMD/snapshots/StatefulIterator.hpp" #include "openPMD/backend/Attributable.hpp" diff --git a/include/openPMD/snapshots/ContainerImpls.hpp b/include/openPMD/snapshots/ContainerImpls.hpp index 913fb53d3a..adc14c1be7 100644 --- a/include/openPMD/snapshots/ContainerImpls.hpp +++ b/include/openPMD/snapshots/ContainerImpls.hpp @@ -3,6 +3,7 @@ #include "openPMD/snapshots/ContainerTraits.hpp" #include "openPMD/snapshots/RandomAccessIterator.hpp" #include "openPMD/snapshots/StatefulIterator.hpp" +#include namespace openPMD { @@ -19,6 +20,9 @@ class StatefulSnapshotsContainer : public AbstractSnapshotsContainer auto get() const -> StatefulIterator const *; public: + using AbstractSnapshotsContainer::currentIteration; + auto currentIteration() const -> std::optional override; + auto begin() -> iterator override; auto end() -> iterator override; auto begin() const -> const_iterator override; diff --git a/include/openPMD/snapshots/ContainerTraits.hpp b/include/openPMD/snapshots/ContainerTraits.hpp index 3972c3f7f8..94faf919d1 100644 --- a/include/openPMD/snapshots/ContainerTraits.hpp +++ b/include/openPMD/snapshots/ContainerTraits.hpp @@ -58,6 +58,9 @@ class AbstractSnapshotsContainer using reverse_iterator = OpaqueSeriesIterator; using const_reverse_iterator = OpaqueSeriesIterator; + virtual auto currentIteration() -> std::optional; + virtual auto currentIteration() const -> std::optional; + virtual auto begin() -> iterator = 0; virtual auto begin() const -> const_iterator = 0; virtual auto end() -> iterator = 0; diff --git a/include/openPMD/snapshots/Snapshots.hpp b/include/openPMD/snapshots/Snapshots.hpp index 94711a7e2e..7150de82be 100644 --- a/include/openPMD/snapshots/Snapshots.hpp +++ b/include/openPMD/snapshots/Snapshots.hpp @@ -38,6 +38,9 @@ class Snapshots Snapshots(std::shared_ptr snapshots); + inline auto get() const -> AbstractSnapshotsContainer const &; + inline auto get() -> AbstractSnapshotsContainer &; + public: using key_type = AbstractSnapshotsContainer::key_type; using value_type = AbstractSnapshotsContainer::value_type; @@ -50,6 +53,9 @@ class Snapshots using const_reverse_iterator = AbstractSnapshotsContainer::const_reverse_iterator; + auto currentIteration() -> std::optional; + auto currentIteration() const -> std::optional; + iterator begin(); iterator end(); const_iterator begin() const; @@ -66,4 +72,7 @@ class Snapshots mapped_type &operator[](key_type const &key); }; + +// backwards compatibility +using WriteIterations = Snapshots; } // namespace openPMD diff --git a/include/openPMD/snapshots/StatefulIterator.hpp b/include/openPMD/snapshots/StatefulIterator.hpp index 1f70918b08..e7a57c1134 100644 --- a/include/openPMD/snapshots/StatefulIterator.hpp +++ b/include/openPMD/snapshots/StatefulIterator.hpp @@ -33,12 +33,18 @@ namespace openPMD { +namespace internal +{ + class SeriesData; +} class StatefulIterator : public AbstractSeriesIterator< StatefulIterator, Container::value_type> { friend class StatefulSnapshotsContainer; + friend class Series; + friend class internal::SeriesData; using iteration_index_t = IndexedIteration::index_t; @@ -52,6 +58,8 @@ class StatefulIterator SharedData &operator=(SharedData const &) = delete; SharedData &operator=(SharedData &&) = delete; + ~SharedData(); + Series series; std::deque iterationsInCurrentStep; // nullopt <-> currently out of step @@ -141,7 +149,9 @@ class StatefulIterator auto setCurrentIteration() -> bool; auto peekCurrentIteration() -> std::optional; - auto peekCurrentlyOpenIteration() -> std::optional; + auto peekCurrentlyOpenIteration() const + -> std::optional; + auto peekCurrentlyOpenIteration() -> std::optional; }; class LegacyIteratorAdaptor diff --git a/src/Series.cpp b/src/Series.cpp index ec918c67aa..8b3613472d 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -2855,9 +2855,9 @@ namespace internal void SeriesData::close() { // WriteIterations gets the first shot at flushing - if (this->m_writeIterations.has_value()) + if (this->m_sharedReadIterations) { - this->m_writeIterations.value().close(); + this->m_sharedReadIterations->close(); } /* * Scenario: A user calls `Series::flush()` but does not check for @@ -2945,9 +2945,45 @@ namespace }; } +namespace +{ + auto make_writing_stateful_iterator( + Series copied_series, internal::SeriesData &series) + -> std::function + { + if (!series.m_sharedReadIterations) + { + series.m_sharedReadIterations = std::make_unique( + StatefulIterator::tag_write, std::move(copied_series)); + } + return [ptr = series.m_sharedReadIterations.get()]() { return ptr; }; + } + auto make_reading_stateful_iterator( + Series copied_series, internal::SeriesData &series) + -> std::function + { + return [s = std::move(copied_series), &series]() mutable { + if (!series.m_sharedReadIterations) + { + auto parse_preference = series.m_parsePreference; + series.m_sharedReadIterations = + std::make_unique( + StatefulIterator::tag_read, + std::move(s), + parse_preference); + } + return series.m_sharedReadIterations.get(); + }; + } +} // namespace + Snapshots Series::snapshots() { auto &series = get(); + if (series.m_deferred_initialization.has_value()) + { + runDeferredInitialization(); + } IteratorKind iterator_kind{}; { using IK = IteratorKind; @@ -2997,43 +3033,17 @@ Snapshots Series::snapshots() new RandomAccessIteratorContainer(series.iterations)}); } case IteratorKind::Stateful: { - // Use private constructor instead of copy constructor to avoid - // object slicing - Series copied_series; - copied_series.setData( - std::dynamic_pointer_cast(this->m_attri)); std::function begin; // @todo: distinguish read/write access if (access::write(IOHandler()->m_frontendAccess)) { - if (!series.m_sharedReadIterations) - { - series.m_sharedReadIterations = - std::make_unique( - StatefulIterator::tag_write, std::move(copied_series)); - } - begin = [ptr = series.m_sharedReadIterations.get()]() { - return ptr; - }; + begin = make_writing_stateful_iterator(*this, series); } else { - begin = [s = std::move(copied_series)]() mutable { - auto &series_data = s.get(); - if (!series_data.m_sharedReadIterations) - { - auto parse_preference = series_data.m_parsePreference; - series_data.m_sharedReadIterations = - std::make_unique( - StatefulIterator::tag_read, - std::move(s), - parse_preference); - } - return series_data.m_sharedReadIterations.get(); - }; + begin = make_reading_stateful_iterator(*this, series); } - return Snapshots(std::shared_ptr( new StatefulSnapshotsContainer(std::move(begin)))); } @@ -3049,15 +3059,13 @@ void Series::parseBase() WriteIterations Series::writeIterations() { auto &series = get(); - if (!series.m_writeIterations.has_value()) - { - series.m_writeIterations = WriteIterations(this->iterations); - } if (series.m_deferred_initialization.has_value()) { runDeferredInitialization(); } - return series.m_writeIterations.value(); + auto begin = make_writing_stateful_iterator(*this, series); + return Snapshots(std::shared_ptr( + new StatefulSnapshotsContainer(std::move(begin)))); } void Series::close() diff --git a/src/WriteIterations.cpp b/src/WriteIterations.cpp deleted file mode 100644 index 0ae7246ae0..0000000000 --- a/src/WriteIterations.cpp +++ /dev/null @@ -1,124 +0,0 @@ -/* Copyright 2021 Franz Poeschel - * - * This file is part of openPMD-api. - * - * openPMD-api is free software: you can redistribute it and/or modify - * it under the terms of of either the GNU General Public License or - * the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * openPMD-api is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License and the GNU Lesser General Public License - * for more details. - * - * You should have received a copy of the GNU General Public License - * and the GNU Lesser General Public License along with openPMD-api. - * If not, see . - */ - -#include "openPMD/WriteIterations.hpp" -#include "openPMD/Error.hpp" - -#include "openPMD/Series.hpp" - -namespace openPMD -{ -WriteIterations::SharedResources::SharedResources( - IterationsContainer_t _iterations) - : iterations(std::move(_iterations)) -{} - -WriteIterations::SharedResources::~SharedResources() -{ - if (auto IOHandler = iterations.IOHandler(); currentlyOpen.has_value() && - IOHandler && IOHandler->m_lastFlushSuccessful) - { - auto lastIterationIndex = currentlyOpen.value(); - auto &lastIteration = iterations.at(lastIterationIndex); - if (!lastIteration.closed()) - { - lastIteration.close(); - } - } -} - -WriteIterations::WriteIterations(IterationsContainer_t iterations) - : shared{std::make_shared>( - std::move(iterations))} -{} - -void WriteIterations::close() -{ - *shared = std::nullopt; -} - -WriteIterations::mapped_type &WriteIterations::operator[](key_type const &key) -{ - // make a copy - // explicit cast so MSVC can figure out how to do it correctly - return operator[](static_cast(key_type{key})); -} -WriteIterations::mapped_type &WriteIterations::operator[](key_type &&key) -{ - if (!shared || !shared->has_value()) - { - throw error::WrongAPIUsage( - "[WriteIterations] Trying to access after closing Series."); - } - auto &s = shared->value(); - auto lastIteration = currentIteration(); - if (lastIteration.has_value()) - { - auto lastIteration_v = lastIteration.value(); - if (lastIteration_v.iterationIndex == key) - { - return s.iterations.at(std::forward(key)); - } - else - { - lastIteration_v.close(); // continue below - } - } - s.currentlyOpen = key; - auto &res = s.iterations[std::forward(key)]; - if (res.getStepStatus() == StepStatus::NoStep) - { - try - { - res.beginStep(/* reread = */ false); - } - catch (error::OperationUnsupportedInBackend const &) - { - s.iterations.retrieveSeries() - .get() - .m_currentlyActiveIterations.clear(); - throw; - } - res.setStepStatus(StepStatus::DuringStep); - } - return res; -} - -std::optional WriteIterations::currentIteration() -{ - if (!shared || !shared->has_value()) - { - return std::nullopt; - } - auto &s = shared->value(); - if (!s.currentlyOpen.has_value()) - { - return std::nullopt; - } - Iteration ¤tIteration = s.iterations.at(s.currentlyOpen.value()); - if (currentIteration.closed()) - { - return std::nullopt; - } - return std::make_optional( - IndexedIteration(currentIteration, s.currentlyOpen.value())); -} -} // namespace openPMD diff --git a/src/binding/python/Series.cpp b/src/binding/python/Series.cpp index 882ffbe500..88c40b9640 100644 --- a/src/binding/python/Series.cpp +++ b/src/binding/python/Series.cpp @@ -20,13 +20,16 @@ */ #include "openPMD/Series.hpp" #include "openPMD/IO/Access.hpp" +#include "openPMD/Iteration.hpp" #include "openPMD/IterationEncoding.hpp" #include "openPMD/auxiliary/JSON.hpp" #include "openPMD/binding/python/Pickle.hpp" #include "openPMD/config.hpp" +#include "openPMD/snapshots/Snapshots.hpp" #include "openPMD/snapshots/StatefulIterator.hpp" #include "openPMD/binding/python/Common.hpp" +#include #if openPMD_HAVE_MPI // re-implemented signatures: @@ -72,13 +75,13 @@ not possible once it has been closed. )END") .def( "__getitem__", - [](WriteIterations writeIterations, Series::IterationIndex_t key) { + [](WriteIterations &writeIterations, Series::IterationIndex_t key) { auto lastIteration = writeIterations.currentIteration(); if (lastIteration.has_value() && - lastIteration.value().iterationIndex != key) + lastIteration.value()->first != key) { // this must happen under the GIL - lastIteration.value().close(); + lastIteration.value()->second.close(); } py::gil_scoped_release release; return writeIterations[key]; @@ -87,7 +90,18 @@ not possible once it has been closed. py::return_value_policy::copy) .def( "current_iteration", - &WriteIterations::currentIteration, + [](WriteIterations &writeIterations) + -> std::optional { + if (auto currentIteration = writeIterations.currentIteration(); + currentIteration.has_value()) + { + return IndexedIteration(**currentIteration); + } + else + { + return std::nullopt; + } + }, "Return the iteration that is currently being written to, if it " "exists."); diff --git a/src/snapshots/ContainerImpls.cpp b/src/snapshots/ContainerImpls.cpp index 66621946a1..11524d5eec 100644 --- a/src/snapshots/ContainerImpls.cpp +++ b/src/snapshots/ContainerImpls.cpp @@ -4,6 +4,7 @@ #include "openPMD/snapshots/ContainerTraits.hpp" #include "openPMD/snapshots/StatefulIterator.hpp" #include +#include #include namespace openPMD @@ -38,6 +39,11 @@ auto StatefulSnapshotsContainer::get() const -> StatefulIterator const * { return m_bufferedIterator.value_or(nullptr); } +auto StatefulSnapshotsContainer::currentIteration() const + -> std::optional +{ + return get()->peekCurrentlyOpenIteration(); +} auto StatefulSnapshotsContainer::begin() -> iterator { return stateful_to_opaque(*get()); @@ -124,13 +130,13 @@ auto StatefulSnapshotsContainer::operator[](key_type const &key) if (lastIteration.has_value()) { auto lastIteration_v = lastIteration.value(); - if (lastIteration_v.iterationIndex == key) + if (lastIteration_v->first == key) { return s.series.iterations.at(key); } else { - lastIteration_v.close(); // continue below + lastIteration_v->second.close(); // continue below } } s.currentIteration = key; diff --git a/src/snapshots/ContainerTraits.cpp b/src/snapshots/ContainerTraits.cpp index 8392ea9f0d..547b3020b5 100644 --- a/src/snapshots/ContainerTraits.cpp +++ b/src/snapshots/ContainerTraits.cpp @@ -1,5 +1,6 @@ #include "openPMD/snapshots/ContainerTraits.hpp" #include "openPMD/Iteration.hpp" +#include namespace openPMD { @@ -84,6 +85,26 @@ using value_type = template class OpaqueSeriesIterator; template class OpaqueSeriesIterator; +auto AbstractSnapshotsContainer::currentIteration() + -> std::optional +{ + if (auto maybe_value = static_cast(this) + ->currentIteration(); + maybe_value.has_value()) + { + return {const_cast(*maybe_value)}; + } + else + { + return std::nullopt; + } +} +auto AbstractSnapshotsContainer::currentIteration() const + -> std::optional +{ + return std::nullopt; +} + auto AbstractSnapshotsContainer::at(key_type const &key) -> mapped_type & { return const_cast( diff --git a/src/snapshots/Snapshots.cpp b/src/snapshots/Snapshots.cpp index ce9001d2d1..9f8905be77 100644 --- a/src/snapshots/Snapshots.cpp +++ b/src/snapshots/Snapshots.cpp @@ -4,13 +4,29 @@ namespace openPMD Snapshots::Snapshots(std::shared_ptr snapshots) : m_snapshots(std::move(snapshots)) {} +inline auto Snapshots::get() const -> AbstractSnapshotsContainer const & +{ + return *m_snapshots; +} +inline auto Snapshots::get() -> AbstractSnapshotsContainer & +{ + return *m_snapshots; +} +auto Snapshots::currentIteration() -> std::optional +{ + return get().currentIteration(); +} +auto Snapshots::currentIteration() const -> std::optional +{ + return get().currentIteration(); +} auto Snapshots::begin() -> iterator { - return m_snapshots->begin(); + return get().begin(); } auto Snapshots::end() -> iterator { - return m_snapshots->end(); + return get().end(); } auto Snapshots::begin() const -> const_iterator { @@ -23,11 +39,11 @@ auto Snapshots::end() const -> const_iterator } auto Snapshots::rbegin() -> reverse_iterator { - return m_snapshots->begin(); + return get().begin(); } auto Snapshots::rend() -> reverse_iterator { - return m_snapshots->end(); + return get().end(); } auto Snapshots::rbegin() const -> const_reverse_iterator { @@ -36,24 +52,24 @@ auto Snapshots::rbegin() const -> const_reverse_iterator } auto Snapshots::rend() const -> const_reverse_iterator { - return static_cast(*m_snapshots).end(); + return get().end(); } bool Snapshots::empty() const { - return m_snapshots->empty(); + return get().empty(); } auto Snapshots::at(key_type const &key) const -> mapped_type const & { - return m_snapshots->at(key); + return get().at(key); } auto Snapshots::at(key_type const &key) -> mapped_type & { - return m_snapshots->at(key); + return get().at(key); } auto Snapshots::operator[](key_type const &key) -> mapped_type & { - return m_snapshots->operator[](key); + return get().operator[](key); } } // namespace openPMD diff --git a/src/snapshots/StatefulIterator.cpp b/src/snapshots/StatefulIterator.cpp index 8714cb3f49..c14f688205 100644 --- a/src/snapshots/StatefulIterator.cpp +++ b/src/snapshots/StatefulIterator.cpp @@ -30,6 +30,19 @@ namespace openPMD { +StatefulIterator::SharedData::~SharedData() +{ + if (auto IOHandler = series.IOHandler(); currentIteration.has_value() && + IOHandler && IOHandler->m_lastFlushSuccessful) + { + auto lastIterationIndex = currentIteration.value(); + auto &lastIteration = series.iterations.at(lastIterationIndex); + if (!lastIteration.closed()) + { + lastIteration.close(); + } + } +} namespace { @@ -162,8 +175,8 @@ auto StatefulIterator::peekCurrentIteration() -> std::optional return {*data.iterationsInCurrentStep.begin()}; } } -auto StatefulIterator::peekCurrentlyOpenIteration() - -> std::optional +auto StatefulIterator::peekCurrentlyOpenIteration() const + -> std::optional { if (!m_data || !m_data->has_value()) { @@ -174,19 +187,49 @@ auto StatefulIterator::peekCurrentlyOpenIteration() { return std::nullopt; } - Iteration ¤tIteration = s.series.iterations.at(*s.currentIteration); - if (currentIteration.closed()) + // Iteration ¤tIteration = + // s.series.iterations.at(*s.currentIteration); + auto currentIteration = s.series.iterations.find(*s.currentIteration); + if (currentIteration == s.series.iterations.end()) + { + return std::nullopt; + } + if (currentIteration->second.closed()) + { + return std::nullopt; + } + return std::make_optional(&*currentIteration); +} +auto StatefulIterator::peekCurrentlyOpenIteration() + -> std::optional +{ + if (auto res = static_cast(this) + ->peekCurrentlyOpenIteration(); + res.has_value()) + { + return {const_cast(*res)}; + } + else { return std::nullopt; } - return std::make_optional( - IndexedIteration(currentIteration, *s.currentIteration)); } StatefulIterator::StatefulIterator(tag_write_t, Series const &series_in) : m_data{std::make_shared>(std::in_place)} { - m_data->value().series = series_in; + auto &data = get(); + /* + * Since the iterator is stored in + * internal::SeriesData::m_sharedReadIterations, + * we need to use a non-owning Series instance here for tie-breaking + * purposes. + * This is ok due to the usual C++ iterator invalidation workflows + * (deleting the original container invalidates the iterator). + */ + data.series = Series(); + data.series.setData(std::shared_ptr( + series_in.m_series.get(), [](auto const *) {})); } StatefulIterator::StatefulIterator( From 0e8e994320fe51a6baa28ee4aa4c8c123db1b66e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 31 Jan 2024 15:09:25 +0100 Subject: [PATCH 32/93] Fix nullpointer issue --- include/openPMD/snapshots/ContainerImpls.hpp | 2 +- src/snapshots/ContainerImpls.cpp | 24 ++++++++++++++++++-- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/include/openPMD/snapshots/ContainerImpls.hpp b/include/openPMD/snapshots/ContainerImpls.hpp index adc14c1be7..510fe1b768 100644 --- a/include/openPMD/snapshots/ContainerImpls.hpp +++ b/include/openPMD/snapshots/ContainerImpls.hpp @@ -20,7 +20,7 @@ class StatefulSnapshotsContainer : public AbstractSnapshotsContainer auto get() const -> StatefulIterator const *; public: - using AbstractSnapshotsContainer::currentIteration; + auto currentIteration() -> std::optional override; auto currentIteration() const -> std::optional override; auto begin() -> iterator override; diff --git a/src/snapshots/ContainerImpls.cpp b/src/snapshots/ContainerImpls.cpp index 11524d5eec..5719d125d1 100644 --- a/src/snapshots/ContainerImpls.cpp +++ b/src/snapshots/ContainerImpls.cpp @@ -39,10 +39,29 @@ auto StatefulSnapshotsContainer::get() const -> StatefulIterator const * { return m_bufferedIterator.value_or(nullptr); } +auto StatefulSnapshotsContainer::currentIteration() + -> std::optional +{ + if (auto it = get(); it) + { + return it->peekCurrentlyOpenIteration(); + } + else + { + return nullptr; + } +} auto StatefulSnapshotsContainer::currentIteration() const -> std::optional { - return get()->peekCurrentlyOpenIteration(); + if (auto it = get(); it) + { + return it->peekCurrentlyOpenIteration(); + } + else + { + return nullptr; + } } auto StatefulSnapshotsContainer::begin() -> iterator { @@ -93,7 +112,8 @@ auto StatefulSnapshotsContainer::rend() const -> const_iterator bool StatefulSnapshotsContainer::empty() const { - return get()->operator bool(); + auto it = get(); + return (!it) || it->operator bool(); } auto StatefulSnapshotsContainer::at(key_type const &) const From b9b71876bb07e0b7dc833db6f0ee768218298907 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 6 Feb 2024 17:02:18 +0100 Subject: [PATCH 33/93] Little fixes --- src/Series.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/Series.cpp b/src/Series.cpp index 8b3613472d..df45145523 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -2948,13 +2948,13 @@ namespace namespace { auto make_writing_stateful_iterator( - Series copied_series, internal::SeriesData &series) + Series const &copied_series, internal::SeriesData &series) -> std::function { if (!series.m_sharedReadIterations) { series.m_sharedReadIterations = std::make_unique( - StatefulIterator::tag_write, std::move(copied_series)); + StatefulIterator::tag_write, copied_series); } return [ptr = series.m_sharedReadIterations.get()]() { return ptr; }; } @@ -3001,6 +3001,10 @@ Snapshots Series::snapshots() iterator_kind = IK::RandomAccess; break; case internal::ParsePreference::PerStep: + std::cerr + << "[Warning] Series: Use READ_LINEAR access mode to " + "access Series that requires collective processing." + << std::endl; iterator_kind = IK::Stateful; break; } From 05f9c2712d35c0ddd29e5333a441f6c23bae698e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 6 Feb 2024 17:52:19 +0100 Subject: [PATCH 34/93] Add some further API calls --- include/openPMD/snapshots/ContainerImpls.hpp | 16 +++++ include/openPMD/snapshots/ContainerTraits.hpp | 8 +++ include/openPMD/snapshots/Snapshots.hpp | 18 ++++- src/snapshots/ContainerImpls.cpp | 69 +++++++++++++++++-- src/snapshots/Snapshots.cpp | 29 ++++++++ 5 files changed, 133 insertions(+), 7 deletions(-) diff --git a/include/openPMD/snapshots/ContainerImpls.hpp b/include/openPMD/snapshots/ContainerImpls.hpp index 510fe1b768..eaa589aaa4 100644 --- a/include/openPMD/snapshots/ContainerImpls.hpp +++ b/include/openPMD/snapshots/ContainerImpls.hpp @@ -33,11 +33,19 @@ class StatefulSnapshotsContainer : public AbstractSnapshotsContainer auto rend() const -> const_iterator override; auto empty() const -> bool override; + auto size() const -> size_t override; auto at(key_type const &key) const -> mapped_type const & override; auto at(key_type const &key) -> mapped_type & override; auto operator[](key_type const &key) -> mapped_type & override; + + auto clear() -> void override; + + auto find(key_type const &key) -> iterator override; + auto find(key_type const &key) const -> const_iterator override; + + auto contains(key_type const &key) const -> bool override; }; /* @@ -63,9 +71,17 @@ class RandomAccessIteratorContainer : public AbstractSnapshotsContainer auto rend() const -> const_iterator override; auto empty() const -> bool override; + auto size() const -> size_t override; using AbstractSnapshotsContainer::at; auto at(key_type const &key) const -> mapped_type const & override; auto operator[](key_type const &key) -> mapped_type & override; + + auto clear() -> void override; + + auto find(key_type const &key) -> iterator override; + auto find(key_type const &key) const -> const_iterator override; + + auto contains(key_type const &key) const -> bool override; }; } // namespace openPMD diff --git a/include/openPMD/snapshots/ContainerTraits.hpp b/include/openPMD/snapshots/ContainerTraits.hpp index 94faf919d1..8c7b278ecc 100644 --- a/include/openPMD/snapshots/ContainerTraits.hpp +++ b/include/openPMD/snapshots/ContainerTraits.hpp @@ -71,10 +71,18 @@ class AbstractSnapshotsContainer virtual auto rend() const -> const_reverse_iterator = 0; virtual auto empty() const -> bool = 0; + virtual auto size() const -> size_t = 0; virtual auto at(key_type const &key) const -> mapped_type const & = 0; virtual auto at(key_type const &key) -> mapped_type &; virtual auto operator[](key_type const &key) -> mapped_type & = 0; + + virtual auto clear() -> void = 0; + + virtual auto find(key_type const &key) -> iterator = 0; + virtual auto find(key_type const &key) const -> const_iterator = 0; + + virtual auto contains(key_type const &key) const -> bool = 0; }; } // namespace openPMD diff --git a/include/openPMD/snapshots/Snapshots.hpp b/include/openPMD/snapshots/Snapshots.hpp index 7150de82be..98f0725f02 100644 --- a/include/openPMD/snapshots/Snapshots.hpp +++ b/include/openPMD/snapshots/Snapshots.hpp @@ -65,12 +65,28 @@ class Snapshots const_reverse_iterator rbegin() const; const_reverse_iterator rend() const; - bool empty() const; + auto empty() const -> bool; + auto size() const -> size_t; mapped_type const &at(key_type const &key) const; mapped_type &at(key_type const &key); mapped_type &operator[](key_type const &key); + + auto clear() -> void; + + // insert + // swap + + auto find(key_type const &key) -> iterator; + auto find(key_type const &key) const -> const_iterator; + + auto count(key_type const &key) const -> size_t; + + auto contains(key_type const &key) const -> bool; + + // erase + // emplace }; // backwards compatibility diff --git a/src/snapshots/ContainerImpls.cpp b/src/snapshots/ContainerImpls.cpp index 5719d125d1..466178197c 100644 --- a/src/snapshots/ContainerImpls.cpp +++ b/src/snapshots/ContainerImpls.cpp @@ -76,12 +76,14 @@ auto StatefulSnapshotsContainer::end() -> iterator auto StatefulSnapshotsContainer::begin() const -> const_iterator { throw error::WrongAPIUsage( - "Const iteration not possible on a stateful container/iterator."); + "[StatefulSnapshotsContainer::begin] Const iteration not possible on a " + "stateful container/iterator."); } auto StatefulSnapshotsContainer::end() const -> const_iterator { throw error::WrongAPIUsage( - "Const iteration not possible on a stateful container/iterator."); + "[StatefulSnapshotsContainer::end] Const iteration not possible on a " + "stateful container/iterator."); } auto StatefulSnapshotsContainer::rbegin() -> iterator { @@ -102,18 +104,24 @@ auto StatefulSnapshotsContainer::rend() -> iterator auto StatefulSnapshotsContainer::rbegin() const -> const_iterator { throw error::WrongAPIUsage( - "Const iteration not possible on a stateful container/iterator."); + "[StatefulSnapshotsContainer::rbegin] Const iteration not possible on " + "a stateful container/iterator."); } auto StatefulSnapshotsContainer::rend() const -> const_iterator { throw error::WrongAPIUsage( - "Const iteration not possible on a stateful container/iterator."); + "[StatefulSnapshotsContainer::rend] Const iteration not possible on a " + "stateful container/iterator."); } bool StatefulSnapshotsContainer::empty() const { auto it = get(); - return (!it) || it->operator bool(); + return (!it) || !it->operator bool(); +} +auto StatefulSnapshotsContainer::size() const -> size_t +{ + throw std::runtime_error("Unimplemented"); } auto StatefulSnapshotsContainer::at(key_type const &) const @@ -187,6 +195,27 @@ auto StatefulSnapshotsContainer::operator[](key_type const &key) throw error::Internal("Control flow error: This should be unreachable."); } +auto StatefulSnapshotsContainer::clear() -> void +{ + throw std::runtime_error("Unimplemented"); +} + +auto StatefulSnapshotsContainer::find(key_type const &) -> iterator +{ + throw std::runtime_error("Unimplemented"); +} +auto StatefulSnapshotsContainer::find(key_type const &) const -> const_iterator +{ + throw error::WrongAPIUsage( + "[StatefulSnapshotsContainer::find] Const iteration not possible on a " + "stateful container/iterator."); +} + +auto StatefulSnapshotsContainer::contains(key_type const &) const -> bool +{ + throw std::runtime_error("Unimplemented"); +} + RandomAccessIteratorContainer::RandomAccessIteratorContainer( Container cont) : m_cont(std::move(cont)) @@ -240,10 +269,14 @@ auto RandomAccessIteratorContainer::rend() const -> const_reverse_iterator new RandomAccessIterator(m_cont.rend())}); } -bool RandomAccessIteratorContainer::empty() const +auto RandomAccessIteratorContainer::empty() const -> bool { return m_cont.empty(); } +auto RandomAccessIteratorContainer::size() const -> size_t +{ + return m_cont.size(); +} auto RandomAccessIteratorContainer::at(key_type const &key) const -> mapped_type const & @@ -256,4 +289,28 @@ auto RandomAccessIteratorContainer::operator[](key_type const &key) { return m_cont[key]; } + +auto RandomAccessIteratorContainer::clear() -> void +{ + throw std::runtime_error("Unimplemented"); +} + +auto RandomAccessIteratorContainer::find(key_type const &key) -> iterator +{ + return OpaqueSeriesIterator( + std::unique_ptr>{ + new RandomAccessIterator(m_cont.find(key))}); +} +auto RandomAccessIteratorContainer::find(key_type const &key) const + -> const_iterator +{ + return OpaqueSeriesIterator( + std::unique_ptr>{ + new RandomAccessIterator(m_cont.find(key))}); +} + +auto RandomAccessIteratorContainer::contains(key_type const &key) const -> bool +{ + return m_cont.contains(key); +} } // namespace openPMD diff --git a/src/snapshots/Snapshots.cpp b/src/snapshots/Snapshots.cpp index 9f8905be77..527ff09bf4 100644 --- a/src/snapshots/Snapshots.cpp +++ b/src/snapshots/Snapshots.cpp @@ -59,6 +59,10 @@ bool Snapshots::empty() const { return get().empty(); } +auto Snapshots::size() const -> size_t +{ + return get().size(); +} auto Snapshots::at(key_type const &key) const -> mapped_type const & { @@ -72,4 +76,29 @@ auto Snapshots::operator[](key_type const &key) -> mapped_type & { return get().operator[](key); } + +auto Snapshots::clear() -> void +{ + return get().clear(); +} + +auto Snapshots::find(key_type const &key) -> iterator +{ + return get().find(key); +} +auto Snapshots::find(key_type const &key) const -> const_iterator +{ + return get().find(key); +} + +auto Snapshots::count(key_type const &key) const -> size_t +{ + return contains(key) ? 1 : 0; +} + +auto Snapshots::contains(key_type const &key) const -> bool +{ + return get().contains(key); +} + } // namespace openPMD From 465a2aeab9471981833e85aec071be55a07378fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 6 Feb 2024 17:53:50 +0100 Subject: [PATCH 35/93] Some postfix form transformations --- include/openPMD/snapshots/Snapshots.hpp | 22 +++++++++++----------- src/snapshots/Snapshots.cpp | 2 +- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/include/openPMD/snapshots/Snapshots.hpp b/include/openPMD/snapshots/Snapshots.hpp index 98f0725f02..822727a267 100644 --- a/include/openPMD/snapshots/Snapshots.hpp +++ b/include/openPMD/snapshots/Snapshots.hpp @@ -56,22 +56,22 @@ class Snapshots auto currentIteration() -> std::optional; auto currentIteration() const -> std::optional; - iterator begin(); - iterator end(); - const_iterator begin() const; - const_iterator end() const; - reverse_iterator rbegin(); - reverse_iterator rend(); - const_reverse_iterator rbegin() const; - const_reverse_iterator rend() const; + auto begin() -> iterator; + auto end() -> iterator; + auto begin() const -> const_iterator; + auto end() const -> const_iterator; + auto rbegin() -> reverse_iterator; + auto rend() -> reverse_iterator; + auto rbegin() const -> const_reverse_iterator; + auto rend() const -> const_reverse_iterator; auto empty() const -> bool; auto size() const -> size_t; - mapped_type const &at(key_type const &key) const; - mapped_type &at(key_type const &key); + auto at(key_type const &key) const -> mapped_type const &; + auto at(key_type const &key) -> mapped_type &; - mapped_type &operator[](key_type const &key); + auto operator[](key_type const &key) -> mapped_type &; auto clear() -> void; diff --git a/src/snapshots/Snapshots.cpp b/src/snapshots/Snapshots.cpp index 527ff09bf4..e535afffbd 100644 --- a/src/snapshots/Snapshots.cpp +++ b/src/snapshots/Snapshots.cpp @@ -55,7 +55,7 @@ auto Snapshots::rend() const -> const_reverse_iterator return get().end(); } -bool Snapshots::empty() const +auto Snapshots::empty() const -> bool { return get().empty(); } From 894014feeed265df765e627590f1379b3214d374 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 7 Feb 2024 13:32:43 +0100 Subject: [PATCH 36/93] Use snapshots() in read example 2 --- examples/2_read_serial.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/2_read_serial.cpp b/examples/2_read_serial.cpp index d1f613c20b..85bd5aed1c 100644 --- a/examples/2_read_serial.cpp +++ b/examples/2_read_serial.cpp @@ -34,13 +34,13 @@ int main() cout << "Read a Series with openPMD standard version " << series.openPMD() << '\n'; - cout << "The Series contains " << series.iterations.size() + cout << "The Series contains " << series.snapshots().size() << " iterations:"; - for (auto const &i : series.iterations) + for (auto const &i : series.snapshots()) cout << "\n\t" << i.first; cout << '\n'; - Iteration i = series.iterations[100]; + Iteration i = series.snapshots()[100]; cout << "Iteration 100 contains " << i.meshes.size() << " meshes:"; for (auto const &m : i.meshes) cout << "\n\t" << m.first; From 937a5d34a29471ed92e03024b0ec2a21531bcbf7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 7 Feb 2024 13:33:00 +0100 Subject: [PATCH 37/93] Simplify ReadIterations implementation --- .../openPMD/snapshots/StatefulIterator.hpp | 2 +- src/snapshots/StatefulIterator.cpp | 81 +++---------------- test/SerialIOTest.cpp | 22 +++++ 3 files changed, 35 insertions(+), 70 deletions(-) diff --git a/include/openPMD/snapshots/StatefulIterator.hpp b/include/openPMD/snapshots/StatefulIterator.hpp index e7a57c1134..93287410e0 100644 --- a/include/openPMD/snapshots/StatefulIterator.hpp +++ b/include/openPMD/snapshots/StatefulIterator.hpp @@ -137,7 +137,7 @@ class StatefulIterator * the /data/snapshot attribute, this helps figuring out which iteration * is now active. Hence, recursion_depth. */ - std::optional nextStep(size_t recursion_depth); + std::optional nextStep(); std::optional loopBody(); diff --git a/src/snapshots/StatefulIterator.cpp b/src/snapshots/StatefulIterator.cpp index c14f688205..881d457ba8 100644 --- a/src/snapshots/StatefulIterator.cpp +++ b/src/snapshots/StatefulIterator.cpp @@ -313,7 +313,9 @@ StatefulIterator::StatefulIterator( */ Iteration::BeginStepStatus::AvailableIterations_t availableIterations; - std::tie(status, availableIterations) = it->second.beginStep( + std::tie(status, availableIterations) = Iteration::beginStep( + {}, + series, /* reread = */ reread(data.parsePreference)); /* * In random-access mode, do not use the information read in the @@ -420,8 +422,12 @@ std::optional StatefulIterator::nextIterationInStep() throw std::runtime_error("Unreachable!"); } -std::optional -StatefulIterator::nextStep(size_t recursion_depth) +void breakpoint() +{ + std::cout << "BREAKPOINT" << std::endl; +} + +std::optional StatefulIterator::nextStep() { auto &data = get(); // since we are in group-based iteration layout, it does not @@ -442,7 +448,7 @@ StatefulIterator::nextStep(size_t recursion_depth) "below, will skip it.\n" << err.what() << std::endl; data.series.advance(AdvanceMode::ENDSTEP); - return nextStep(recursion_depth + 1); + return nextStep(); } if (status != AdvanceStatus::RANDOMACCESS) @@ -451,70 +457,7 @@ StatefulIterator::nextStep(size_t recursion_depth) } else { - /* - * Fallback implementation: Assume that each step corresponds - * with an iteration in ascending order. - */ - if (!data.currentIteration.has_value()) - { - throw std::runtime_error( - "CATASTROPHE. need to think how to resolve this one."); - } - auto &series = data.series; - auto it = series.iterations.find(*data.currentIteration); - auto itEnd = series.iterations.end(); - if (it == itEnd) - { - if (status == AdvanceStatus::RANDOMACCESS || - status == AdvanceStatus::OVER) - { - this->close(); - return {this}; - } - else - { - /* - * Stream still going but there was no iteration found in the - * current IO step? - * Might be a duplicate iteration resulting from appending, - * will skip such iterations and hope to find something in a - * later IO step. No need to finish right now. - */ - data.iterationsInCurrentStep = {}; - } - } - else - { - for (size_t i = 0; i < recursion_depth && it != itEnd; ++i) - { - ++it; - } - - if (it == itEnd) - { - if (status == AdvanceStatus::RANDOMACCESS || - status == AdvanceStatus::OVER) - { - this->close(); - return {this}; - } - else - { - /* - * Stream still going but there was no iteration found in - * the current IO step? Might be a duplicate iteration - * resulting from appending, will skip such iterations and - * hope to find something in a later IO step. No need to - * finish right now. - */ - data.iterationsInCurrentStep = {}; - } - } - else - { - data.iterationsInCurrentStep = {it->first}; - } - } + this->close(); } if (status == AdvanceStatus::OVER) @@ -619,7 +562,7 @@ std::optional StatefulIterator::loopBody() return {this}; } - auto option = nextStep(/*recursion_depth = */ 1); + auto option = nextStep(); return guardReturn(option); } diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index 7323a32582..106ce7b6ce 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -6615,6 +6615,28 @@ void deferred_parsing(std::string const &extension) std::numeric_limits::epsilon()); } } + { + Series series( + basename + "%06T." + extension, + Access::READ_ONLY, + "{\"defer_iteration_parsing\": true}"); + for (auto iteration : series.readIterations()) + { + auto dataset = + iteration.meshes["E"]["x"].loadChunk({0}, {20}); + iteration.close(); + for (size_t i = 0; i < 20; ++i) + { + REQUIRE( + std::abs(dataset.get()[i] - float(i)) <= + std::numeric_limits::epsilon()); + } + if (iteration.iterationIndex == 0) + { + break; + } + } + } { Series series( basename + "%06T." + extension, From 913c1cdee22087820c9e7f228dede8c805e3f540 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 7 Feb 2024 17:47:58 +0100 Subject: [PATCH 38/93] Further cleanup --- src/snapshots/StatefulIterator.cpp | 54 ++++++++++++++---------------- 1 file changed, 25 insertions(+), 29 deletions(-) diff --git a/src/snapshots/StatefulIterator.cpp b/src/snapshots/StatefulIterator.cpp index 881d457ba8..45fbdddd8c 100644 --- a/src/snapshots/StatefulIterator.cpp +++ b/src/snapshots/StatefulIterator.cpp @@ -22,6 +22,7 @@ #include "openPMD/snapshots/StatefulIterator.hpp" #include "openPMD/Error.hpp" +#include "openPMD/Iteration.hpp" #include "openPMD/Series.hpp" #include @@ -361,7 +362,6 @@ std::optional StatefulIterator::nextIterationInStep() { return ret_t{}; } - auto oldIterationIndex = data.currentIteration; data.currentIteration = *data.iterationsInCurrentStep.begin(); auto &series = data.series; @@ -369,17 +369,7 @@ std::optional StatefulIterator::nextIterationInStep() { case IterationEncoding::groupBased: case IterationEncoding::variableBased: { - if (oldIterationIndex.has_value()) - { - auto begin = series.iterations.find(*oldIterationIndex); - auto end = begin; - ++end; - series.flush_impl( - begin, - end, - {FlushLevel::UserFlush}, - /* flushIOHandler = */ true); - } + try { series.iterations[*data.currentIteration].open(); @@ -451,19 +441,15 @@ std::optional StatefulIterator::nextStep() return nextStep(); } - if (status != AdvanceStatus::RANDOMACCESS) + switch (status) { + case AdvanceStatus::OK: data.iterationsInCurrentStep = availableIterations; - } - else - { - this->close(); - } - - if (status == AdvanceStatus::OVER) - { + break; + case AdvanceStatus::OVER: + case AdvanceStatus::RANDOMACCESS: this->close(); - return {this}; + break; } return {this}; @@ -485,6 +471,7 @@ std::optional StatefulIterator::loopBody() if (!currentIteration.closed()) { currentIteration.close(); + data.currentIteration = std::nullopt; } } @@ -495,6 +482,11 @@ std::optional StatefulIterator::loopBody() { return option; } + /* + * A step was successfully opened, but no iterations are contained. + * This might happen when iterations from the step are ignored, e.g. + * a duplicate iteration has been written by Append mode. + */ auto currentIterationIndex = option.value()->peekCurrentIteration(); if (!currentIterationIndex.has_value()) { @@ -509,14 +501,13 @@ std::optional StatefulIterator::loopBody() if (iterations.contains(index)) { auto iteration = iterations.at(index); - if (iteration.get().m_closed != - internal::CloseStatus::ClosedInBackend) + switch (iteration.get().m_closed) { + case internal::CloseStatus::ParseAccessDeferred: try { iterations.at(index).open(); - option.value()->setCurrentIteration(); - return option; + [[fallthrough]]; } catch (error::ReadError const &err) { @@ -528,13 +519,18 @@ std::optional StatefulIterator::loopBody() currentIterationIndex.value()); return std::nullopt; } - } - else - { + case internal::CloseStatus::Open: + option.value()->setCurrentIteration(); + return option; + case internal::CloseStatus::ClosedInBackend: // we had this iteration already, skip it iteration.endStep(); return std::nullopt; // empty, go into next iteration + case internal::CloseStatus::ClosedInFrontend: + case internal::CloseStatus::ClosedTemporarily: + throw error::Internal("Next found iteration is closed?"); } + throw std::runtime_error("Unreachable!"); } else { From a7d75c8f1663e993962c1555f7691b0b9bcdcab3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 7 Feb 2024 19:48:20 +0100 Subject: [PATCH 39/93] Change representation of iterations in current step --- include/openPMD/Iteration.hpp | 2 +- include/openPMD/Series.hpp | 3 +- .../openPMD/snapshots/StatefulIterator.hpp | 8 +- src/Series.cpp | 25 ++-- src/snapshots/ContainerImpls.cpp | 7 + src/snapshots/StatefulIterator.cpp | 129 +++++++----------- 6 files changed, 72 insertions(+), 102 deletions(-) diff --git a/include/openPMD/Iteration.hpp b/include/openPMD/Iteration.hpp index c594860a92..5209372350 100644 --- a/include/openPMD/Iteration.hpp +++ b/include/openPMD/Iteration.hpp @@ -315,7 +315,7 @@ class Iteration : public Attributable */ struct BeginStepStatus { - using AvailableIterations_t = std::deque; + using AvailableIterations_t = std::vector; AdvanceStatus stepStatus{}; /* diff --git a/include/openPMD/Series.hpp b/include/openPMD/Series.hpp index e36c610bec..f185502622 100644 --- a/include/openPMD/Series.hpp +++ b/include/openPMD/Series.hpp @@ -50,6 +50,7 @@ #include #include #include +#include // expose private and protected members for invasive testing #ifndef OPENPMD_private @@ -813,7 +814,7 @@ OPENPMD_private * ReadIterations since those methods should be aware when the current * step is broken). */ - std::deque readGorVBased( + std::vector readGorVBased( bool do_always_throw_errors, bool init, std::set const &ignoreIterations = {}); diff --git a/include/openPMD/snapshots/StatefulIterator.hpp b/include/openPMD/snapshots/StatefulIterator.hpp index 93287410e0..8e15a4964d 100644 --- a/include/openPMD/snapshots/StatefulIterator.hpp +++ b/include/openPMD/snapshots/StatefulIterator.hpp @@ -30,6 +30,7 @@ #include #include #include +#include namespace openPMD { @@ -61,9 +62,9 @@ class StatefulIterator ~SharedData(); Series series; - std::deque iterationsInCurrentStep; + std::vector iterationsInCurrentStep; // nullopt <-> currently out of step - std::optional currentIteration{}; + std::optional currentIteration{}; std::optional parsePreference; /* * Necessary because in the old ADIOS2 schema, old iterations' metadata @@ -147,8 +148,7 @@ class StatefulIterator void close(); - auto setCurrentIteration() -> bool; - auto peekCurrentIteration() -> std::optional; + auto resetCurrentIterationToBegin() -> bool; auto peekCurrentlyOpenIteration() const -> std::optional; auto peekCurrentlyOpenIteration() -> std::optional; diff --git a/src/Series.cpp b/src/Series.cpp index df45145523..d1d0c6e381 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -57,6 +57,7 @@ #include #include #include +#include namespace openPMD { @@ -1876,7 +1877,7 @@ auto Series::readGorVBased( bool do_always_throw_errors, bool do_init, std::set const &ignoreIterations) - -> std::deque + -> std::vector { auto &series = get(); Parameter fOpen; @@ -2074,8 +2075,9 @@ creating new iterations. * Sic! This happens when a file-based Series is opened in group-based mode. */ case IterationEncoding::fileBased: { - std::deque unreadableIterations; - std::deque readableIterations; + std::vector unreadableIterations; + std::vector readableIterations; + readableIterations.reserve(pList.paths->size()); for (auto const &it : *pList.paths) { IterationIndex_t index = std::stoull(it); @@ -2108,7 +2110,7 @@ creating new iterations. { auto &vec = currentSteps.value(); vectorDifference(vec, unreadableIterations); - return std::deque{vec.begin(), vec.end()}; + return vec; } else { @@ -2116,22 +2118,15 @@ creating new iterations. } } case IterationEncoding::variableBased: { - std::deque res{}; if (currentSteps.has_value() && !currentSteps.value().empty()) { - for (auto index : currentSteps.value()) - { - if (ignoreIterations.find(index) == ignoreIterations.end()) - { - res.push_back(index); - } - } + vectorDifference(*currentSteps, ignoreIterations); } else { - res = {0}; + currentSteps = std::vector{0}; } - for (auto it : res) + for (auto it : *currentSteps) { /* * Variable-based iteration encoding relies on steps, so parsing @@ -2153,7 +2148,7 @@ creating new iterations. throw *err; } } - return res; + return *currentSteps; } } throw std::runtime_error("Unreachable!"); diff --git a/src/snapshots/ContainerImpls.cpp b/src/snapshots/ContainerImpls.cpp index 466178197c..0ef3903a96 100644 --- a/src/snapshots/ContainerImpls.cpp +++ b/src/snapshots/ContainerImpls.cpp @@ -168,6 +168,13 @@ auto StatefulSnapshotsContainer::operator[](key_type const &key) } } s.currentIteration = key; + if (std::find( + s.iterationsInCurrentStep.begin(), + s.iterationsInCurrentStep.end(), + key) != s.iterationsInCurrentStep.end()) + { + s.iterationsInCurrentStep.push_back(key); + } auto &res = s.series.iterations[key]; if (res.getStepStatus() == StepStatus::NoStep) { diff --git a/src/snapshots/StatefulIterator.cpp b/src/snapshots/StatefulIterator.cpp index 45fbdddd8c..3e1fb97d86 100644 --- a/src/snapshots/StatefulIterator.cpp +++ b/src/snapshots/StatefulIterator.cpp @@ -145,37 +145,21 @@ void StatefulIterator::close() *m_data = std::nullopt; // turn this into end iterator } -auto StatefulIterator::setCurrentIteration() -> bool +auto StatefulIterator::resetCurrentIterationToBegin() -> bool { auto &data = get(); if (data.iterationsInCurrentStep.empty()) { - std::cerr << "[ReadIterations] Encountered a step without " - "iterations. Closing the Series." - << std::endl; - *this = end(); + data.currentIteration = std::nullopt; return false; } - data.currentIteration = *data.iterationsInCurrentStep.begin(); - return true; -} - -auto StatefulIterator::peekCurrentIteration() -> std::optional -{ - if (!m_data || !m_data->has_value()) - { - return std::nullopt; - } - auto &data = m_data->value(); - if (data.iterationsInCurrentStep.empty()) - { - return std::nullopt; - } else { - return {*data.iterationsInCurrentStep.begin()}; + data.currentIteration = *data.iterationsInCurrentStep.begin(); + return true; } } + auto StatefulIterator::peekCurrentlyOpenIteration() const -> std::optional { @@ -339,7 +323,7 @@ StatefulIterator::StatefulIterator( this->close(); return; } - if (!setCurrentIteration()) + if (!resetCurrentIterationToBegin()) { this->close(); return; @@ -351,65 +335,50 @@ StatefulIterator::StatefulIterator( std::optional StatefulIterator::nextIterationInStep() { auto &data = get(); - using ret_t = std::optional; + auto no_result = [&]() { + data.currentIteration = std::nullopt; + return std::nullopt; + }; - if (data.iterationsInCurrentStep.empty()) - { - return ret_t{}; - } - data.iterationsInCurrentStep.pop_front(); - if (data.iterationsInCurrentStep.empty()) + if (!data.currentIteration.has_value()) { - return ret_t{}; + return no_result(); } - data.currentIteration = *data.iterationsInCurrentStep.begin(); - auto &series = data.series; - switch (series.iterationEncoding()) + if (auto it = std::find( + data.iterationsInCurrentStep.begin(), + data.iterationsInCurrentStep.end(), + *data.currentIteration); + it != data.iterationsInCurrentStep.end()) { - case IterationEncoding::groupBased: - case IterationEncoding::variableBased: { - - try + ++it; + if (it == data.iterationsInCurrentStep.end()) { - series.iterations[*data.currentIteration].open(); + return no_result(); } - catch (error::ReadError const &err) - { - std::cerr << "Cannot read iteration '" << *data.currentIteration - << "' and will skip it due to read error:\n" - << err.what() << std::endl; - return nextIterationInStep(); - } - - return {this}; + data.currentIteration = *it; + } + else + { + return no_result(); } - case IterationEncoding::fileBased: - try - { - /* - * Errors in here might appear due to deferred iteration parsing. - */ - series.iterations[*data.currentIteration].open(); - /* - * Errors in here might appear due to reparsing after opening a - * new step. - */ - series.iterations[*data.currentIteration].beginStep( - /* reread = */ true); - } - catch (error::ReadError const &err) - { - std::cerr - << "[StatefulIterator] Cannot read iteration due to error " - "below, will skip it.\n" - << err.what() << std::endl; - return nextIterationInStep(); - } - return {this}; + auto &series = data.series; + + try + { + series.iterations.at(*data.currentIteration).open(); } - throw std::runtime_error("Unreachable!"); + catch (error::ReadError const &err) + { + std::cerr << "[StatefulIterator] Cannot read iteration '" + << *data.currentIteration + << "' and will skip it due to read error:\n" + << err.what() << std::endl; + return nextIterationInStep(); + } + + return {this}; } void breakpoint() @@ -443,15 +412,15 @@ std::optional StatefulIterator::nextStep() switch (status) { - case AdvanceStatus::OK: - data.iterationsInCurrentStep = availableIterations; - break; case AdvanceStatus::OVER: case AdvanceStatus::RANDOMACCESS: this->close(); break; + case AdvanceStatus::OK: + data.iterationsInCurrentStep = availableIterations; + resetCurrentIterationToBegin(); + break; } - return {this}; } @@ -467,16 +436,15 @@ std::optional StatefulIterator::loopBody() if (data.currentIteration.has_value() && iterations.contains(*data.currentIteration)) { - auto ¤tIteration = iterations[*data.currentIteration]; + auto ¤tIteration = iterations.at(*data.currentIteration); if (!currentIteration.closed()) { currentIteration.close(); - data.currentIteration = std::nullopt; } } auto guardReturn = - [&series, &iterations]( + [&series, &iterations, &data]( auto const &option) -> std::optional { if (!option.has_value() || *option.value() == end()) { @@ -487,7 +455,7 @@ std::optional StatefulIterator::loopBody() * This might happen when iterations from the step are ignored, e.g. * a duplicate iteration has been written by Append mode. */ - auto currentIterationIndex = option.value()->peekCurrentIteration(); + auto currentIterationIndex = data.currentIteration; if (!currentIterationIndex.has_value()) { series.advance(AdvanceMode::ENDSTEP); @@ -520,7 +488,6 @@ std::optional StatefulIterator::loopBody() return std::nullopt; } case internal::CloseStatus::Open: - option.value()->setCurrentIteration(); return option; case internal::CloseStatus::ClosedInBackend: // we had this iteration already, skip it @@ -609,7 +576,7 @@ StatefulIterator &StatefulIterator::operator++() { auto &series = data.series; auto index = data.currentIteration; - auto &iteration = series.iterations[index.value()]; + auto &iteration = series.iterations.at(index.value()); iteration.setStepStatus(StepStatus::DuringStep); if (series.IOHandler()->m_frontendAccess == Access::READ_LINEAR && From 43c3ab04472b22a866fa5c2aae05f01d3ec05874 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 8 Feb 2024 10:57:01 +0100 Subject: [PATCH 40/93] Initiate reading of group/variable-based encoding with nextStep() --- .../openPMD/snapshots/StatefulIterator.hpp | 26 ++- src/snapshots/StatefulIterator.cpp | 170 ++++++++---------- 2 files changed, 101 insertions(+), 95 deletions(-) diff --git a/include/openPMD/snapshots/StatefulIterator.hpp b/include/openPMD/snapshots/StatefulIterator.hpp index 8e15a4964d..b468ad9b31 100644 --- a/include/openPMD/snapshots/StatefulIterator.hpp +++ b/include/openPMD/snapshots/StatefulIterator.hpp @@ -38,6 +38,27 @@ namespace internal { class SeriesData; } + +namespace detail +{ + namespace seek_types + { + struct InitNonFileBased_t + {}; + struct Next_t + {}; + using seek_impl = std::variant; + } // namespace seek_types + struct Seek : seek_types::seek_impl + { + using InitNonFileBased_t = seek_types::InitNonFileBased_t; + using Next_t = seek_types::Next_t; + + constexpr static InitNonFileBased_t InitNonFileBased{}; + constexpr static Next_t Next{}; + }; +} // namespace detail + class StatefulIterator : public AbstractSeriesIterator< StatefulIterator, @@ -89,6 +110,7 @@ class StatefulIterator using value_type = typename Container::value_type; using typename parent_t ::difference_type; + using Seek = detail::Seek; //! construct the end() iterator explicit StatefulIterator(); @@ -138,9 +160,9 @@ class StatefulIterator * the /data/snapshot attribute, this helps figuring out which iteration * is now active. Hence, recursion_depth. */ - std::optional nextStep(); + std::optional nextStep(Seek const &); - std::optional loopBody(); + std::optional loopBody(Seek const &); void deactivateDeadIteration(iteration_index_t); diff --git a/src/snapshots/StatefulIterator.cpp b/src/snapshots/StatefulIterator.cpp index 3e1fb97d86..dc32175512 100644 --- a/src/snapshots/StatefulIterator.cpp +++ b/src/snapshots/StatefulIterator.cpp @@ -24,8 +24,10 @@ #include "openPMD/Iteration.hpp" #include "openPMD/Series.hpp" +#include "openPMD/auxiliary/Variant.hpp" #include +#include #include #include @@ -243,92 +245,55 @@ StatefulIterator::StatefulIterator( initSeriesInLinearReadMode(); } - auto it = series.get().iterations.begin(); - if (it == series.get().iterations.end()) + switch (series.iterationEncoding()) { - this->close(); - return; - } - else if ( - it->second.get().m_closed == internal::CloseStatus::ClosedInBackend) - { - throw error::WrongAPIUsage( - "Trying to call Series::readIterations() on a (partially) read " - "Series."); - } - else - { - auto openIteration = [](Iteration &iteration) { - /* - * @todo - * Is that really clean? - * Use case: See Python ApiTest testListSeries: - * Call listSeries twice. - */ - if (iteration.get().m_closed != - internal::CloseStatus::ClosedInBackend) - { - iteration.open(); - } - }; - AdvanceStatus status{}; - switch (series.iterationEncoding()) - { - case IterationEncoding::fileBased: - /* - * The file needs to be accessed before beginning a step upon it. - * In file-based iteration layout it maybe is not accessed yet, - * so do that now. There is only one step per file, so beginning - * the step after parsing the file is ok. - */ - openIteration(series.iterations.begin()->second); - status = it->second.beginStep(/* reread = */ true); - for (auto const &pair : series.iterations) + case IterationEncoding::fileBased: { + if (series.iterations.empty()) + { + this->close(); + return; + } + data.iterationsInCurrentStep.reserve(series.iterations.size()); + std::transform( + series.iterations.begin(), + series.iterations.end(), + std::back_inserter(data.iterationsInCurrentStep), + [](auto const &pair) { return pair.first; }); + auto it = series.iterations.begin(); + auto end = series.iterations.end(); + for (; it != end; ++it) + { + try { - data.iterationsInCurrentStep.push_back(pair.first); + it->second.open(); + break; } - break; - case IterationEncoding::groupBased: - case IterationEncoding::variableBased: { - /* - * In group-based iteration layout, we have definitely already had - * access to the file until now. Better to begin a step right away, - * otherwise we might get another step's data. - */ - Iteration::BeginStepStatus::AvailableIterations_t - availableIterations; - std::tie(status, availableIterations) = Iteration::beginStep( - {}, - series, - /* reread = */ reread(data.parsePreference)); - /* - * In random-access mode, do not use the information read in the - * `snapshot` attribute, instead simply go through iterations - * one by one in ascending order (fallback implementation in the - * second if branch). - */ - data.iterationsInCurrentStep = availableIterations; - if (!data.iterationsInCurrentStep.empty()) + catch (error::ReadError const &err) { - openIteration( - series.iterations.at(data.iterationsInCurrentStep.at(0))); + std::cerr << "[StatefulIterator] Cannot read iteration '" + << it->first + << "' and will skip it due to read error:\n" + << err.what() << std::endl; } - break; } + if (it != end) + { + data.currentIteration = it->first; } - - if (status == AdvanceStatus::OVER) + else { this->close(); - return; } - if (!resetCurrentIterationToBegin()) + break; + } + case IterationEncoding::groupBased: + case IterationEncoding::variableBased: + if (!loopBody({Seek::InitNonFileBased}).has_value()) { this->close(); - return; } - it->second.setStepStatus(StepStatus::DuringStep); + break; } } @@ -381,12 +346,7 @@ std::optional StatefulIterator::nextIterationInStep() return {this}; } -void breakpoint() -{ - std::cout << "BREAKPOINT" << std::endl; -} - -std::optional StatefulIterator::nextStep() +std::optional StatefulIterator::nextStep(Seek const &seek) { auto &data = get(); // since we are in group-based iteration layout, it does not @@ -407,24 +367,40 @@ std::optional StatefulIterator::nextStep() "below, will skip it.\n" << err.what() << std::endl; data.series.advance(AdvanceMode::ENDSTEP); - return nextStep(); + return nextStep(seek); } - switch (status) + bool close = [&]() { + switch (status) + { + + case AdvanceStatus::OK: + return false; + case AdvanceStatus::OVER: + return true; + case AdvanceStatus::RANDOMACCESS: + return std::visit( + auxiliary::overloaded{ + [](Seek::InitNonFileBased_t const &) { return false; }, + [](Seek::Next_t const &) { return true; }}, + seek); + } + throw std::runtime_error("Unreachable!"); + }(); + + if (close) { - case AdvanceStatus::OVER: - case AdvanceStatus::RANDOMACCESS: this->close(); - break; - case AdvanceStatus::OK: + } + else + { data.iterationsInCurrentStep = availableIterations; resetCurrentIterationToBegin(); - break; } return {this}; } -std::optional StatefulIterator::loopBody() +std::optional StatefulIterator::loopBody(Seek const &seek) { auto &data = get(); Series &series = data.series; @@ -507,12 +483,20 @@ std::optional StatefulIterator::loopBody() } }; + using res_t = std::optional; + auto optionallyAStep = std::visit( + auxiliary::overloaded{ + [](Seek::InitNonFileBased_t const &) -> res_t { + return std::nullopt; + }, + [this](Seek::Next_t const &) -> res_t { + return nextIterationInStep(); + }}, + seek); + + if (optionallyAStep.has_value()) { - auto optionallyAStep = nextIterationInStep(); - if (optionallyAStep.has_value()) - { - return guardReturn(optionallyAStep); - } + return guardReturn(optionallyAStep); } // The currently active iterations have been exhausted. @@ -525,7 +509,7 @@ std::optional StatefulIterator::loopBody() return {this}; } - auto option = nextStep(); + auto option = nextStep(seek); return guardReturn(option); } @@ -568,7 +552,7 @@ StatefulIterator &StatefulIterator::operator++() */ do { - res = loopBody(); + res = loopBody({Seek::Next}); } while (!res.has_value()); auto resvalue = res.value(); From 26d87be27faac5b3cd40166d897d6ef29107cc40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 14 Feb 2024 13:15:54 +0100 Subject: [PATCH 41/93] Prepare internal representation to be aware of steps --- .../openPMD/snapshots/StatefulIterator.hpp | 116 +++++- src/snapshots/ContainerImpls.cpp | 17 +- src/snapshots/StatefulIterator.cpp | 337 ++++++++++++------ 3 files changed, 345 insertions(+), 125 deletions(-) diff --git a/include/openPMD/snapshots/StatefulIterator.hpp b/include/openPMD/snapshots/StatefulIterator.hpp index b468ad9b31..6fc6df6c95 100644 --- a/include/openPMD/snapshots/StatefulIterator.hpp +++ b/include/openPMD/snapshots/StatefulIterator.hpp @@ -23,6 +23,8 @@ #include "openPMD/Error.hpp" #include "openPMD/Iteration.hpp" #include "openPMD/Series.hpp" +#include "openPMD/Streaming.hpp" +#include "openPMD/auxiliary/Variant.hpp" #include "openPMD/backend/ParsePreference.hpp" #include "openPMD/snapshots/IteratorTraits.hpp" @@ -41,21 +43,58 @@ namespace internal namespace detail { - namespace seek_types + namespace step_status_types { - struct InitNonFileBased_t + struct Before_t {}; - struct Next_t + struct During_t + { + size_t idx; + std::optional iteration_idx; + }; + struct After_t {}; - using seek_impl = std::variant; - } // namespace seek_types - struct Seek : seek_types::seek_impl + } // namespace step_status_types + struct CurrentStep + : std::variant< + step_status_types::Before_t, + step_status_types::During_t, + step_status_types::After_t> { - using InitNonFileBased_t = seek_types::InitNonFileBased_t; - using Next_t = seek_types::Next_t; - - constexpr static InitNonFileBased_t InitNonFileBased{}; - constexpr static Next_t Next{}; + using Before_t = step_status_types::Before_t; + constexpr static Before_t Before{}; + using During_t = step_status_types::During_t; + constexpr static During_t During{}; + using After_t = step_status_types::After_t; + constexpr static After_t After{}; + + using variant_t = std::variant< + step_status_types::Before_t, + step_status_types::During_t, + step_status_types::After_t>; + + using variant_t::operator=; + + template + auto get_variant() -> std::optional; + template + auto get_variant() const -> std::optional; + + auto get_iteration_index() const + -> std::optional; + auto get_iteration_index() + -> std::optional; + + enum class AtTheEdge : bool + { + Begin, + End + }; + + template + auto map_during_t(F &&map, G &&create_new); + template + auto map_during_t(F &&map); }; } // namespace detail @@ -72,6 +111,8 @@ class StatefulIterator using maybe_series_t = std::optional; + using CurrentStep = detail::CurrentStep; + struct SharedData { SharedData() = default; @@ -84,8 +125,7 @@ class StatefulIterator Series series; std::vector iterationsInCurrentStep; - // nullopt <-> currently out of step - std::optional currentIteration{}; + CurrentStep currentStep = {CurrentStep::Before}; std::optional parsePreference; /* * Necessary because in the old ADIOS2 schema, old iterations' metadata @@ -93,6 +133,10 @@ class StatefulIterator * are still there and the iterations can be parsed again. */ std::set ignoreIterations; + + auto currentIteration() -> std::optional; + auto currentIteration() const + -> std::optional; }; /* @@ -110,7 +154,6 @@ class StatefulIterator using value_type = typename Container::value_type; using typename parent_t ::difference_type; - using Seek = detail::Seek; //! construct the end() iterator explicit StatefulIterator(); @@ -160,9 +203,11 @@ class StatefulIterator * the /data/snapshot attribute, this helps figuring out which iteration * is now active. Hence, recursion_depth. */ - std::optional nextStep(Seek const &); + std::optional nextStep(size_t recursion_depth); - std::optional loopBody(Seek const &); + std::optional loopBody(); + + void initIteratorFilebased(); void deactivateDeadIteration(iteration_index_t); @@ -170,7 +215,7 @@ class StatefulIterator void close(); - auto resetCurrentIterationToBegin() -> bool; + auto resetCurrentIterationToBegin(size_t num_skipped_iterations) -> void; auto peekCurrentlyOpenIteration() const -> std::optional; auto peekCurrentlyOpenIteration() -> std::optional; @@ -231,3 +276,40 @@ class ReadIterations auto end() -> iterator_t; }; } // namespace openPMD + +// Template definitions + +namespace openPMD::detail +{ +template +auto CurrentStep::map_during_t(F &&map, G &&create_new) +{ + std::visit( + auxiliary::overloaded{ + [&](During_t &during) { std::forward(map)(during); }, + [&](Before_t const &) { + std::optional res = + std::forward(create_new)(AtTheEdge::Begin); + if (res.has_value()) + { + this->swap(*res); + } + }, + [&](After_t const &) { + std::optional res = + std::forward(create_new)(AtTheEdge::End); + if (res.has_value()) + { + this->swap(*res); + } + }}, + *this); +} + +template +auto CurrentStep::map_during_t(F &&map) +{ + map_during_t( + std::forward(map), [](auto const &) { return std::nullopt; }); +} +} // namespace openPMD::detail diff --git a/src/snapshots/ContainerImpls.cpp b/src/snapshots/ContainerImpls.cpp index 0ef3903a96..0a788172e5 100644 --- a/src/snapshots/ContainerImpls.cpp +++ b/src/snapshots/ContainerImpls.cpp @@ -167,7 +167,22 @@ auto StatefulSnapshotsContainer::operator[](key_type const &key) lastIteration_v->second.close(); // continue below } } - s.currentIteration = key; + s.currentStep.map_during_t( + [&](auto &during) { + ++during.idx; + during.iteration_idx = key; + }, + [&](detail::CurrentStep::AtTheEdge whereAmI) { + switch (whereAmI) + { + case detail::CurrentStep::AtTheEdge::Begin: + return detail::CurrentStep::During_t{0, key}; + case detail::CurrentStep::AtTheEdge::End: + throw error::WrongAPIUsage( + "Creating a new step on a Series that is closed."); + } + throw std::runtime_error("Unreachable!"); + }); if (std::find( s.iterationsInCurrentStep.begin(), s.iterationsInCurrentStep.end(), diff --git a/src/snapshots/StatefulIterator.cpp b/src/snapshots/StatefulIterator.cpp index dc32175512..678fd977a4 100644 --- a/src/snapshots/StatefulIterator.cpp +++ b/src/snapshots/StatefulIterator.cpp @@ -30,15 +30,90 @@ #include #include #include +#include namespace openPMD { + +namespace detail +{ + template + auto CurrentStep::get_variant() -> std::optional + { + auto res = std::get_if(this); + if (res) + { + return std::make_optional(res); + } + else + { + return std::nullopt; + } + } + + template + auto CurrentStep::get_variant() const -> std::optional + { + auto res = std::get_if(*this); + if (res) + { + return {res}; + } + else + { + return std::nullopt; + } + } + + auto CurrentStep::get_iteration_index() const + -> std::optional + { + using res_t = std::optional; + return std::visit( + auxiliary::overloaded{ + [](auto const &) -> res_t { return std::nullopt; }, + [](During_t const &during) -> res_t { + if (during.iteration_idx.has_value()) + { + return std::make_optional< + Iteration::IterationIndex_t const *>( + &*during.iteration_idx); + } + else + { + return std::nullopt; + } + }}, + *this); + } + auto CurrentStep::get_iteration_index() + -> std::optional + { + auto res = + static_cast(this)->get_iteration_index(); + if (res.has_value()) + { + return const_cast(*res); + } + else + { + return std::nullopt; + } + } +} // namespace detail + StatefulIterator::SharedData::~SharedData() { - if (auto IOHandler = series.IOHandler(); currentIteration.has_value() && - IOHandler && IOHandler->m_lastFlushSuccessful) + auto IOHandler = series.IOHandler(); + auto current_iteration = currentIteration(); + if (IOHandler && current_iteration.has_value() && IOHandler && + IOHandler->m_lastFlushSuccessful) { - auto lastIterationIndex = currentIteration.value(); + auto lastIterationIndex = **current_iteration; + if (!series.iterations.contains(**current_iteration)) + { + return; + } auto &lastIteration = series.iterations.at(lastIterationIndex); if (!lastIteration.closed()) { @@ -47,6 +122,17 @@ StatefulIterator::SharedData::~SharedData() } } +auto StatefulIterator::SharedData::currentIteration() + -> std::optional +{ + return currentStep.get_iteration_index(); +} +auto StatefulIterator::SharedData::currentIteration() const + -> std::optional +{ + return currentStep.get_iteration_index(); +} + namespace { bool reread(std::optional parsePreference) @@ -147,19 +233,39 @@ void StatefulIterator::close() *m_data = std::nullopt; // turn this into end iterator } -auto StatefulIterator::resetCurrentIterationToBegin() -> bool +auto StatefulIterator::resetCurrentIterationToBegin( + size_t num_skipped_iterations) -> void { auto &data = get(); - if (data.iterationsInCurrentStep.empty()) - { - data.currentIteration = std::nullopt; - return false; - } - else - { - data.currentIteration = *data.iterationsInCurrentStep.begin(); - return true; - } + data.currentStep.map_during_t( + [&](CurrentStep::During_t &during) { + if (data.iterationsInCurrentStep.empty()) + { + during.iteration_idx = std::nullopt; + } + else + { + during.iteration_idx = *data.iterationsInCurrentStep.begin(); + } + }, + [&](CurrentStep::AtTheEdge whereAmI) + -> std::optional { + switch (whereAmI) + { + case detail::CurrentStep::AtTheEdge::Begin: + if (data.iterationsInCurrentStep.empty()) + { + return std::nullopt; + } + // Begin iterating + return detail::CurrentStep::During_t{ + num_skipped_iterations, + *data.iterationsInCurrentStep.begin()}; + case detail::CurrentStep::AtTheEdge::End: + return std::nullopt; + } + throw std::runtime_error("Unreachable!"); + }); } auto StatefulIterator::peekCurrentlyOpenIteration() const @@ -170,13 +276,14 @@ auto StatefulIterator::peekCurrentlyOpenIteration() const return std::nullopt; } auto &s = m_data->value(); - if (!s.currentIteration.has_value()) + auto const &maybeCurrentIteration = s.currentIteration(); + if (!maybeCurrentIteration.has_value()) { return std::nullopt; } // Iteration ¤tIteration = // s.series.iterations.at(*s.currentIteration); - auto currentIteration = s.series.iterations.find(*s.currentIteration); + auto currentIteration = s.series.iterations.find(**maybeCurrentIteration); if (currentIteration == s.series.iterations.end()) { return std::nullopt; @@ -249,47 +356,12 @@ StatefulIterator::StatefulIterator( { case IterationEncoding::fileBased: { - if (series.iterations.empty()) - { - this->close(); - return; - } - data.iterationsInCurrentStep.reserve(series.iterations.size()); - std::transform( - series.iterations.begin(), - series.iterations.end(), - std::back_inserter(data.iterationsInCurrentStep), - [](auto const &pair) { return pair.first; }); - auto it = series.iterations.begin(); - auto end = series.iterations.end(); - for (; it != end; ++it) - { - try - { - it->second.open(); - break; - } - catch (error::ReadError const &err) - { - std::cerr << "[StatefulIterator] Cannot read iteration '" - << it->first - << "' and will skip it due to read error:\n" - << err.what() << std::endl; - } - } - if (it != end) - { - data.currentIteration = it->first; - } - else - { - this->close(); - } + initIteratorFilebased(); break; } case IterationEncoding::groupBased: case IterationEncoding::variableBased: - if (!loopBody({Seek::InitNonFileBased}).has_value()) + if (!loopBody().has_value()) { this->close(); } @@ -300,20 +372,31 @@ StatefulIterator::StatefulIterator( std::optional StatefulIterator::nextIterationInStep() { auto &data = get(); + auto maybeCurrentIteration = + data.currentStep.get_variant(); + + if (!maybeCurrentIteration.has_value()) + { + return std::nullopt; + } + CurrentStep::During_t ¤tIteration = **maybeCurrentIteration; + auto no_result = [&]() { - data.currentIteration = std::nullopt; + currentIteration.iteration_idx = std::nullopt; return std::nullopt; }; - if (!data.currentIteration.has_value()) + if (!currentIteration.iteration_idx.has_value()) { return no_result(); } + auto ¤t_iteration_idx = *currentIteration.iteration_idx; + if (auto it = std::find( data.iterationsInCurrentStep.begin(), data.iterationsInCurrentStep.end(), - *data.currentIteration); + current_iteration_idx); it != data.iterationsInCurrentStep.end()) { ++it; @@ -321,7 +404,7 @@ std::optional StatefulIterator::nextIterationInStep() { return no_result(); } - data.currentIteration = *it; + current_iteration_idx = *it; } else { @@ -332,12 +415,12 @@ std::optional StatefulIterator::nextIterationInStep() try { - series.iterations.at(*data.currentIteration).open(); + series.iterations.at(current_iteration_idx).open(); } catch (error::ReadError const &err) { std::cerr << "[StatefulIterator] Cannot read iteration '" - << *data.currentIteration + << *maybeCurrentIteration << "' and will skip it due to read error:\n" << err.what() << std::endl; return nextIterationInStep(); @@ -346,16 +429,16 @@ std::optional StatefulIterator::nextIterationInStep() return {this}; } -std::optional StatefulIterator::nextStep(Seek const &seek) +std::optional +StatefulIterator::nextStep(size_t recursion_depth) { auto &data = get(); // since we are in group-based iteration layout, it does not // matter which iteration we begin a step upon AdvanceStatus status{}; - Iteration::BeginStepStatus::AvailableIterations_t availableIterations; try { - std::tie(status, availableIterations) = Iteration::beginStep( + std::tie(status, data.iterationsInCurrentStep) = Iteration::beginStep( {}, data.series, /* reread = */ reread(data.parsePreference), @@ -367,7 +450,7 @@ std::optional StatefulIterator::nextStep(Seek const &seek) "below, will skip it.\n" << err.what() << std::endl; data.series.advance(AdvanceMode::ENDSTEP); - return nextStep(seek); + return nextStep(recursion_depth + 1); } bool close = [&]() { @@ -381,9 +464,9 @@ std::optional StatefulIterator::nextStep(Seek const &seek) case AdvanceStatus::RANDOMACCESS: return std::visit( auxiliary::overloaded{ - [](Seek::InitNonFileBased_t const &) { return false; }, - [](Seek::Next_t const &) { return true; }}, - seek); + [](CurrentStep::Before_t const &) { return false; }, + [](auto const &) { return true; }}, + data.currentStep); } throw std::runtime_error("Unreachable!"); }(); @@ -394,28 +477,29 @@ std::optional StatefulIterator::nextStep(Seek const &seek) } else { - data.iterationsInCurrentStep = availableIterations; - resetCurrentIterationToBegin(); + resetCurrentIterationToBegin(recursion_depth); } return {this}; } -std::optional StatefulIterator::loopBody(Seek const &seek) +std::optional StatefulIterator::loopBody() { auto &data = get(); Series &series = data.series; auto &iterations = series.iterations; - /* - * Might not be present because parsing might have failed in previous step - */ - if (data.currentIteration.has_value() && - iterations.contains(*data.currentIteration)) - { - auto ¤tIteration = iterations.at(*data.currentIteration); - if (!currentIteration.closed()) + { /* + * Might not be present because parsing might have failed in previous step + */ + auto maybe_current_iteration = data.currentStep.get_iteration_index(); + if (maybe_current_iteration.has_value() && + iterations.contains(**maybe_current_iteration)) { - currentIteration.close(); + auto ¤tIteration = iterations.at(**maybe_current_iteration); + if (!currentIteration.closed()) + { + currentIteration.close(); + } } } @@ -431,36 +515,34 @@ std::optional StatefulIterator::loopBody(Seek const &seek) * This might happen when iterations from the step are ignored, e.g. * a duplicate iteration has been written by Append mode. */ - auto currentIterationIndex = data.currentIteration; - if (!currentIterationIndex.has_value()) + auto maybe_current_iteration = data.currentStep.get_iteration_index(); + if (!maybe_current_iteration.has_value()) { series.advance(AdvanceMode::ENDSTEP); return std::nullopt; } + auto ¤t_iteration = **maybe_current_iteration; // If we had the iteration already, then it's either not there at all // (because old iterations are deleted in linear access mode), // or it's still there but closed in random-access mode - auto index = currentIterationIndex.value(); - if (iterations.contains(index)) + if (iterations.contains(current_iteration)) { - auto iteration = iterations.at(index); + auto iteration = iterations.at(current_iteration); switch (iteration.get().m_closed) { case internal::CloseStatus::ParseAccessDeferred: try { - iterations.at(index).open(); + iterations.at(current_iteration).open(); [[fallthrough]]; } catch (error::ReadError const &err) { - std::cerr << "Cannot read iteration '" - << currentIterationIndex.value() + std::cerr << "Cannot read iteration '" << current_iteration << "' and will skip it due to read error:\n" << err.what() << std::endl; - option.value()->deactivateDeadIteration( - currentIterationIndex.value()); + option.value()->deactivateDeadIteration(current_iteration); return std::nullopt; } case internal::CloseStatus::Open: @@ -483,17 +565,7 @@ std::optional StatefulIterator::loopBody(Seek const &seek) } }; - using res_t = std::optional; - auto optionallyAStep = std::visit( - auxiliary::overloaded{ - [](Seek::InitNonFileBased_t const &) -> res_t { - return std::nullopt; - }, - [this](Seek::Next_t const &) -> res_t { - return nextIterationInStep(); - }}, - seek); - + auto optionallyAStep = nextIterationInStep(); if (optionallyAStep.has_value()) { return guardReturn(optionallyAStep); @@ -509,10 +581,51 @@ std::optional StatefulIterator::loopBody(Seek const &seek) return {this}; } - auto option = nextStep(seek); + auto option = nextStep(/* recursion_depth = */ 0); return guardReturn(option); } +void StatefulIterator::initIteratorFilebased() +{ + auto &data = get(); + auto &series = data.series; + if (series.iterations.empty()) + { + this->close(); + return; + } + data.iterationsInCurrentStep.reserve(series.iterations.size()); + std::transform( + series.iterations.begin(), + series.iterations.end(), + std::back_inserter(data.iterationsInCurrentStep), + [](auto const &pair) { return pair.first; }); + auto it = series.iterations.begin(); + auto end = series.iterations.end(); + for (; it != end; ++it) + { + try + { + it->second.open(); + break; + } + catch (error::ReadError const &err) + { + std::cerr << "[StatefulIterator] Cannot read iteration '" + << it->first << "' and will skip it due to read error:\n" + << err.what() << std::endl; + } + } + if (it != end) + { + data.currentStep = CurrentStep::During_t{0, it->first}; + } + else + { + this->close(); + } +} + void StatefulIterator::deactivateDeadIteration(iteration_index_t index) { auto &data = get(); @@ -541,7 +654,17 @@ void StatefulIterator::deactivateDeadIteration(iteration_index_t index) StatefulIterator &StatefulIterator::operator++() { auto &data = get(); - auto oldIterationIndex = data.currentIteration; + auto oldIterationIndex = [&]() -> std::optional { + auto res = data.currentIteration(); + if (res.has_value()) + { + return **res; + } + else + { + return std::nullopt; + } + }(); std::optional res; /* * loopBody() might return an empty option to indicate a skipped iteration. @@ -552,15 +675,15 @@ StatefulIterator &StatefulIterator::operator++() */ do { - res = loopBody({Seek::Next}); + res = loopBody(); } while (!res.has_value()); auto resvalue = res.value(); if (*resvalue != end()) { auto &series = data.series; - auto index = data.currentIteration; - auto &iteration = series.iterations.at(index.value()); + auto index = data.currentIteration(); + auto &iteration = series.iterations.at(*index.value()); iteration.setStepStatus(StepStatus::DuringStep); if (series.IOHandler()->m_frontendAccess == Access::READ_LINEAR && @@ -587,12 +710,12 @@ StatefulIterator &StatefulIterator::operator++() auto StatefulIterator::operator*() const -> value_type const & { auto &data = get(); - if (data.currentIteration.has_value()) + if (auto cur = data.currentIteration(); cur.has_value()) { auto iterator = static_cast( data.series.iterations) - .find(*data.currentIteration); + .find(**cur); return iterator.operator*(); } else @@ -629,7 +752,7 @@ bool StatefulIterator::operator==(StatefulIterator const &other) const return // either both iterators are filled (this->m_data->has_value() && other.m_data->has_value() && - (this->get().currentIteration == other.get().currentIteration)) || + (this->get().currentIteration() == other.get().currentIteration())) || // or both are empty (!this->m_data->has_value() && !other.m_data->has_value()); } From 8ac9d3877dbde4101c1c33dcf84c2c77c0d9001f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=B6schel?= Date: Fri, 16 Feb 2024 11:04:36 +0100 Subject: [PATCH 42/93] Windows fixes --- include/openPMD/snapshots/ContainerTraits.hpp | 3 ++- include/openPMD/snapshots/RandomAccessIterator.hpp | 4 +++- include/openPMD/snapshots/StatefulIterator.hpp | 4 +++- src/IO/ADIOS/ADIOS2IOHandler.cpp | 1 + src/snapshots/StatefulIterator.cpp | 2 +- 5 files changed, 10 insertions(+), 4 deletions(-) diff --git a/include/openPMD/snapshots/ContainerTraits.hpp b/include/openPMD/snapshots/ContainerTraits.hpp index 8c7b278ecc..90e2d0ee1f 100644 --- a/include/openPMD/snapshots/ContainerTraits.hpp +++ b/include/openPMD/snapshots/ContainerTraits.hpp @@ -14,7 +14,8 @@ class OpaqueSeriesIterator friend class Series; friend class StatefulSnapshotsContainer; friend class RandomAccessIteratorContainer; - using parent_t = AbstractSeriesIterator; + using parent_t = + AbstractSeriesIterator; // no shared_ptr since copied iterators should not share state std::unique_ptr> m_internal_iterator; diff --git a/include/openPMD/snapshots/RandomAccessIterator.hpp b/include/openPMD/snapshots/RandomAccessIterator.hpp index 7b6253d3a6..bc5d494137 100644 --- a/include/openPMD/snapshots/RandomAccessIterator.hpp +++ b/include/openPMD/snapshots/RandomAccessIterator.hpp @@ -41,7 +41,9 @@ class RandomAccessIterator { private: friend class RandomAccessIteratorContainer; - using parent_t = AbstractSeriesIterator>; + using parent_t = AbstractSeriesIterator< + RandomAccessIterator, + detail::iterator_to_value_type>; RandomAccessIterator(iterator_t it); diff --git a/include/openPMD/snapshots/StatefulIterator.hpp b/include/openPMD/snapshots/StatefulIterator.hpp index 6fc6df6c95..1a6becdbf9 100644 --- a/include/openPMD/snapshots/StatefulIterator.hpp +++ b/include/openPMD/snapshots/StatefulIterator.hpp @@ -148,7 +148,9 @@ class StatefulIterator auto get() -> SharedData &; auto get() const -> SharedData const &; - using parent_t = AbstractSeriesIterator; + using parent_t = AbstractSeriesIterator< + StatefulIterator, + Container::value_type>; public: using value_type = diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index bdbd43325a..28524cfcb2 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -1707,6 +1707,7 @@ detail::ADIOS2File &ADIOS2IOHandlerImpl::getFileData( { return *it->second; } + throw std::runtime_error("Unreachable!"); } void ADIOS2IOHandlerImpl::dropFileData(InvalidatableFile const &file) diff --git a/src/snapshots/StatefulIterator.cpp b/src/snapshots/StatefulIterator.cpp index 678fd977a4..59cce4412c 100644 --- a/src/snapshots/StatefulIterator.cpp +++ b/src/snapshots/StatefulIterator.cpp @@ -535,7 +535,6 @@ std::optional StatefulIterator::loopBody() try { iterations.at(current_iteration).open(); - [[fallthrough]]; } catch (error::ReadError const &err) { @@ -545,6 +544,7 @@ std::optional StatefulIterator::loopBody() option.value()->deactivateDeadIteration(current_iteration); return std::nullopt; } + [[fallthrough]]; case internal::CloseStatus::Open: return option; case internal::CloseStatus::ClosedInBackend: From 402cc5bfa1081fa8d86289c33df90535244faf47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 15 Feb 2024 15:49:13 +0100 Subject: [PATCH 43/93] Adapt tests --- test/ParallelIOTest.cpp | 6 +++--- test/SerialIOTest.cpp | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/test/ParallelIOTest.cpp b/test/ParallelIOTest.cpp index e0275eabeb..3853282f2b 100644 --- a/test/ParallelIOTest.cpp +++ b/test/ParallelIOTest.cpp @@ -802,8 +802,8 @@ void close_iteration_test(std::string const &file_ending) it1.close(/* flush = */ true); // illegally access iteration after closing - E_x.storeChunk(data, {mpi_rank, 0}, {1, 4}); - REQUIRE_THROWS(write.flush()); + // E_x.storeChunk(data, {mpi_rank, 0}, {1, 4}); + // REQUIRE_THROWS(write.flush()); } { @@ -817,7 +817,7 @@ void close_iteration_test(std::string const &file_ending) REQUIRE(data[i % 4] == chunk.get()[i]); } auto read_again = E_x_read.loadChunk({0, 0}, {mpi_size, 4}); - REQUIRE_THROWS(read.flush()); + // REQUIRE_THROWS(read.flush()); } chunk_assignment::RankMeta compare; diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index 106ce7b6ce..1ba508f7e0 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -459,7 +459,7 @@ void close_iteration_test(std::string const &file_ending) // illegally access iteration after closing E_x.storeChunk(data, {0, 0}, {2, 2}); - REQUIRE_THROWS(write.flush()); + // REQUIRE_THROWS(write.flush()); } { @@ -473,7 +473,7 @@ void close_iteration_test(std::string const &file_ending) REQUIRE(data[i] == chunk.get()[i]); } auto read_again = E_x_read.loadChunk({0, 0}, {2, 2}); - REQUIRE_THROWS(read.flush()); + // REQUIRE_THROWS(read.flush()); } { @@ -760,7 +760,7 @@ TEST_CASE("close_iteration_throws_test", "[serial]") auto B_y = it0.meshes["B"]["y"]; B_y.resetDataset({Datatype::INT, {5}}); B_y.storeChunk(data, {0}, {5}); - REQUIRE_THROWS(series.flush()); + // REQUIRE_THROWS(series.flush()); } { Series series("../samples/close_iteration_throws_2.bp", Access::CREATE); @@ -774,7 +774,7 @@ TEST_CASE("close_iteration_throws_test", "[serial]") auto e_position_x = it0.particles["e"]["position"]["x"]; e_position_x.resetDataset({Datatype::INT, {5}}); e_position_x.storeChunk(data, {0}, {5}); - REQUIRE_THROWS(series.flush()); + // REQUIRE_THROWS(series.flush()); } { Series series("../samples/close_iteration_throws_3.bp", Access::CREATE); @@ -786,7 +786,7 @@ TEST_CASE("close_iteration_throws_test", "[serial]") it0.close(); it0.setTimeUnitSI(2.0); - REQUIRE_THROWS(series.flush()); + // REQUIRE_THROWS(series.flush()); } } #endif From 1391534ca267703b33dda22db7ae13a759ca1ad2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 15 Feb 2024 15:49:19 +0100 Subject: [PATCH 44/93] Unify close status --- include/openPMD/Iteration.hpp | 4 +-- src/Iteration.cpp | 20 +++----------- src/Series.cpp | 43 +++++++++++------------------- src/snapshots/StatefulIterator.cpp | 4 +-- 4 files changed, 22 insertions(+), 49 deletions(-) diff --git a/include/openPMD/Iteration.hpp b/include/openPMD/Iteration.hpp index 5209372350..54f9875d31 100644 --- a/include/openPMD/Iteration.hpp +++ b/include/openPMD/Iteration.hpp @@ -48,10 +48,8 @@ namespace internal Open, //!< Iteration has not been closed ClosedInFrontend, /*!< Iteration has been closed, but task has not yet been propagated to the backend */ - ClosedInBackend, /*!< Iteration has been closed and task has been + Closed, /*!< Iteration has been closed and task has been propagated to the backend */ - ClosedTemporarily /*!< Iteration has been closed internally and may - be reopened later */ }; struct DeferredParseAccess diff --git a/src/Iteration.cpp b/src/Iteration.cpp index 8052b77efa..c4fcc2a51d 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -99,21 +99,8 @@ Iteration &Iteration::close(bool _flush) case CloseStatus::ClosedInFrontend: it.m_closed = CloseStatus::ClosedInFrontend; break; - case CloseStatus::ClosedTemporarily: - // should we bother to reopen? - if (dirtyRecursive()) - { - // let's reopen - it.m_closed = CloseStatus::ClosedInFrontend; - } - else - { - // don't reopen - it.m_closed = CloseStatus::ClosedInBackend; - } - break; case CloseStatus::ParseAccessDeferred: - case CloseStatus::ClosedInBackend: + case CloseStatus::Closed: // just keep it like it is // (this means that closing an iteration that has not been parsed // yet keeps it re-openable) @@ -172,15 +159,14 @@ bool Iteration::closed() const { case CloseStatus::ParseAccessDeferred: case CloseStatus::Open: + return false; /* * Temporarily closing a file is something that the openPMD API * does for optimization purposes. * Logically to the user, it is still open. */ - case CloseStatus::ClosedTemporarily: - return false; case CloseStatus::ClosedInFrontend: - case CloseStatus::ClosedInBackend: + case CloseStatus::Closed: return true; } throw std::runtime_error("Unreachable!"); diff --git a/src/Series.cpp b/src/Series.cpp index d1d0c6e381..4402a1b6c0 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -1329,8 +1329,7 @@ void Series::flushFileBased( { Parameter fClose; IOHandler()->enqueue(IOTask(&it->second, std::move(fClose))); - it->second.get().m_closed = - internal::CloseStatus::ClosedInBackend; + it->second.get().m_closed = internal::CloseStatus::Closed; } } @@ -1389,8 +1388,7 @@ void Series::flushFileBased( { Parameter fClose; IOHandler()->enqueue(IOTask(&it->second, std::move(fClose))); - it->second.get().m_closed = - internal::CloseStatus::ClosedInBackend; + it->second.get().m_closed = internal::CloseStatus::Closed; } /* reset the dirty bit for every iteration (i.e. file) * otherwise only the first iteration will have updates attributes @@ -1441,8 +1439,7 @@ void Series::flushGorVBased( internal::CloseStatus::ClosedInFrontend) { // the iteration has no dedicated file in group-based mode - it->second.get().m_closed = - internal::CloseStatus::ClosedInBackend; + it->second.get().m_closed = internal::CloseStatus::Closed; } } @@ -1520,8 +1517,7 @@ void Series::flushGorVBased( internal::CloseStatus::ClosedInFrontend) { // the iteration has no dedicated file in group-based mode - it->second.get().m_closed = - internal::CloseStatus::ClosedInBackend; + it->second.get().m_closed = internal::CloseStatus::Closed; } } @@ -1627,7 +1623,7 @@ void Series::readFileBased() Parameter fClose; iteration.IOHandler()->enqueue(IOTask(&iteration, fClose)); iteration.IOHandler()->flush(internal::defaultFlushParams); - iteration.get().m_closed = internal::CloseStatus::ClosedTemporarily; + iteration.get().m_closed = internal::CloseStatus::Closed; return {}; }; std::vector unparseableIterations; @@ -2381,10 +2377,10 @@ AdvanceStatus Series::advance( { // Series::flush() would normally turn a `ClosedInFrontend` into // a `ClosedInBackend`. Do that manually. - itData.m_closed = internal::CloseStatus::ClosedInBackend; + itData.m_closed = internal::CloseStatus::Closed; } else if ( - oldCloseStatus == internal::CloseStatus::ClosedInBackend && + oldCloseStatus == internal::CloseStatus::Closed && series.m_iterationEncoding == IterationEncoding::fileBased) { /* @@ -2392,7 +2388,7 @@ AdvanceStatus Series::advance( * opening an iteration's file by beginning a step on it. * So, return now. */ - iteration.get().m_closed = internal::CloseStatus::ClosedInBackend; + iteration.get().m_closed = internal::CloseStatus::Closed; return AdvanceStatus::OK; } @@ -2402,7 +2398,7 @@ AdvanceStatus Series::advance( } Parameter param; - if (itData.m_closed == internal::CloseStatus::ClosedTemporarily && + if (itData.m_closed == internal::CloseStatus::Closed && series.m_iterationEncoding == IterationEncoding::fileBased) { /* @@ -2433,12 +2429,12 @@ AdvanceStatus Series::advance( switch (series.m_iterationEncoding) { case IE::fileBased: { - if (itData.m_closed != internal::CloseStatus::ClosedTemporarily) + if (itData.m_closed != internal::CloseStatus::Closed) { Parameter fClose; IOHandler()->enqueue(IOTask(&iteration, std::move(fClose))); } - itData.m_closed = internal::CloseStatus::ClosedInBackend; + itData.m_closed = internal::CloseStatus::Closed; break; } case IE::groupBased: { @@ -2448,7 +2444,7 @@ AdvanceStatus Series::advance( // In group-based iteration layout, files are // not closed on a per-iteration basis // We will treat it as such nonetheless - itData.m_closed = internal::CloseStatus::ClosedInBackend; + itData.m_closed = internal::CloseStatus::Closed; break; } case IE::variableBased: // no action necessary @@ -2572,7 +2568,7 @@ auto Series::openIterationIfDirty(IterationIndex_t index, Iteration iteration) return IterationOpened::RemainsClosed; } bool const dirtyRecursive = iteration.dirtyRecursive(); - if (iteration.get().m_closed == internal::CloseStatus::ClosedInBackend) + if (iteration.get().m_closed == internal::CloseStatus::Closed) { // file corresponding with the iteration has previously been // closed and fully flushed @@ -2583,13 +2579,10 @@ auto Series::openIterationIfDirty(IterationIndex_t index, Iteration iteration) "[Series] Closed iteration has not been written. This " "is an internal error."); } - if (dirtyRecursive) + if (!dirtyRecursive) { - throw std::runtime_error( - "[Series] Detected illegal access to iteration that " - "has been closed previously."); + return IterationOpened::RemainsClosed; } - return IterationOpened::RemainsClosed; } switch (iterationEncoding()) @@ -2628,13 +2621,9 @@ void Series::openIteration(IterationIndex_t index, Iteration iteration) switch (oldStatus) { using CL = internal::CloseStatus; - case CL::ClosedInBackend: - throw std::runtime_error( - "[Series] Detected illegal access to iteration that " - "has been closed previously."); + case CL::Closed: case CL::ParseAccessDeferred: case CL::Open: - case CL::ClosedTemporarily: iteration.get().m_closed = CL::Open; break; case CL::ClosedInFrontend: diff --git a/src/snapshots/StatefulIterator.cpp b/src/snapshots/StatefulIterator.cpp index 59cce4412c..72ff08a86c 100644 --- a/src/snapshots/StatefulIterator.cpp +++ b/src/snapshots/StatefulIterator.cpp @@ -547,12 +547,12 @@ std::optional StatefulIterator::loopBody() [[fallthrough]]; case internal::CloseStatus::Open: return option; - case internal::CloseStatus::ClosedInBackend: + case internal::CloseStatus::Closed: + std::cout << "TODO: NEED A BETTER LOGIC FOR THIS" << std::endl; // we had this iteration already, skip it iteration.endStep(); return std::nullopt; // empty, go into next iteration case internal::CloseStatus::ClosedInFrontend: - case internal::CloseStatus::ClosedTemporarily: throw error::Internal("Next found iteration is closed?"); } throw std::runtime_error("Unreachable!"); From 304ae4b8d86d8587c66675db8eb5f393b1ff0d4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 15 Feb 2024 17:17:38 +0100 Subject: [PATCH 45/93] Add basic test for opening after closing --- CMakeLists.txt | 4 +- test/Files_SerialIO/SerialIOTests.hpp | 6 ++ test/Files_SerialIO/filebased_write_test.cpp | 62 ++++++++++++++++++++ test/SerialIOTest.cpp | 5 ++ 4 files changed, 76 insertions(+), 1 deletion(-) create mode 100644 test/Files_SerialIO/SerialIOTests.hpp create mode 100644 test/Files_SerialIO/filebased_write_test.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 7c4cf15ee0..a4b80070ed 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -853,7 +853,9 @@ if(openPMD_BUILD_TESTING) endif() foreach(testname ${openPMD_TEST_NAMES}) - add_executable(${testname}Tests test/${testname}Test.cpp) + file(GLOB ADDITIONAL_SOURCE_FILES test/Files_${testname}/*.cpp) + add_executable(${testname}Tests test/${testname}Test.cpp ${ADDITIONAL_SOURCE_FILES}) + target_include_directories(${testname}Tests PRIVATE test/Files_${testname}/) openpmd_cxx_required(${testname}Tests) set_target_properties(${testname}Tests PROPERTIES COMPILE_PDB_NAME ${testname}Tests diff --git a/test/Files_SerialIO/SerialIOTests.hpp b/test/Files_SerialIO/SerialIOTests.hpp new file mode 100644 index 0000000000..7d4d1d699d --- /dev/null +++ b/test/Files_SerialIO/SerialIOTests.hpp @@ -0,0 +1,6 @@ +#include + +namespace filebased_write_test +{ +void close_and_reopen_iterations(std::string const &filename); +} diff --git a/test/Files_SerialIO/filebased_write_test.cpp b/test/Files_SerialIO/filebased_write_test.cpp new file mode 100644 index 0000000000..de42b5f2fb --- /dev/null +++ b/test/Files_SerialIO/filebased_write_test.cpp @@ -0,0 +1,62 @@ +#include "SerialIOTests.hpp" + +namespace filebased_write_test +{ +using namespace openPMD; + +void close_and_reopen_iterations(const std::string &filename) +{ + Series list(filename, Access::READ_ONLY); + + auto test_read = [](Iteration &iteration) { + auto component = iteration.particles["e"]["position"]["x"]; + auto chunk = component.loadChunkVariant(); + iteration.seriesFlush(); + auto num_particles = component.getExtent()[0]; + std::cout << "Particles: "; + if (num_particles > 0) + { + std::visit( + [&](auto const &shared_ptr) { + auto it = shared_ptr.get(); + auto end = it + num_particles; + std::cout << '[' << *it++; + for (; it != end; ++it) + { + std::cout << ", " << *it; + } + }, + chunk); + std::cout << "]"; + } + else + { + std::cout << "[]"; + } + std::cout << std::endl; + }; + + for (auto &[idx, iteration] : list.snapshots()) + { + std::cout << "Seeing iteration " << idx << std::endl; + if (iteration.particles.contains("e")) + { + test_read(iteration); + } + std::cout << "Closing iteration " << idx << std::endl; + iteration.close(); + } + std::cout << "Trying to read iteration 3 out of line" << std::endl; + test_read(list.snapshots()[3]); + + std::cout << "----------\nGoing again\n----------" << std::endl; + for (auto &[idx, iteration] : list.snapshots()) + { + std::cout << "Seeing iteration " << idx << std::endl; + if (iteration.particles.contains("e")) + { + test_read(iteration); + } + } +} +} // namespace filebased_write_test diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index 1ba508f7e0..e754b1e0dd 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -7,6 +7,8 @@ #define OPENPMD_protected public: #endif +#include "SerialIOTests.hpp" + #include "openPMD/IO/ADIOS/macros.hpp" #include "openPMD/auxiliary/Environment.hpp" #include "openPMD/auxiliary/Filesystem.hpp" @@ -2221,6 +2223,9 @@ inline void fileBased_write_test(const std::string &backend) close(dirfd); } #endif // defined(__unix__) + + filebased_write_test::close_and_reopen_iterations( + "../samples/subdir/serial_fileBased_write%T." + backend); } TEST_CASE("fileBased_write_test", "[serial]") From cf5b0978091088d9d0bce69aaaead515d90fea45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 16 Feb 2024 14:34:14 +0100 Subject: [PATCH 46/93] Add new end() iterator representations --- .../openPMD/snapshots/StatefulIterator.hpp | 18 ++++++ src/snapshots/StatefulIterator.cpp | 59 ++++++++++++++++--- test/Files_SerialIO/filebased_write_test.cpp | 2 +- 3 files changed, 71 insertions(+), 8 deletions(-) diff --git a/include/openPMD/snapshots/StatefulIterator.hpp b/include/openPMD/snapshots/StatefulIterator.hpp index 1a6becdbf9..bea99c0b15 100644 --- a/include/openPMD/snapshots/StatefulIterator.hpp +++ b/include/openPMD/snapshots/StatefulIterator.hpp @@ -189,6 +189,17 @@ class StatefulIterator auto operator<(StatefulIterator const &) const -> bool; static auto end() -> StatefulIterator; + /* + * This is considered an end iterator if: + * + * 1. The iterator has no state at all + * (generic statically created end iterator) + * 2. The state is During_t with no iteration index + * (finished reading iterations in a randomly-accessible Series) + * 3. The state is After_t + * (closed the last step in a step-wise Series) + */ + auto is_end() const -> bool; operator bool() const; @@ -216,6 +227,13 @@ class StatefulIterator void initSeriesInLinearReadMode(); void close(); + enum class TypeOfEndIterator : char + { + NoMoreSteps, + NoMoreIterationsInStep + }; + auto turn_into_end_iterator(TypeOfEndIterator) -> void; + auto assert_end_iterator() const -> void; auto resetCurrentIterationToBegin(size_t num_skipped_iterations) -> void; auto peekCurrentlyOpenIteration() const diff --git a/src/snapshots/StatefulIterator.cpp b/src/snapshots/StatefulIterator.cpp index 72ff08a86c..1135086e0c 100644 --- a/src/snapshots/StatefulIterator.cpp +++ b/src/snapshots/StatefulIterator.cpp @@ -233,6 +233,28 @@ void StatefulIterator::close() *m_data = std::nullopt; // turn this into end iterator } +auto StatefulIterator::turn_into_end_iterator(TypeOfEndIterator type) -> void +{ + auto &data = get(); + switch (type) + { + case TypeOfEndIterator::NoMoreSteps: + data.currentStep = CurrentStep::After; + break; + case TypeOfEndIterator::NoMoreIterationsInStep: + data.currentStep.map_during_t( + [](CurrentStep::During_t &during) { + during.iteration_idx = std::nullopt; + }, + [](auto const &) { + return CurrentStep::During_t{0, std::nullopt}; + } + + ); + break; + } +} + auto StatefulIterator::resetCurrentIterationToBegin( size_t num_skipped_iterations) -> void { @@ -363,7 +385,7 @@ StatefulIterator::StatefulIterator( case IterationEncoding::variableBased: if (!loopBody().has_value()) { - this->close(); + this->assert_end_iterator(); } break; } @@ -473,7 +495,7 @@ StatefulIterator::nextStep(size_t recursion_depth) if (close) { - this->close(); + this->turn_into_end_iterator(TypeOfEndIterator::NoMoreSteps); } else { @@ -506,7 +528,9 @@ std::optional StatefulIterator::loopBody() auto guardReturn = [&series, &iterations, &data]( auto const &option) -> std::optional { - if (!option.has_value() || *option.value() == end()) + if (!option.has_value() || + std::holds_alternative( + (*option)->get().currentStep)) { return option; } @@ -577,7 +601,7 @@ std::optional StatefulIterator::loopBody() if (series.iterationEncoding() == IterationEncoding::fileBased) { // this one is handled above, stream is over once it proceeds to here - this->close(); + this->turn_into_end_iterator(TypeOfEndIterator::NoMoreIterationsInStep); return {this}; } @@ -591,7 +615,7 @@ void StatefulIterator::initIteratorFilebased() auto &series = data.series; if (series.iterations.empty()) { - this->close(); + this->turn_into_end_iterator(TypeOfEndIterator::NoMoreIterationsInStep); return; } data.iterationsInCurrentStep.reserve(series.iterations.size()); @@ -622,7 +646,7 @@ void StatefulIterator::initIteratorFilebased() } else { - this->close(); + this->turn_into_end_iterator(TypeOfEndIterator::NoMoreIterationsInStep); } } @@ -754,7 +778,7 @@ bool StatefulIterator::operator==(StatefulIterator const &other) const (this->m_data->has_value() && other.m_data->has_value() && (this->get().currentIteration() == other.get().currentIteration())) || // or both are empty - (!this->m_data->has_value() && !other.m_data->has_value()); + (this->is_end() && other.is_end()); } auto StatefulIterator::operator<(StatefulIterator const &) const -> bool { @@ -767,6 +791,27 @@ StatefulIterator StatefulIterator::end() return StatefulIterator{}; } +auto StatefulIterator::is_end() const -> bool +{ + return !m_data || !m_data->has_value() || + std::visit( + auxiliary::overloaded{ + [](CurrentStep::Before_t const &) { return false; }, + [](CurrentStep::During_t const &during) { + return !during.iteration_idx.has_value(); + }, + [](CurrentStep::After_t const &) { return true; }}, + (**m_data).currentStep); +} + +auto StatefulIterator::assert_end_iterator() const -> void +{ + if (!is_end()) + { + throw error::Internal("Assertion error: Iterator is no end iterator."); + } +} + StatefulIterator::operator bool() const { return m_data->has_value(); diff --git a/test/Files_SerialIO/filebased_write_test.cpp b/test/Files_SerialIO/filebased_write_test.cpp index de42b5f2fb..72892978ad 100644 --- a/test/Files_SerialIO/filebased_write_test.cpp +++ b/test/Files_SerialIO/filebased_write_test.cpp @@ -6,7 +6,7 @@ using namespace openPMD; void close_and_reopen_iterations(const std::string &filename) { - Series list(filename, Access::READ_ONLY); + Series list(filename, Access::READ_LINEAR); auto test_read = [](Iteration &iteration) { auto component = iteration.particles["e"]["position"]["x"]; From a5f30a82884e2780c17572449584b796d2a821d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 16 Feb 2024 15:39:12 +0100 Subject: [PATCH 47/93] Reopening logic in Iterator, not yet in Series itself --- .../openPMD/snapshots/StatefulIterator.hpp | 27 +++++++- src/snapshots/ContainerImpls.cpp | 5 +- src/snapshots/StatefulIterator.cpp | 68 ++++++++++++++++--- 3 files changed, 88 insertions(+), 12 deletions(-) diff --git a/include/openPMD/snapshots/StatefulIterator.hpp b/include/openPMD/snapshots/StatefulIterator.hpp index bea99c0b15..7a7bc5229a 100644 --- a/include/openPMD/snapshots/StatefulIterator.hpp +++ b/include/openPMD/snapshots/StatefulIterator.hpp @@ -32,6 +32,7 @@ #include #include #include +#include #include namespace openPMD @@ -96,6 +97,24 @@ namespace detail template auto map_during_t(F &&map); }; + + namespace seek_types + { + struct Next_t + {}; + struct Seek_Iteration_t + { + Iteration::IterationIndex_t iteration_idx; + }; + } // namespace seek_types + + struct Seek : std::variant + { + using Next_t = seek_types::Next_t; + using Seek_Iteration_t = seek_types::Seek_Iteration_t; + + static constexpr Next_t const Next{}; + }; } // namespace detail class StatefulIterator @@ -156,6 +175,8 @@ class StatefulIterator using value_type = typename Container::value_type; using typename parent_t ::difference_type; + using Seek = detail::Seek; + //! construct the end() iterator explicit StatefulIterator(); @@ -203,8 +224,12 @@ class StatefulIterator operator bool() const; + // Custom non-iterator methods + auto seek(Seek const &) -> StatefulIterator *; + private: std::optional nextIterationInStep(); + std::optional skipToIterationInStep(iteration_index_t); /* * When a step cannot successfully be opened, the method nextStep() calls @@ -218,7 +243,7 @@ class StatefulIterator */ std::optional nextStep(size_t recursion_depth); - std::optional loopBody(); + std::optional loopBody(Seek const &); void initIteratorFilebased(); diff --git a/src/snapshots/ContainerImpls.cpp b/src/snapshots/ContainerImpls.cpp index 0a788172e5..159f407558 100644 --- a/src/snapshots/ContainerImpls.cpp +++ b/src/snapshots/ContainerImpls.cpp @@ -210,9 +210,8 @@ auto StatefulSnapshotsContainer::operator[](key_type const &key) } else if (access::read(access)) { - throw std::runtime_error( - "Jumping to existing iteration not implemented in stateful " - "container/iterator."); + auto result = it->seek({StatefulIterator::Seek::Seek_Iteration_t{key}}); + return (*result)->second; } throw error::Internal("Control flow error: This should be unreachable."); } diff --git a/src/snapshots/StatefulIterator.cpp b/src/snapshots/StatefulIterator.cpp index 1135086e0c..23f5fa8cb8 100644 --- a/src/snapshots/StatefulIterator.cpp +++ b/src/snapshots/StatefulIterator.cpp @@ -383,7 +383,7 @@ StatefulIterator::StatefulIterator( } case IterationEncoding::groupBased: case IterationEncoding::variableBased: - if (!loopBody().has_value()) + if (!loopBody({Seek::Next}).has_value()) { this->assert_end_iterator(); } @@ -451,6 +451,34 @@ std::optional StatefulIterator::nextIterationInStep() return {this}; } +auto StatefulIterator::skipToIterationInStep(Iteration::IterationIndex_t idx) + -> std::optional +{ + auto &data = get(); + auto maybeCurrentIteration = + data.currentStep.get_variant(); + + if (!maybeCurrentIteration.has_value()) + { + return std::nullopt; + } + CurrentStep::During_t ¤tIteration = **maybeCurrentIteration; + + if (std::find( + data.iterationsInCurrentStep.begin(), + data.iterationsInCurrentStep.end(), + idx) == data.iterationsInCurrentStep.end()) + { + return std::nullopt; + } + + // This is called upon user request, i.e. ReadErrors should be caught in + // user code + data.series.iterations.at(idx).open(); + currentIteration.iteration_idx = idx; + return {this}; +} + std::optional StatefulIterator::nextStep(size_t recursion_depth) { @@ -504,7 +532,7 @@ StatefulIterator::nextStep(size_t recursion_depth) return {this}; } -std::optional StatefulIterator::loopBody() +std::optional StatefulIterator::loopBody(Seek const &seek) { auto &data = get(); Series &series = data.series; @@ -589,10 +617,19 @@ std::optional StatefulIterator::loopBody() } }; - auto optionallyAStep = nextIterationInStep(); - if (optionallyAStep.has_value()) { - return guardReturn(optionallyAStep); + std::optional optionallyAStep = std::visit( + auxiliary::overloaded{ + [&](Seek::Next_t) { return nextIterationInStep(); }, + [&](Seek::Seek_Iteration_t const &skip_to_iteration) { + return skipToIterationInStep( + skip_to_iteration.iteration_idx); + }}, + seek); + if (optionallyAStep.has_value()) + { + return guardReturn(optionallyAStep); + } } // The currently active iterations have been exhausted. @@ -605,7 +642,17 @@ std::optional StatefulIterator::loopBody() return {this}; } - auto option = nextStep(/* recursion_depth = */ 0); + auto option = std::visit( + auxiliary::overloaded{ + [&](Seek::Next_t) { return nextStep(/* recursion_depth = */ 0); }, + [](Seek::Seek_Iteration_t const &skip_to_iteration) + -> std::optional { + throw std::runtime_error( + "[StatefulIterator] Skipping to iteration " + + std::to_string(skip_to_iteration.iteration_idx) + + "from another step not supported yet"); + }}, + seek); return guardReturn(option); } @@ -676,6 +723,11 @@ void StatefulIterator::deactivateDeadIteration(iteration_index_t index) } StatefulIterator &StatefulIterator::operator++() +{ + return *seek({Seek::Next}); +} + +auto StatefulIterator::seek(Seek const &seek) -> StatefulIterator * { auto &data = get(); auto oldIterationIndex = [&]() -> std::optional { @@ -699,7 +751,7 @@ StatefulIterator &StatefulIterator::operator++() */ do { - res = loopBody(); + res = loopBody(seek); } while (!res.has_value()); auto resvalue = res.value(); @@ -728,7 +780,7 @@ StatefulIterator &StatefulIterator::operator++() data.ignoreIterations.emplace(*oldIterationIndex); } } - return *resvalue; + return resvalue; } auto StatefulIterator::operator*() const -> value_type const & From 52029397ba9c5d995e74824ab9db0500fe7bad44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 16 Feb 2024 17:29:34 +0100 Subject: [PATCH 48/93] Reopening fundamentally working in READ_LINEAR --- include/openPMD/Iteration.hpp | 7 +-- include/openPMD/Series.hpp | 5 +- .../openPMD/snapshots/StatefulIterator.hpp | 8 +-- src/Iteration.cpp | 9 ++- src/Series.cpp | 55 +++++++++++++++---- src/snapshots/StatefulIterator.cpp | 41 ++++++++++++-- 6 files changed, 91 insertions(+), 34 deletions(-) diff --git a/include/openPMD/Iteration.hpp b/include/openPMD/Iteration.hpp index 54f9875d31..b3fb6e9334 100644 --- a/include/openPMD/Iteration.hpp +++ b/include/openPMD/Iteration.hpp @@ -355,11 +355,8 @@ class Iteration : public Attributable * Useful in group-based iteration encoding where the Iteration will only * be known after opening the step. */ - static BeginStepStatus beginStep( - std::optional thisObject, - Series &series, - bool reread, - std::set const &ignoreIterations = {}); + static BeginStepStatus + beginStep(std::optional thisObject, Series &series, bool reread); /** * @brief End an IO step on the IO file (or file-like object) diff --git a/include/openPMD/Series.hpp b/include/openPMD/Series.hpp index f185502622..e0ebb8f3b7 100644 --- a/include/openPMD/Series.hpp +++ b/include/openPMD/Series.hpp @@ -799,7 +799,8 @@ OPENPMD_private void flushMeshesPath(); void flushParticlesPath(); void flushRankTable(); - void readFileBased(); + void readFileBased( + std::optional read_only_this_single_iteration); void readOneIterationFileBased(std::string const &filePath); /** * Note on re-parsing of a Series: @@ -817,7 +818,7 @@ OPENPMD_private std::vector readGorVBased( bool do_always_throw_errors, bool init, - std::set const &ignoreIterations = {}); + std::optional read_only_this_single_iteration); void readBase(); std::string iterationFilename(IterationIndex_t i); diff --git a/include/openPMD/snapshots/StatefulIterator.hpp b/include/openPMD/snapshots/StatefulIterator.hpp index 7a7bc5229a..3ea4a4ff69 100644 --- a/include/openPMD/snapshots/StatefulIterator.hpp +++ b/include/openPMD/snapshots/StatefulIterator.hpp @@ -146,12 +146,6 @@ class StatefulIterator std::vector iterationsInCurrentStep; CurrentStep currentStep = {CurrentStep::Before}; std::optional parsePreference; - /* - * Necessary because in the old ADIOS2 schema, old iterations' metadata - * will leak into new steps, making the frontend think that the groups - * are still there and the iterations can be parsed again. - */ - std::set ignoreIterations; auto currentIteration() -> std::optional; auto currentIteration() const @@ -264,6 +258,8 @@ class StatefulIterator auto peekCurrentlyOpenIteration() const -> std::optional; auto peekCurrentlyOpenIteration() -> std::optional; + + auto reparse_possibly_deleted_iteration(iteration_index_t) -> void; }; class LegacyIteratorAdaptor diff --git a/src/Iteration.cpp b/src/Iteration.cpp index c4fcc2a51d..05cedb4b53 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -35,6 +35,7 @@ #include #include #include +#include #include #include @@ -684,10 +685,8 @@ auto Iteration::beginStep(bool reread) -> BeginStepStatus } auto Iteration::beginStep( - std::optional thisObject, - Series &series, - bool reread, - std::set const &ignoreIterations) -> BeginStepStatus + std::optional thisObject, Series &series, bool reread) + -> BeginStepStatus { BeginStepStatus res; using IE = IterationEncoding; @@ -756,7 +755,7 @@ auto Iteration::beginStep( res.iterationsInOpenedStep = series.readGorVBased( /* do_always_throw_errors = */ true, /* init = */ false, - ignoreIterations); + /* read_only_this_single_iteration = */ std::nullopt); } catch (...) { diff --git a/src/Series.cpp b/src/Series.cpp index 4402a1b6c0..515386da6a 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -1140,10 +1140,13 @@ Given file pattern: ')END" try { if (input->iterationEncoding == IterationEncoding::fileBased) - readFileBased(); + readFileBased( + /* read_only_this_single_iteration = */ std::nullopt); else readGorVBased( - /* do_always_throw_errors = */ false, /* init = */ true); + /* do_always_throw_errors = */ false, + /* init = */ true, + /* read_only_this_single_iteration = */ std::nullopt); if (series.iterations.empty()) { @@ -1551,7 +1554,8 @@ void Series::flushParticlesPath() IOHandler()->enqueue(IOTask(this, aWrite)); } -void Series::readFileBased() +void Series::readFileBased( + std::optional read_only_this_single_iteration) { auto &series = get(); Parameter fOpen; @@ -1581,8 +1585,14 @@ void Series::readFileBased() isPartOfSeries, IOHandler()->directory, // foreach found file with `filename` and `index`: - [&series](std::string const &filename, Match const &match) { + [&series, &read_only_this_single_iteration]( + std::string const &filename, Match const &match) { auto index = match.iteration; + if (read_only_this_single_iteration.has_value() && + *read_only_this_single_iteration != index) + { + return; + } Iteration &i = series.iterations[index]; i.deferParseAccess( {std::to_string(index), @@ -1639,6 +1649,11 @@ void Series::readFileBased() std::optional forwardFirstError; for (auto &pair : series.iterations) { + if (read_only_this_single_iteration.has_value() && + *read_only_this_single_iteration != pair.first) + { + continue; + } if (auto error = readIterationEagerly(pair.second); error) { std::cerr << "Cannot read iteration '" << pair.first @@ -1688,6 +1703,11 @@ void Series::readFileBased() std::optional forwardFirstError; for (auto &iteration : series.iterations) { + if (read_only_this_single_iteration.has_value() && + *read_only_this_single_iteration != iteration.first) + { + continue; + } if (auto error = readIterationEagerly(iteration.second); error) { std::cerr << "Cannot read iteration '" << iteration.first @@ -1872,7 +1892,7 @@ namespace auto Series::readGorVBased( bool do_always_throw_errors, bool do_init, - std::set const &ignoreIterations) + std::optional read_only_this_single_iteration) -> std::vector { auto &series = get(); @@ -2077,7 +2097,8 @@ creating new iterations. for (auto const &it : *pList.paths) { IterationIndex_t index = std::stoull(it); - if (ignoreIterations.find(index) != ignoreIterations.end()) + if (read_only_this_single_iteration.has_value() && + index != *read_only_this_single_iteration) { continue; } @@ -2114,14 +2135,28 @@ creating new iterations. } } case IterationEncoding::variableBased: { - if (currentSteps.has_value() && !currentSteps.value().empty()) + if (!currentSteps.has_value() || currentSteps.value().empty()) { - vectorDifference(*currentSteps, ignoreIterations); + currentSteps = std::vector{ + read_only_this_single_iteration.has_value() + ? *read_only_this_single_iteration + : 0}; } - else + else if (read_only_this_single_iteration.has_value()) { - currentSteps = std::vector{0}; + if (std::find( + currentSteps->begin(), + currentSteps->end(), + *read_only_this_single_iteration) != currentSteps->end()) + { + *currentSteps = {*read_only_this_single_iteration}; + } + else + { + currentSteps->clear(); + } } + for (auto it : *currentSteps) { /* diff --git a/src/snapshots/StatefulIterator.cpp b/src/snapshots/StatefulIterator.cpp index 23f5fa8cb8..f6babc5b58 100644 --- a/src/snapshots/StatefulIterator.cpp +++ b/src/snapshots/StatefulIterator.cpp @@ -180,7 +180,7 @@ void StatefulIterator::initSeriesInLinearReadMode() { using IE = IterationEncoding; case IE::fileBased: - series.readFileBased(); + series.readFileBased(std::nullopt); break; case IE::groupBased: case IE::variableBased: { @@ -194,11 +194,15 @@ void StatefulIterator::initSeriesInLinearReadMode() case PP::PerStep: series.advance(AdvanceMode::BEGINSTEP); series.readGorVBased( - /* do_always_throw_errors = */ false, /* init = */ true); + /* do_always_throw_errors = */ false, + /* init = */ true, + std::nullopt); break; case PP::UpFront: series.readGorVBased( - /* do_always_throw_errors = */ false, /* init = */ true); + /* do_always_throw_errors = */ false, + /* init = */ true, + std::nullopt); /* * In linear read mode (where no parsing at all is done upon * constructing the Series), it might turn out after parsing @@ -331,6 +335,31 @@ auto StatefulIterator::peekCurrentlyOpenIteration() } } +auto StatefulIterator::reparse_possibly_deleted_iteration(iteration_index_t idx) + -> void +{ + auto &data = get(); + if (!data.series.iterations.contains(idx)) + { + withRWAccess(data.series.IOHandler()->m_seriesStatus, [&]() { + switch (data.series.iterationEncoding()) + { + + case IterationEncoding::fileBased: + data.series.readFileBased({idx}); + break; + case IterationEncoding::groupBased: + case IterationEncoding::variableBased: + data.series.readGorVBased( + /* do_always_throw_errors = */ true, + /* init = */ false, + {idx}); + break; + } + }); + } +} + StatefulIterator::StatefulIterator(tag_write_t, Series const &series_in) : m_data{std::make_shared>(std::in_place)} { @@ -437,6 +466,7 @@ std::optional StatefulIterator::nextIterationInStep() try { + reparse_possibly_deleted_iteration(current_iteration_idx); series.iterations.at(current_iteration_idx).open(); } catch (error::ReadError const &err) @@ -474,6 +504,7 @@ auto StatefulIterator::skipToIterationInStep(Iteration::IterationIndex_t idx) // This is called upon user request, i.e. ReadErrors should be caught in // user code + reparse_possibly_deleted_iteration(idx); data.series.iterations.at(idx).open(); currentIteration.iteration_idx = idx; return {this}; @@ -491,8 +522,7 @@ StatefulIterator::nextStep(size_t recursion_depth) std::tie(status, data.iterationsInCurrentStep) = Iteration::beginStep( {}, data.series, - /* reread = */ reread(data.parsePreference), - data.ignoreIterations); + /* reread = */ reread(data.parsePreference)); } catch (error::ReadError const &err) { @@ -777,7 +807,6 @@ auto StatefulIterator::seek(Seek const &seek) -> StatefulIterator * */ auto &container = series.iterations.container(); container.erase(*oldIterationIndex); - data.ignoreIterations.emplace(*oldIterationIndex); } } return resvalue; From 27ba327b74bfeedf56ab0459b532e7ccc9097e38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 16 Feb 2024 17:47:38 +0100 Subject: [PATCH 49/93] Extend test still sth wrong in append_mode test, but see about this next week --- test/Files_SerialIO/filebased_write_test.cpp | 32 ++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/test/Files_SerialIO/filebased_write_test.cpp b/test/Files_SerialIO/filebased_write_test.cpp index 72892978ad..3c2738a9af 100644 --- a/test/Files_SerialIO/filebased_write_test.cpp +++ b/test/Files_SerialIO/filebased_write_test.cpp @@ -4,9 +4,13 @@ namespace filebased_write_test { using namespace openPMD; -void close_and_reopen_iterations(const std::string &filename) +void close_and_reopen_iterations( + const std::string &filename, + openPMD::Access access, + std::string const &json_config, + bool need_to_explitly_open_iterations) { - Series list(filename, Access::READ_LINEAR); + Series list(filename, access, json_config); auto test_read = [](Iteration &iteration) { auto component = iteration.particles["e"]["position"]["x"]; @@ -39,6 +43,10 @@ void close_and_reopen_iterations(const std::string &filename) for (auto &[idx, iteration] : list.snapshots()) { std::cout << "Seeing iteration " << idx << std::endl; + if (need_to_explitly_open_iterations) + { + iteration.open(); + } if (iteration.particles.contains("e")) { test_read(iteration); @@ -47,16 +55,36 @@ void close_and_reopen_iterations(const std::string &filename) iteration.close(); } std::cout << "Trying to read iteration 3 out of line" << std::endl; + if (need_to_explitly_open_iterations) + { + list.snapshots()[3].open(); + } test_read(list.snapshots()[3]); std::cout << "----------\nGoing again\n----------" << std::endl; for (auto &[idx, iteration] : list.snapshots()) { std::cout << "Seeing iteration " << idx << std::endl; + if (need_to_explitly_open_iterations) + { + iteration.open(); + } if (iteration.particles.contains("e")) { test_read(iteration); } } } + +void close_and_reopen_iterations(std::string const &filename) +{ + close_and_reopen_iterations( + filename, Access::READ_LINEAR, "defer_iteration_parsing=false", false); + close_and_reopen_iterations( + filename, Access::READ_LINEAR, "defer_iteration_parsing=true", false); + close_and_reopen_iterations( + filename, Access::READ_ONLY, "defer_iteration_parsing=false", false); + close_and_reopen_iterations( + filename, Access::READ_ONLY, "defer_iteration_parsing=true", true); +} } // namespace filebased_write_test From bb0a776e0f52af986335ded7a092aeb8ab035e2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 19 Feb 2024 10:33:31 +0100 Subject: [PATCH 50/93] For now, adapt the append_mode test --- test/ParallelIOTest.cpp | 8 ++++---- test/SerialIOTest.cpp | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/test/ParallelIOTest.cpp b/test/ParallelIOTest.cpp index 3853282f2b..670f2e9926 100644 --- a/test/ParallelIOTest.cpp +++ b/test/ParallelIOTest.cpp @@ -1689,14 +1689,14 @@ void append_mode( // time but as they go Series read(filename, Access::READ_LINEAR, MPI_COMM_WORLD); unsigned counter = 0; - uint64_t iterationOrder[] = {0, 1, 3, 2, 4, 10, 7, 11}; + uint64_t iterationOrder[] = {0, 1, 3, 2, 4, 3, 10, 7, 1, 11}; for (auto iteration : read.readIterations()) { REQUIRE(iteration.iterationIndex == iterationOrder[counter]); verifyIteration(iteration); ++counter; } - REQUIRE(counter == 8); + REQUIRE(counter == 10); // listSeries will not see any iterations since they have already // been read helper::listSeries(read); @@ -1802,7 +1802,7 @@ void append_mode( // in variable-based encodings, iterations are not parsed ahead // of time but as they go unsigned counter = 0; - uint64_t iterationOrder[] = {0, 1, 3, 2, 4, 10, 7, 5}; + uint64_t iterationOrder[] = {0, 1, 3, 2, 4, 3, 10, 7, 4, 5}; for (auto iteration : read.readIterations()) { REQUIRE( @@ -1810,7 +1810,7 @@ void append_mode( verifyIteration(iteration); ++counter; } - REQUIRE(counter == 8); + REQUIRE(counter == 10); } break; default: diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index e754b1e0dd..4722ebc32b 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -7209,14 +7209,14 @@ void append_mode( // time but as they go Series read(filename, Access::READ_LINEAR); unsigned counter = 0; - uint64_t iterationOrder[] = {0, 1, 3, 2, 4, 10, 7, 11}; + uint64_t iterationOrder[] = {0, 1, 3, 2, 4, 3, 10, 7, 1, 11}; for (auto iteration : read.readIterations()) { REQUIRE(iteration.iterationIndex == iterationOrder[counter]); verifyIteration(iteration); ++counter; } - REQUIRE(counter == 8); + REQUIRE(counter == 10); // listSeries will not see any iterations since they have already // been read helper::listSeries(read); From 849fb9153300e05b243fd820fb1db65192d24911 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 19 Feb 2024 11:59:39 +0100 Subject: [PATCH 51/93] fixes --- src/snapshots/StatefulIterator.cpp | 49 +++++++++--------------------- test/SerialIOTest.cpp | 9 ++++++ 2 files changed, 24 insertions(+), 34 deletions(-) diff --git a/src/snapshots/StatefulIterator.cpp b/src/snapshots/StatefulIterator.cpp index f6babc5b58..4bfe3932e3 100644 --- a/src/snapshots/StatefulIterator.cpp +++ b/src/snapshots/StatefulIterator.cpp @@ -265,6 +265,7 @@ auto StatefulIterator::resetCurrentIterationToBegin( auto &data = get(); data.currentStep.map_during_t( [&](CurrentStep::During_t &during) { + during.idx += num_skipped_iterations; if (data.iterationsInCurrentStep.empty()) { during.iteration_idx = std::nullopt; @@ -412,7 +413,7 @@ StatefulIterator::StatefulIterator( } case IterationEncoding::groupBased: case IterationEncoding::variableBased: - if (!loopBody({Seek::Next}).has_value()) + if (!seek({Seek::Next})) { this->assert_end_iterator(); } @@ -557,6 +558,11 @@ StatefulIterator::nextStep(size_t recursion_depth) } else { + if (auto during = data.currentStep.get_variant(); + during.has_value()) + { + ++(*during)->idx; + } resetCurrentIterationToBegin(recursion_depth); } return {this}; @@ -580,6 +586,11 @@ std::optional StatefulIterator::loopBody(Seek const &seek) { currentIteration.close(); } + if (series.IOHandler()->m_frontendAccess == Access::READ_LINEAR) + { + data.series.iterations.container().erase( + **maybe_current_iteration); + } } } @@ -630,12 +641,10 @@ std::optional StatefulIterator::loopBody(Seek const &seek) case internal::CloseStatus::Open: return option; case internal::CloseStatus::Closed: - std::cout << "TODO: NEED A BETTER LOGIC FOR THIS" << std::endl; - // we had this iteration already, skip it - iteration.endStep(); - return std::nullopt; // empty, go into next iteration case internal::CloseStatus::ClosedInFrontend: - throw error::Internal("Next found iteration is closed?"); + throw error::Internal( + "[StatefulIterator] Next step returned an iteration that " + "is already closed, should have opened it."); } throw std::runtime_error("Unreachable!"); } @@ -760,17 +769,6 @@ StatefulIterator &StatefulIterator::operator++() auto StatefulIterator::seek(Seek const &seek) -> StatefulIterator * { auto &data = get(); - auto oldIterationIndex = [&]() -> std::optional { - auto res = data.currentIteration(); - if (res.has_value()) - { - return **res; - } - else - { - return std::nullopt; - } - }(); std::optional res; /* * loopBody() might return an empty option to indicate a skipped iteration. @@ -791,23 +789,6 @@ auto StatefulIterator::seek(Seek const &seek) -> StatefulIterator * auto index = data.currentIteration(); auto &iteration = series.iterations.at(*index.value()); iteration.setStepStatus(StepStatus::DuringStep); - - if (series.IOHandler()->m_frontendAccess == Access::READ_LINEAR && - oldIterationIndex.has_value()) - { - /* - * Linear read mode: Any data outside the current iteration is - * inaccessible. Delete the iteration. This has two effects: - * - * 1) Avoid confusion. - * 2) Avoid memory buildup in long-running workflows with many - * iterations. - * - * @todo Also delete data in the backends upon doing this. - */ - auto &container = series.iterations.container(); - container.erase(*oldIterationIndex); - } } return resvalue; } diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index 4722ebc32b..f2489315b5 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -5663,18 +5663,24 @@ void adios2_group_table( write.writeIterations()[1].meshes["E"]["x"].makeEmpty(Datatype::FLOAT, 1); write.close(); + size_t counter = 0; + bool saw_iteration_0{false}, saw_iteration_1{false}; + Series read("../samples/group_table.bp", Access::READ_LINEAR, jsonRead); // NOLINTNEXTLINE(performance-for-range-copy) for (auto iteration : read.readIterations()) { + ++counter; switch (iteration.iterationIndex) { case 0: + saw_iteration_0 = true; REQUIRE(iteration.meshes["E"].contains("x")); REQUIRE(iteration.meshes["E"].contains("y")); REQUIRE(iteration.meshes["E"].size() == 2); break; case 1: + saw_iteration_1 = true; if (canDeleteGroups) { REQUIRE(iteration.meshes["E"].contains("x")); @@ -5689,6 +5695,9 @@ void adios2_group_table( break; } } + REQUIRE(counter == 2); + REQUIRE(saw_iteration_0); + REQUIRE(saw_iteration_1); } TEST_CASE("adios2_group_table", "[serial]") From 5ebdab36060a576e6bd3bd96a146288147babe67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 19 Feb 2024 12:16:03 +0100 Subject: [PATCH 52/93] BUGFIX: modifiable attributes, maybe extract this to dev --- src/IO/ADIOS/ADIOS2IOHandler.cpp | 34 ++++++++++++++++---------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index 28524cfcb2..da9ef88844 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -1054,23 +1054,23 @@ void ADIOS2IOHandlerImpl::writeAttribute( return; } #if openPMD_HAS_ADIOS_2_9 - switch (useGroupTable()) - { - case UseGroupTable::No: - if (parameters.changesOverSteps == - Parameter::ChangesOverSteps::Yes) - { - // cannot do this - return; - } - - break; - case UseGroupTable::Yes: { - break; - } - default: - throw std::runtime_error("Unreachable!"); - } + // switch (useGroupTable()) + // { + // case UseGroupTable::No: + // if (parameters.changesOverSteps == + // Parameter::ChangesOverSteps::Yes) + // { + // // cannot do this + // return; + // } + + // break; + // case UseGroupTable::Yes: { + // break; + // } + // default: + // throw std::runtime_error("Unreachable!"); + // } #else if (parameters.changesOverSteps == Parameter::ChangesOverSteps::Yes) From 01d065da6f5f426eb2031d084c3968148b586119 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 19 Feb 2024 12:52:15 +0100 Subject: [PATCH 53/93] Ensure that iterations are never parsed twice --- src/Series.cpp | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/Series.cpp b/src/Series.cpp index 515386da6a..6719a86635 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -2031,7 +2031,6 @@ creating new iterations. [&series, &pOpen, this]( IterationIndex_t index, std::string const &path, - bool guardAgainstRereading, bool beginStep) -> std::optional { if (series.iterations.contains(index)) { @@ -2039,7 +2038,7 @@ creating new iterations. auto &i = series.iterations.at(index); // i.written(): the iteration has already been parsed // reparsing is not needed - if (guardAgainstRereading && i.written()) + if (i.written()) { return {}; } @@ -2104,9 +2103,7 @@ creating new iterations. } if (auto err = internal::withRWAccess( IOHandler()->m_seriesStatus, - [&]() { - return readSingleIteration(index, it, true, false); - }); + [&]() { return readSingleIteration(index, it, false); }); err) { std::cerr << "Cannot read iteration " << index @@ -2166,7 +2163,7 @@ creating new iterations. if (auto err = internal::withRWAccess( IOHandler()->m_seriesStatus, [&readSingleIteration, it]() { - return readSingleIteration(it, "", false, true); + return readSingleIteration(it, "", true); }); err) { From 29ff452182c034fa10d54548ad1ed2bf948ce0a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 19 Feb 2024 14:22:35 +0100 Subject: [PATCH 54/93] Move currently_available_iterations to During_t --- .../openPMD/snapshots/StatefulIterator.hpp | 14 ++++- src/snapshots/ContainerImpls.cpp | 22 +++---- src/snapshots/StatefulIterator.cpp | 60 +++++++++++++------ 3 files changed, 60 insertions(+), 36 deletions(-) diff --git a/include/openPMD/snapshots/StatefulIterator.hpp b/include/openPMD/snapshots/StatefulIterator.hpp index 3ea4a4ff69..6c5a55454c 100644 --- a/include/openPMD/snapshots/StatefulIterator.hpp +++ b/include/openPMD/snapshots/StatefulIterator.hpp @@ -52,6 +52,14 @@ namespace detail { size_t idx; std::optional iteration_idx; + std::vector + available_iterations_in_step; + + During_t( + size_t idx, + std::optional iteration_idx, + std::vector + available_iterations_in_step); }; struct After_t {}; @@ -65,7 +73,6 @@ namespace detail using Before_t = step_status_types::Before_t; constexpr static Before_t Before{}; using During_t = step_status_types::During_t; - constexpr static During_t During{}; using After_t = step_status_types::After_t; constexpr static After_t After{}; @@ -143,7 +150,6 @@ class StatefulIterator ~SharedData(); Series series; - std::vector iterationsInCurrentStep; CurrentStep currentStep = {CurrentStep::Before}; std::optional parsePreference; @@ -254,7 +260,9 @@ class StatefulIterator auto turn_into_end_iterator(TypeOfEndIterator) -> void; auto assert_end_iterator() const -> void; - auto resetCurrentIterationToBegin(size_t num_skipped_iterations) -> void; + auto resetCurrentIterationToBegin( + size_t num_skipped_iterations, + std::vector current_iterations) -> void; auto peekCurrentlyOpenIteration() const -> std::optional; auto peekCurrentlyOpenIteration() -> std::optional; diff --git a/src/snapshots/ContainerImpls.cpp b/src/snapshots/ContainerImpls.cpp index 159f407558..f4cc3e6622 100644 --- a/src/snapshots/ContainerImpls.cpp +++ b/src/snapshots/ContainerImpls.cpp @@ -168,28 +168,22 @@ auto StatefulSnapshotsContainer::operator[](key_type const &key) } } s.currentStep.map_during_t( - [&](auto &during) { + [&](detail::CurrentStep::During_t &during) { ++during.idx; during.iteration_idx = key; + during.available_iterations_in_step = {key}; }, - [&](detail::CurrentStep::AtTheEdge whereAmI) { - switch (whereAmI) + [&](detail::CurrentStep::AtTheEdge where_am_i) { + switch (where_am_i) { case detail::CurrentStep::AtTheEdge::Begin: - return detail::CurrentStep::During_t{0, key}; + return detail::CurrentStep::During_t{0, key, {key}}; case detail::CurrentStep::AtTheEdge::End: - throw error::WrongAPIUsage( - "Creating a new step on a Series that is closed."); + throw error::Internal( + "Trying to create a new output step, but the stream is " + "closed?"); } - throw std::runtime_error("Unreachable!"); }); - if (std::find( - s.iterationsInCurrentStep.begin(), - s.iterationsInCurrentStep.end(), - key) != s.iterationsInCurrentStep.end()) - { - s.iterationsInCurrentStep.push_back(key); - } auto &res = s.series.iterations[key]; if (res.getStepStatus() == StepStatus::NoStep) { diff --git a/src/snapshots/StatefulIterator.cpp b/src/snapshots/StatefulIterator.cpp index 4bfe3932e3..aa24ab6416 100644 --- a/src/snapshots/StatefulIterator.cpp +++ b/src/snapshots/StatefulIterator.cpp @@ -31,12 +31,24 @@ #include #include #include +#include namespace openPMD { namespace detail { + step_status_types::During_t::During_t( + size_t idx_in, + std::optional iteration_idx_in, + std::vector + available_iterations_in_step_in) + : idx(idx_in) + , iteration_idx(iteration_idx_in) + , available_iterations_in_step( + std::move(available_iterations_in_step_in)) + {} + template auto CurrentStep::get_variant() -> std::optional { @@ -251,7 +263,7 @@ auto StatefulIterator::turn_into_end_iterator(TypeOfEndIterator type) -> void during.iteration_idx = std::nullopt; }, [](auto const &) { - return CurrentStep::During_t{0, std::nullopt}; + return CurrentStep::During_t{0, std::nullopt, {}}; } ); @@ -260,34 +272,40 @@ auto StatefulIterator::turn_into_end_iterator(TypeOfEndIterator type) -> void } auto StatefulIterator::resetCurrentIterationToBegin( - size_t num_skipped_iterations) -> void + size_t num_skipped_iterations, std::vector indexes) + -> void { auto &data = get(); data.currentStep.map_during_t( [&](CurrentStep::During_t &during) { during.idx += num_skipped_iterations; - if (data.iterationsInCurrentStep.empty()) + during.available_iterations_in_step = std::move(indexes); + if (during.available_iterations_in_step.empty()) { during.iteration_idx = std::nullopt; } else { - during.iteration_idx = *data.iterationsInCurrentStep.begin(); + during.iteration_idx = + *during.available_iterations_in_step.begin(); } }, [&](CurrentStep::AtTheEdge whereAmI) -> std::optional { switch (whereAmI) { - case detail::CurrentStep::AtTheEdge::Begin: - if (data.iterationsInCurrentStep.empty()) + case detail::CurrentStep::AtTheEdge::Begin: { + if (indexes.empty()) { return std::nullopt; } + auto first_iteration = *indexes.begin(); // Begin iterating return detail::CurrentStep::During_t{ num_skipped_iterations, - *data.iterationsInCurrentStep.begin()}; + first_iteration, + std::move(indexes)}; + } case detail::CurrentStep::AtTheEdge::End: return std::nullopt; } @@ -446,13 +464,13 @@ std::optional StatefulIterator::nextIterationInStep() auto ¤t_iteration_idx = *currentIteration.iteration_idx; if (auto it = std::find( - data.iterationsInCurrentStep.begin(), - data.iterationsInCurrentStep.end(), + currentIteration.available_iterations_in_step.begin(), + currentIteration.available_iterations_in_step.end(), current_iteration_idx); - it != data.iterationsInCurrentStep.end()) + it != currentIteration.available_iterations_in_step.end()) { ++it; - if (it == data.iterationsInCurrentStep.end()) + if (it == currentIteration.available_iterations_in_step.end()) { return no_result(); } @@ -496,9 +514,9 @@ auto StatefulIterator::skipToIterationInStep(Iteration::IterationIndex_t idx) CurrentStep::During_t ¤tIteration = **maybeCurrentIteration; if (std::find( - data.iterationsInCurrentStep.begin(), - data.iterationsInCurrentStep.end(), - idx) == data.iterationsInCurrentStep.end()) + currentIteration.available_iterations_in_step.begin(), + currentIteration.available_iterations_in_step.end(), + idx) == currentIteration.available_iterations_in_step.end()) { return std::nullopt; } @@ -515,12 +533,13 @@ std::optional StatefulIterator::nextStep(size_t recursion_depth) { auto &data = get(); + std::vector current_iterations; // since we are in group-based iteration layout, it does not // matter which iteration we begin a step upon AdvanceStatus status{}; try { - std::tie(status, data.iterationsInCurrentStep) = Iteration::beginStep( + std::tie(status, current_iterations) = Iteration::beginStep( {}, data.series, /* reread = */ reread(data.parsePreference)); @@ -563,7 +582,8 @@ StatefulIterator::nextStep(size_t recursion_depth) { ++(*during)->idx; } - resetCurrentIterationToBegin(recursion_depth); + resetCurrentIterationToBegin( + recursion_depth, std::move(current_iterations)); } return {this}; } @@ -704,11 +724,12 @@ void StatefulIterator::initIteratorFilebased() this->turn_into_end_iterator(TypeOfEndIterator::NoMoreIterationsInStep); return; } - data.iterationsInCurrentStep.reserve(series.iterations.size()); + std::vector indexes; + indexes.reserve(series.iterations.size()); std::transform( series.iterations.begin(), series.iterations.end(), - std::back_inserter(data.iterationsInCurrentStep), + std::back_inserter(indexes), [](auto const &pair) { return pair.first; }); auto it = series.iterations.begin(); auto end = series.iterations.end(); @@ -728,7 +749,8 @@ void StatefulIterator::initIteratorFilebased() } if (it != end) { - data.currentStep = CurrentStep::During_t{0, it->first}; + data.currentStep = + CurrentStep::During_t{0, it->first, std::move(indexes)}; } else { From 64027e7d8527d45f2aeaf8431dd8cfc2a93b329d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 19 Feb 2024 15:29:03 +0100 Subject: [PATCH 55/93] Revert "For now, adapt the append_mode test" This reverts commit 19b68ee762876cba9c04cf596664030238c56343. --- test/ParallelIOTest.cpp | 8 ++++---- test/SerialIOTest.cpp | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/test/ParallelIOTest.cpp b/test/ParallelIOTest.cpp index 670f2e9926..3853282f2b 100644 --- a/test/ParallelIOTest.cpp +++ b/test/ParallelIOTest.cpp @@ -1689,14 +1689,14 @@ void append_mode( // time but as they go Series read(filename, Access::READ_LINEAR, MPI_COMM_WORLD); unsigned counter = 0; - uint64_t iterationOrder[] = {0, 1, 3, 2, 4, 3, 10, 7, 1, 11}; + uint64_t iterationOrder[] = {0, 1, 3, 2, 4, 10, 7, 11}; for (auto iteration : read.readIterations()) { REQUIRE(iteration.iterationIndex == iterationOrder[counter]); verifyIteration(iteration); ++counter; } - REQUIRE(counter == 10); + REQUIRE(counter == 8); // listSeries will not see any iterations since they have already // been read helper::listSeries(read); @@ -1802,7 +1802,7 @@ void append_mode( // in variable-based encodings, iterations are not parsed ahead // of time but as they go unsigned counter = 0; - uint64_t iterationOrder[] = {0, 1, 3, 2, 4, 3, 10, 7, 4, 5}; + uint64_t iterationOrder[] = {0, 1, 3, 2, 4, 10, 7, 5}; for (auto iteration : read.readIterations()) { REQUIRE( @@ -1810,7 +1810,7 @@ void append_mode( verifyIteration(iteration); ++counter; } - REQUIRE(counter == 10); + REQUIRE(counter == 8); } break; default: diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index f2489315b5..b48a29b8ed 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -7218,14 +7218,14 @@ void append_mode( // time but as they go Series read(filename, Access::READ_LINEAR); unsigned counter = 0; - uint64_t iterationOrder[] = {0, 1, 3, 2, 4, 3, 10, 7, 1, 11}; + uint64_t iterationOrder[] = {0, 1, 3, 2, 4, 10, 7, 11}; for (auto iteration : read.readIterations()) { REQUIRE(iteration.iterationIndex == iterationOrder[counter]); verifyIteration(iteration); ++counter; } - REQUIRE(counter == 10); + REQUIRE(counter == 8); // listSeries will not see any iterations since they have already // been read helper::listSeries(read); From f0bb16c2d8f05325619020a3b5609ed607998889 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 19 Feb 2024 15:35:08 +0100 Subject: [PATCH 56/93] Remember where we saw what iteration --- .../openPMD/snapshots/StatefulIterator.hpp | 4 ++ src/snapshots/ContainerImpls.cpp | 50 +++++++++++-------- src/snapshots/StatefulIterator.cpp | 33 ++++++++++++ 3 files changed, 66 insertions(+), 21 deletions(-) diff --git a/include/openPMD/snapshots/StatefulIterator.hpp b/include/openPMD/snapshots/StatefulIterator.hpp index 6c5a55454c..19915291cd 100644 --- a/include/openPMD/snapshots/StatefulIterator.hpp +++ b/include/openPMD/snapshots/StatefulIterator.hpp @@ -32,6 +32,7 @@ #include #include #include +#include #include #include @@ -149,9 +150,12 @@ class StatefulIterator ~SharedData(); + using step_index = size_t; + Series series; CurrentStep currentStep = {CurrentStep::Before}; std::optional parsePreference; + std::unordered_map seen_iterations; auto currentIteration() -> std::optional; auto currentIteration() const diff --git a/src/snapshots/ContainerImpls.cpp b/src/snapshots/ContainerImpls.cpp index f4cc3e6622..df4c2d6e28 100644 --- a/src/snapshots/ContainerImpls.cpp +++ b/src/snapshots/ContainerImpls.cpp @@ -141,8 +141,8 @@ auto StatefulSnapshotsContainer::at(key_type const &) -> mapped_type & auto StatefulSnapshotsContainer::operator[](key_type const &key) -> mapped_type & { - auto it = get(); - auto &shared = it->m_data; + auto base_iterator = get(); + auto &shared = base_iterator->m_data; if (!shared || !shared->has_value()) { throw error::WrongAPIUsage( @@ -154,7 +154,7 @@ auto StatefulSnapshotsContainer::operator[](key_type const &key) // @todo distinguish read_write if (access::write(access)) { - auto lastIteration = it->peekCurrentlyOpenIteration(); + auto lastIteration = base_iterator->peekCurrentlyOpenIteration(); if (lastIteration.has_value()) { auto lastIteration_v = lastIteration.value(); @@ -167,23 +167,30 @@ auto StatefulSnapshotsContainer::operator[](key_type const &key) lastIteration_v->second.close(); // continue below } } - s.currentStep.map_during_t( - [&](detail::CurrentStep::During_t &during) { - ++during.idx; - during.iteration_idx = key; - during.available_iterations_in_step = {key}; - }, - [&](detail::CurrentStep::AtTheEdge where_am_i) { - switch (where_am_i) - { - case detail::CurrentStep::AtTheEdge::Begin: - return detail::CurrentStep::During_t{0, key, {key}}; - case detail::CurrentStep::AtTheEdge::End: - throw error::Internal( - "Trying to create a new output step, but the stream is " - "closed?"); - } - }); + if (auto it = s.series.iterations.find(key); + it == s.series.iterations.end()) + { + s.currentStep.map_during_t( + [&](detail::CurrentStep::During_t &during) { + ++during.idx; + base_iterator->get().seen_iterations[key] = during.idx; + during.iteration_idx = key; + during.available_iterations_in_step = {key}; + }, + [&](detail::CurrentStep::AtTheEdge where_am_i) { + base_iterator->get().seen_iterations[key] = 0; + switch (where_am_i) + { + case detail::CurrentStep::AtTheEdge::Begin: + return detail::CurrentStep::During_t{0, key, {key}}; + case detail::CurrentStep::AtTheEdge::End: + throw error::Internal( + "Trying to create a new output step, but the " + "stream is " + "closed?"); + } + }); + } auto &res = s.series.iterations[key]; if (res.getStepStatus() == StepStatus::NoStep) { @@ -204,7 +211,8 @@ auto StatefulSnapshotsContainer::operator[](key_type const &key) } else if (access::read(access)) { - auto result = it->seek({StatefulIterator::Seek::Seek_Iteration_t{key}}); + auto result = base_iterator->seek( + {StatefulIterator::Seek::Seek_Iteration_t{key}}); return (*result)->second; } throw error::Internal("Control flow error: This should be unreachable."); diff --git a/src/snapshots/StatefulIterator.cpp b/src/snapshots/StatefulIterator.cpp index aa24ab6416..d996e94289 100644 --- a/src/snapshots/StatefulIterator.cpp +++ b/src/snapshots/StatefulIterator.cpp @@ -20,6 +20,7 @@ */ #include "openPMD/snapshots/StatefulIterator.hpp" +#include "openPMD/Datatype.hpp" #include "openPMD/Error.hpp" #include "openPMD/Iteration.hpp" @@ -271,6 +272,34 @@ auto StatefulIterator::turn_into_end_iterator(TypeOfEndIterator type) -> void } } +namespace +{ + auto restrict_to_unseen_iterations( + std::vector &indexes, + std::unordered_map + &seen_iterations, + size_t insert_into_step) -> void + { + for (auto vec_it = indexes.rbegin(); vec_it != indexes.rend();) + { + auto map_it = seen_iterations.find(*vec_it); + if (map_it == seen_iterations.end()) + { + seen_iterations.emplace_hint(map_it, *vec_it, insert_into_step); + ++vec_it; + } + else + { + ++vec_it; + // sic! base() refers to the next element... + // erase() only invalidates iterators to the right + // (this is why this iterates in reverse) + indexes.erase(vec_it.base()); + } + } + } +} // namespace + auto StatefulIterator::resetCurrentIterationToBegin( size_t num_skipped_iterations, std::vector indexes) -> void @@ -279,6 +308,8 @@ auto StatefulIterator::resetCurrentIterationToBegin( data.currentStep.map_during_t( [&](CurrentStep::During_t &during) { during.idx += num_skipped_iterations; + restrict_to_unseen_iterations( + indexes, data.seen_iterations, during.idx); during.available_iterations_in_step = std::move(indexes); if (during.available_iterations_in_step.empty()) { @@ -295,6 +326,8 @@ auto StatefulIterator::resetCurrentIterationToBegin( switch (whereAmI) { case detail::CurrentStep::AtTheEdge::Begin: { + restrict_to_unseen_iterations( + indexes, data.seen_iterations, num_skipped_iterations); if (indexes.empty()) { return std::nullopt; From 0d3b96d83c755e1b46d06ea0775909801816a88f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 19 Feb 2024 16:15:17 +0100 Subject: [PATCH 57/93] Bit of cleanup --- src/IO/ADIOS/ADIOS2IOHandler.cpp | 20 +------------------- src/Series.cpp | 15 ++++++++++----- src/snapshots/StatefulIterator.cpp | 14 ++++++++++++++ 3 files changed, 25 insertions(+), 24 deletions(-) diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index da9ef88844..94ec73b37b 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -1053,25 +1053,7 @@ void ADIOS2IOHandlerImpl::writeAttribute( { return; } -#if openPMD_HAS_ADIOS_2_9 - // switch (useGroupTable()) - // { - // case UseGroupTable::No: - // if (parameters.changesOverSteps == - // Parameter::ChangesOverSteps::Yes) - // { - // // cannot do this - // return; - // } - - // break; - // case UseGroupTable::Yes: { - // break; - // } - // default: - // throw std::runtime_error("Unreachable!"); - // } -#else +#if !openPMD_HAS_ADIOS_2_9 if (parameters.changesOverSteps == Parameter::ChangesOverSteps::Yes) { diff --git a/src/Series.cpp b/src/Series.cpp index 6719a86635..bb700c4a6f 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -1621,7 +1621,8 @@ void Series::readFileBased( * Return true if parsing was successful */ auto readIterationEagerly = - [](Iteration &iteration) -> std::optional { + [&read_only_this_single_iteration]( + Iteration &iteration) -> std::optional { try { iteration.runDeferredParseAccess(); @@ -1630,10 +1631,14 @@ void Series::readFileBased( { return err; } - Parameter fClose; - iteration.IOHandler()->enqueue(IOTask(&iteration, fClose)); - iteration.IOHandler()->flush(internal::defaultFlushParams); - iteration.get().m_closed = internal::CloseStatus::Closed; + // If one specific iteration is requested, keep it open afterward. + if (!read_only_this_single_iteration.has_value()) + { + Parameter fClose; + iteration.IOHandler()->enqueue(IOTask(&iteration, fClose)); + iteration.IOHandler()->flush(internal::defaultFlushParams); + iteration.get().m_closed = internal::CloseStatus::Closed; + } return {}; }; std::vector unparseableIterations; diff --git a/src/snapshots/StatefulIterator.cpp b/src/snapshots/StatefulIterator.cpp index d996e94289..cd80be91ca 100644 --- a/src/snapshots/StatefulIterator.cpp +++ b/src/snapshots/StatefulIterator.cpp @@ -117,6 +117,20 @@ namespace detail StatefulIterator::SharedData::~SharedData() { + // debugging block +#if 0 + std::map tmp; + std::copy( + seen_iterations.begin(), + seen_iterations.end(), + std::inserter(tmp, tmp.begin())); + std::cout << "SEEN ITERATIONS:\n"; + for(auto [it, step]: tmp) + { + std::cout << '\t' << it << ":\t" << step << '\n'; + } + std::cout << std::endl; +#endif auto IOHandler = series.IOHandler(); auto current_iteration = currentIteration(); if (IOHandler && current_iteration.has_value() && IOHandler && From e78d5be7ed6827b48a0d3805ee8976c9090ea7f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 19 Feb 2024 18:10:48 +0100 Subject: [PATCH 58/93] [wip] Groupbased writing: close and reopen --- include/openPMD/Series.hpp | 4 +- include/openPMD/Streaming.hpp | 1 + src/Iteration.cpp | 36 ++++++----- src/Series.cpp | 9 ++- src/snapshots/ContainerImpls.cpp | 2 +- src/snapshots/StatefulIterator.cpp | 10 +-- test/Files_SerialIO/SerialIOTests.hpp | 6 +- test/Files_SerialIO/close_iteration_test.cpp | 66 ++++++++++++++++++++ test/SerialIOTest.cpp | 53 +--------------- 9 files changed, 106 insertions(+), 81 deletions(-) create mode 100644 test/Files_SerialIO/close_iteration_test.cpp diff --git a/include/openPMD/Series.hpp b/include/openPMD/Series.hpp index e0ebb8f3b7..7f1678c97e 100644 --- a/include/openPMD/Series.hpp +++ b/include/openPMD/Series.hpp @@ -861,14 +861,12 @@ OPENPMD_private * based layout, it's the Series object. * @param it The iterator within Series::iterations pointing to that * iteration. - * @param iteration The actual Iteration object. * @return AdvanceStatus */ AdvanceStatus advance( AdvanceMode mode, internal::AttributableData &file, - iterations_iterator it, - Iteration &iteration); + iterations_iterator it); AdvanceStatus advance(AdvanceMode mode); diff --git a/include/openPMD/Streaming.hpp b/include/openPMD/Streaming.hpp index 94854662fa..8d1a283761 100644 --- a/include/openPMD/Streaming.hpp +++ b/include/openPMD/Streaming.hpp @@ -45,6 +45,7 @@ enum class AdvanceMode : unsigned char enum class StepStatus : unsigned char { DuringStep, /* step is currently active */ + OutOfStep, /* steps used, but currently no step active */ NoStep /* no step is currently active */ }; } // namespace openPMD diff --git a/src/Iteration.cpp b/src/Iteration.cpp index 05cedb4b53..1e2dc73c8b 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -25,6 +25,7 @@ #include "openPMD/IO/AbstractIOHandler.hpp" #include "openPMD/IO/IOTask.hpp" #include "openPMD/Series.hpp" +#include "openPMD/Streaming.hpp" #include "openPMD/auxiliary/DerefDynamicCast.hpp" #include "openPMD/auxiliary/Filesystem.hpp" #include "openPMD/auxiliary/StringManip.hpp" @@ -112,7 +113,7 @@ Iteration &Iteration::close(bool _flush) if (flag == StepStatus::DuringStep) { endStep(); - setStepStatus(StepStatus::NoStep); + setStepStatus(StepStatus::OutOfStep); } else { @@ -141,16 +142,24 @@ Iteration &Iteration::close(bool _flush) Iteration &Iteration::open() { auto &it = get(); - if (it.m_closed == CloseStatus::ParseAccessDeferred) + if (getStepStatus() == StepStatus::OutOfStep) { - it.m_closed = CloseStatus::Open; - runDeferredParseAccess(); + beginStep(/* reread = */ false); + setStepStatus(StepStatus::DuringStep); + } + else + { + if (it.m_closed == CloseStatus::ParseAccessDeferred) + { + it.m_closed = CloseStatus::Open; + runDeferredParseAccess(); + } + Series s = retrieveSeries(); + // figure out my iteration number + auto begin = s.indexOf(*this); + s.openIteration(begin->first, *this); + IOHandler()->flush(internal::defaultFlushParams); } - Series s = retrieveSeries(); - // figure out my iteration number - auto begin = s.indexOf(*this); - s.openIteration(begin->first, *this); - IOHandler()->flush(internal::defaultFlushParams); return *this; } @@ -717,14 +726,13 @@ auto Iteration::beginStep( AdvanceStatus status; if (thisObject.has_value()) { + thisObject->setStepStatus(StepStatus::DuringStep); status = series.advance( - AdvanceMode::BEGINSTEP, - *file, - series.indexOf(*thisObject), - *thisObject); + AdvanceMode::BEGINSTEP, *file, series.indexOf(*thisObject)); } else { + series.get().m_stepStatus = StepStatus::DuringStep; status = series.advance(AdvanceMode::BEGINSTEP); } @@ -813,7 +821,7 @@ void Iteration::endStep() break; } // @todo filebased check - series.advance(AdvanceMode::ENDSTEP, *file, series.indexOf(*this), *this); + series.advance(AdvanceMode::ENDSTEP, *file, series.indexOf(*this)); series.get().m_currentlyActiveIterations.clear(); } diff --git a/src/Series.cpp b/src/Series.cpp index bb700c4a6f..2513144b3b 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -2369,13 +2369,14 @@ Series::iterations_iterator Series::indexOf(Iteration const &iteration) AdvanceStatus Series::advance( AdvanceMode mode, internal::AttributableData &file, - iterations_iterator begin, - Iteration &iteration) + iterations_iterator begin) { internal::FlushParams const flushParams = {FlushLevel::UserFlush}; auto &series = get(); auto end = begin; ++end; + + auto &iteration = begin->second; /* * We call flush_impl() with flushIOHandler = false, meaning that tasks are * not yet propagated to the backend. @@ -2391,6 +2392,9 @@ AdvanceStatus Series::advance( itData.m_closed = internal::CloseStatus::Open; } + bool old_dirty = iteration.dirty(); + iteration.dirty() = true; // force flush() to open this + switch (mode) { case AdvanceMode::ENDSTEP: @@ -2407,6 +2411,7 @@ AdvanceStatus Series::advance( end, {FlushLevel::CreateOrOpenFiles}, /* flushIOHandler = */ false); + iteration.dirty() = old_dirty; break; } diff --git a/src/snapshots/ContainerImpls.cpp b/src/snapshots/ContainerImpls.cpp index df4c2d6e28..7e625bd1a4 100644 --- a/src/snapshots/ContainerImpls.cpp +++ b/src/snapshots/ContainerImpls.cpp @@ -192,7 +192,7 @@ auto StatefulSnapshotsContainer::operator[](key_type const &key) }); } auto &res = s.series.iterations[key]; - if (res.getStepStatus() == StepStatus::NoStep) + if (res.getStepStatus() != StepStatus::DuringStep) { try { diff --git a/src/snapshots/StatefulIterator.cpp b/src/snapshots/StatefulIterator.cpp index cd80be91ca..7083b38e3d 100644 --- a/src/snapshots/StatefulIterator.cpp +++ b/src/snapshots/StatefulIterator.cpp @@ -851,15 +851,7 @@ auto StatefulIterator::seek(Seek const &seek) -> StatefulIterator * res = loopBody(seek); } while (!res.has_value()); - auto resvalue = res.value(); - if (*resvalue != end()) - { - auto &series = data.series; - auto index = data.currentIteration(); - auto &iteration = series.iterations.at(*index.value()); - iteration.setStepStatus(StepStatus::DuringStep); - } - return resvalue; + return *res; } auto StatefulIterator::operator*() const -> value_type const & diff --git a/test/Files_SerialIO/SerialIOTests.hpp b/test/Files_SerialIO/SerialIOTests.hpp index 7d4d1d699d..541d4dcaf1 100644 --- a/test/Files_SerialIO/SerialIOTests.hpp +++ b/test/Files_SerialIO/SerialIOTests.hpp @@ -2,5 +2,9 @@ namespace filebased_write_test { -void close_and_reopen_iterations(std::string const &filename); +auto close_and_reopen_iterations(std::string const &filename) -> void; +} +namespace close_and_reopen_test +{ +auto close_and_reopen_test() -> void; } diff --git a/test/Files_SerialIO/close_iteration_test.cpp b/test/Files_SerialIO/close_iteration_test.cpp new file mode 100644 index 0000000000..96c9d34b3a --- /dev/null +++ b/test/Files_SerialIO/close_iteration_test.cpp @@ -0,0 +1,66 @@ +#include "SerialIOTests.hpp" + +namespace close_and_reopen_test +{ +using namespace openPMD; +#if openPMD_HAVE_ADIOS2 + +template +auto run_test(WriteIterations &&writeIterations) +{ + Series series( + "../samples/close_iteration_reopen_groupbased.bp", Access::CREATE); + { + auto it = writeIterations(series)[0]; + auto E_x = it.meshes["E"]["x"]; + E_x.resetDataset({Datatype::INT, {5}}); + std::vector data{0, 1, 2, 3, 4}; + E_x.storeChunk(data, {0}, {5}); + it.close(); + + it.open(); + auto B_y = it.meshes["B"]["y"]; + B_y.resetDataset({Datatype::INT, {5}}); + B_y.storeChunk(data, {0}, {5}); + it.close(); + } + + { + auto it = writeIterations(series)[1]; + auto E_x = it.meshes["E"]["x"]; + E_x.resetDataset({Datatype::INT, {5}}); + std::vector data{0, 1, 2, 3, 4}; + E_x.storeChunk(data, {0}, {5}); + it.close(); + + it.open(); + auto e_position_x = it.particles["e"]["position"]["x"]; + e_position_x.resetDataset({Datatype::INT, {5}}); + e_position_x.storeChunk(data, {0}, {5}); + it.close(); + } + { + auto it = writeIterations(series)[2]; + auto E_x = it.meshes["E"]["x"]; + E_x.resetDataset({Datatype::INT, {5}}); + std::vector data{0, 1, 2, 3, 4}; + E_x.storeChunk(data, {0}, {5}); + it.close(); + + it.open(); + it.setTimeUnitSI(2.0); + it.close(); + } +} + +auto close_and_reopen_test() -> void +{ + run_test([](Series &s) { return s.iterations; }); + run_test([](Series &s) { return s.writeIterations(); }); + run_test([](Series &s) { return s.snapshots(); }); +} +#else +auto close_and_reopen_test() -> void +{} +#endif +} // namespace close_and_reopen_test diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index b48a29b8ed..a5909a09fd 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -458,10 +458,6 @@ void close_iteration_test(std::string const &file_ending) E_x.resetDataset({Datatype::INT, {2, 2}}); E_x.storeChunk(data, {0, 0}, {2, 2}); it1.close(/* flush = */ true); - - // illegally access iteration after closing - E_x.storeChunk(data, {0, 0}, {2, 2}); - // REQUIRE_THROWS(write.flush()); } { @@ -474,8 +470,6 @@ void close_iteration_test(std::string const &file_ending) { REQUIRE(data[i] == chunk.get()[i]); } - auto read_again = E_x_read.loadChunk({0, 0}, {2, 2}); - // REQUIRE_THROWS(read.flush()); } { @@ -744,52 +738,9 @@ TEST_CASE("close_and_copy_attributable_test", "[serial]") } #if openPMD_HAVE_ADIOS2 -TEST_CASE("close_iteration_throws_test", "[serial]") +TEST_CASE("close_and_reopen_test", "[serial]") { - /* - * Iterations should not be accessed any more after closing. - * Test that the openPMD API detects that case and throws. - */ - { - Series series("../samples/close_iteration_throws_1.bp", Access::CREATE); - auto it0 = series.iterations[0]; - auto E_x = it0.meshes["E"]["x"]; - E_x.resetDataset({Datatype::INT, {5}}); - std::vector data{0, 1, 2, 3, 4}; - E_x.storeChunk(data, {0}, {5}); - it0.close(); - - auto B_y = it0.meshes["B"]["y"]; - B_y.resetDataset({Datatype::INT, {5}}); - B_y.storeChunk(data, {0}, {5}); - // REQUIRE_THROWS(series.flush()); - } - { - Series series("../samples/close_iteration_throws_2.bp", Access::CREATE); - auto it0 = series.iterations[0]; - auto E_x = it0.meshes["E"]["x"]; - E_x.resetDataset({Datatype::INT, {5}}); - std::vector data{0, 1, 2, 3, 4}; - E_x.storeChunk(data, {0}, {5}); - it0.close(); - - auto e_position_x = it0.particles["e"]["position"]["x"]; - e_position_x.resetDataset({Datatype::INT, {5}}); - e_position_x.storeChunk(data, {0}, {5}); - // REQUIRE_THROWS(series.flush()); - } - { - Series series("../samples/close_iteration_throws_3.bp", Access::CREATE); - auto it0 = series.iterations[0]; - auto E_x = it0.meshes["E"]["x"]; - E_x.resetDataset({Datatype::INT, {5}}); - std::vector data{0, 1, 2, 3, 4}; - E_x.storeChunk(data, {0}, {5}); - it0.close(); - - it0.setTimeUnitSI(2.0); - // REQUIRE_THROWS(series.flush()); - } + close_and_reopen_test::close_and_reopen_test(); } #endif From 5bbb79d183157f868fe9843d2c9ced82b14ec8ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 20 Feb 2024 17:31:34 +0100 Subject: [PATCH 59/93] Further test and implement reopening of Iterations --- include/openPMD/Iteration.hpp | 15 +---- include/openPMD/Series.hpp | 9 ++- src/IO/ADIOS/ADIOS2IOHandler.cpp | 12 ++-- src/Iteration.cpp | 48 ++++++++----- src/Series.cpp | 45 ++++++++----- test/Files_SerialIO/close_iteration_test.cpp | 71 ++++++++++++++++++-- test/Files_SerialIO/filebased_write_test.cpp | 5 +- 7 files changed, 146 insertions(+), 59 deletions(-) diff --git a/include/openPMD/Iteration.hpp b/include/openPMD/Iteration.hpp index b3fb6e9334..24078b5e86 100644 --- a/include/openPMD/Iteration.hpp +++ b/include/openPMD/Iteration.hpp @@ -69,11 +69,6 @@ namespace internal * (Group- and variable-based parsing shares the same code logic.) */ bool fileBased = false; - /** - * If fileBased == true, the file name (without file path) of the file - * containing this iteration. - */ - std::string filename; bool beginStep = false; }; @@ -90,6 +85,7 @@ namespace internal * overwritten. */ CloseStatus m_closed = CloseStatus::Open; + bool allow_reopening_implicitly = false; /** * Whether a step is currently active for this iteration. @@ -105,14 +101,6 @@ namespace internal * Otherwise empty. */ std::optional m_deferredParseAccess{}; - - /** - * Upon reading a file, set this field to the used file name. - * In inconsistent iteration paddings, we must remember the name of the - * file since it cannot be reconstructed from the filename pattern - * alone. - */ - std::optional m_overrideFilebasedFilename{}; }; } // namespace internal /** @brief Logical compilation of data from one snapshot (e.g. a single @@ -298,6 +286,7 @@ class Iteration : public Attributable */ void reread(std::string const &path); void readFileBased( + IterationIndex_t, std::string const &filePath, std::string const &groupPath, bool beginStep); diff --git a/include/openPMD/Series.hpp b/include/openPMD/Series.hpp index 7f1678c97e..d3965ddd4d 100644 --- a/include/openPMD/Series.hpp +++ b/include/openPMD/Series.hpp @@ -49,7 +49,11 @@ #include #include #include +<<<<<<< HEAD #include +======= +#include +>>>>>>> 881f29767 (Further test and implement reopening of Iterations) #include // expose private and protected members for invasive testing @@ -116,6 +120,7 @@ namespace internal * snapshot attribute. */ std::set m_currentlyActiveIterations; + std::unordered_map m_iterationFilenames; /** * Needed if reading a single iteration of a file-based series. * Users may specify the concrete filename of one iteration instead of @@ -834,14 +839,14 @@ OPENPMD_private * parse state. */ IterationOpened - openIterationIfDirty(IterationIndex_t index, Iteration iteration); + openIterationIfDirty(IterationIndex_t index, Iteration &iteration); /* * Open an iteration. Ensures that the iteration's m_closed status * is set properly and that any files pertaining to the iteration * is opened. * Does not create files when called in CREATE mode. */ - void openIteration(IterationIndex_t index, Iteration iteration); + void openIteration(IterationIndex_t index, Iteration &iteration); /** * Find the given iteration in Series::iterations and return an iterator diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index 94ec73b37b..e6aba83a98 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -599,7 +599,8 @@ to drastic performance issues, no matter if I/O steps are used or not. * If using I/O steps: Each step will add new variables and attributes instead of reusing those from earlier steps. ADIOS2 is not optimized for this and especially the BP5 engine will show a quadratic increase in metadata size - as the number of steps increase. + as the number of steps increase. Consider moving to the BP4 engine if you want + to keep using group-based encoding. We advise you to pick either file-based encoding or variable-based encoding (variable-based encoding is not yet feature-complete in the openPMD-api). For more details, refer to @@ -658,15 +659,18 @@ void ADIOS2IOHandlerImpl::createFile( // For a peaceful phase-out of group-based encoding in ADIOS2, // print this warning only in the new layout (with group table) if (m_useGroupTable.value_or(UseGroupTable::No) == - UseGroupTable::Yes) + UseGroupTable::Yes && + (m_engineType == "bp5" || + (openPMD_HAS_ADIOS_2_9 && + (m_engineType == "file" || m_engineType == "filestream" || + m_engineType == "bp")))) { std::cerr << warningADIOS2NoGroupbasedEncoding << std::endl; printedWarningsAlready.noGroupBased = true; } fileData.m_IO.DefineAttribute( adios_defaults::str_groupBasedWarning, - std::string("Consider using file-based or variable-based " - "encoding instead in ADIOS2.")); + std::string(warningADIOS2NoGroupbasedEncoding)); } } } diff --git a/src/Iteration.cpp b/src/Iteration.cpp index 1e2dc73c8b..74c4e9314b 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -141,25 +141,23 @@ Iteration &Iteration::close(bool _flush) Iteration &Iteration::open() { + Series s = retrieveSeries(); auto &it = get(); + // figure out my iteration number + auto begin = s.indexOf(*this); + // ensure that files are accessed + s.openIteration(begin->first, *this); + if (it.m_closed == CloseStatus::ParseAccessDeferred) + { + it.m_closed = CloseStatus::Open; + runDeferredParseAccess(); + } if (getStepStatus() == StepStatus::OutOfStep) { beginStep(/* reread = */ false); setStepStatus(StepStatus::DuringStep); } - else - { - if (it.m_closed == CloseStatus::ParseAccessDeferred) - { - it.m_closed = CloseStatus::Open; - runDeferredParseAccess(); - } - Series s = retrieveSeries(); - // figure out my iteration number - auto begin = s.indexOf(*this); - s.openIteration(begin->first, *this); - IOHandler()->flush(internal::defaultFlushParams); - } + IOHandler()->flush(internal::defaultFlushParams); return *this; } @@ -326,6 +324,7 @@ void Iteration::flush(internal::FlushParams const &flushParams) m.second.flush(m.first, flushParams); for (auto &species : particles) species.second.flush(species.first, flushParams); + dirty() = false; } else { @@ -393,7 +392,10 @@ void Iteration::reread(std::string const &path) } void Iteration::readFileBased( - std::string const &filePath, std::string const &groupPath, bool doBeginStep) + IterationIndex_t idx, + std::string const &filePath, + std::string const &groupPath, + bool doBeginStep) { if (doBeginStep) { @@ -405,7 +407,15 @@ void Iteration::readFileBased( auto series = retrieveSeries(); series.readOneIterationFileBased(filePath); - get().m_overrideFilebasedFilename = filePath; + + auto &series_data = series.get(); + if (series_data.m_iterationFilenames.find(idx) == + series_data.m_iterationFilenames.end()) + { + throw error::Internal( + "[Iteration::readFileBased] Own filename should be placed into " + "buffer for later retrieval."); + } read_impl(groupPath); } @@ -883,8 +893,14 @@ void Iteration::runDeferredParseAccess() { if (deferred.fileBased) { + auto const &filename = + retrieveSeries().get().m_iterationFilenames.at( + deferred.iteration); readFileBased( - deferred.filename, deferred.path, deferred.beginStep); + deferred.iteration, + filename, + deferred.path, + deferred.beginStep); } else { diff --git a/src/Series.cpp b/src/Series.cpp index 2513144b3b..f29c417756 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -1594,11 +1594,9 @@ void Series::readFileBased( return; } Iteration &i = series.iterations[index]; - i.deferParseAccess( - {std::to_string(index), - index, - true, - cleanFilename(filename, series.m_filenameExtension).body}); + series.m_iterationFilenames[index] = + cleanFilename(filename, series.m_filenameExtension).body; + i.deferParseAccess({std::to_string(index), index, true}); }); if (series.iterations.empty()) @@ -1637,7 +1635,9 @@ void Series::readFileBased( Parameter fClose; iteration.IOHandler()->enqueue(IOTask(&iteration, fClose)); iteration.IOHandler()->flush(internal::defaultFlushParams); - iteration.get().m_closed = internal::CloseStatus::Closed; + auto &it_data = iteration.get(); + it_data.m_closed = internal::CloseStatus::Closed; + it_data.allow_reopening_implicitly = true; } return {}; }; @@ -2061,7 +2061,7 @@ creating new iterations. { // parse for the first time, resp. delay the parsing process Iteration &i = series.iterations[index]; - i.deferParseAccess({path, index, false, "", beginStep}); + i.deferParseAccess({path, index, false, beginStep}); if (!series.m_parseLazily) { try @@ -2330,11 +2330,10 @@ std::string Series::iterationFilename(IterationIndex_t i) { return series.m_overrideFilebasedFilename.value(); } - else if (auto iteration = iterations.find(i); // - iteration != iterations.end() && - iteration->second.get().m_overrideFilebasedFilename.has_value()) + else if (auto iteration = series.m_iterationFilenames.find(i); // + iteration != series.m_iterationFilenames.end()) { - return iteration->second.get().m_overrideFilebasedFilename.value(); + return iteration->second; } else { @@ -2598,19 +2597,20 @@ void Series::flushStep(bool doFlush) series.m_wroteAtLeastOneIOStep = true; } -auto Series::openIterationIfDirty(IterationIndex_t index, Iteration iteration) +auto Series::openIterationIfDirty(IterationIndex_t index, Iteration &iteration) -> IterationOpened { + auto &data = iteration.get(); /* * Check side conditions on accessing iterations, and if they are fulfilled, * forward function params to openIteration(). */ - if (iteration.get().m_closed == internal::CloseStatus::ParseAccessDeferred) + if (data.m_closed == internal::CloseStatus::ParseAccessDeferred) { return IterationOpened::RemainsClosed; } bool const dirtyRecursive = iteration.dirtyRecursive(); - if (iteration.get().m_closed == internal::CloseStatus::Closed) + if (data.m_closed == internal::CloseStatus::Closed) { // file corresponding with the iteration has previously been // closed and fully flushed @@ -2621,10 +2621,21 @@ auto Series::openIterationIfDirty(IterationIndex_t index, Iteration iteration) "[Series] Closed iteration has not been written. This " "is an internal error."); } - if (!dirtyRecursive) + else if (!dirtyRecursive) { return IterationOpened::RemainsClosed; } + else if (!data.allow_reopening_implicitly) + { + throw error::WrongAPIUsage( + "[Series] Closed iteration (idx=" + std::to_string(index) + + ") must be open()ed explicitly before interacting with it " + "again."); + } + else + { + data.allow_reopening_implicitly = false; // only allow this once + } } switch (iterationEncoding()) @@ -2657,17 +2668,17 @@ auto Series::openIterationIfDirty(IterationIndex_t index, Iteration iteration) return IterationOpened::RemainsClosed; } -void Series::openIteration(IterationIndex_t index, Iteration iteration) +void Series::openIteration(IterationIndex_t index, Iteration &iteration) { auto oldStatus = iteration.get().m_closed; switch (oldStatus) { using CL = internal::CloseStatus; case CL::Closed: - case CL::ParseAccessDeferred: case CL::Open: iteration.get().m_closed = CL::Open; break; + case CL::ParseAccessDeferred: case CL::ClosedInFrontend: // just keep it like it is break; diff --git a/test/Files_SerialIO/close_iteration_test.cpp b/test/Files_SerialIO/close_iteration_test.cpp index 96c9d34b3a..2a36728b27 100644 --- a/test/Files_SerialIO/close_iteration_test.cpp +++ b/test/Files_SerialIO/close_iteration_test.cpp @@ -1,15 +1,24 @@ #include "SerialIOTests.hpp" +#include "openPMD/IO/Access.hpp" + +#include namespace close_and_reopen_test { using namespace openPMD; #if openPMD_HAVE_ADIOS2 +inline void breakpoint() +{} + template -auto run_test(WriteIterations &&writeIterations) +auto run_test(WriteIterations &&writeIterations, std::vector readModes) { Series series( - "../samples/close_iteration_reopen_groupbased.bp", Access::CREATE); + "../samples/close_iteration_reopen_groupbased.bp4", + Access::CREATE, + R"(adios2.use_group_table = true + adios2.modifiable_attributes = true)"); { auto it = writeIterations(series)[0]; auto E_x = it.meshes["E"]["x"]; @@ -51,13 +60,65 @@ auto run_test(WriteIterations &&writeIterations) it.setTimeUnitSI(2.0); it.close(); } + series.close(); + + for (auto mode : readModes) + { + Series read("../samples/close_iteration_reopen_groupbased.bp4", mode); + { + auto it = read.snapshots()[0]; + std::vector data(5); + it.meshes["E"]["x"].loadChunkRaw(data.data(), {0}, {5}); + it.close(); + REQUIRE((data == std::vector{0, 1, 2, 3, 4})); + } + REQUIRE(read.iterations.size() == 3); + { + auto it = read.snapshots()[1]; + std::vector data(5); + it.meshes["E"]["x"].loadChunkRaw(data.data(), {0}, {5}); + it.close(); + REQUIRE((data == std::vector{0, 1, 2, 3, 4})); + } + { + auto it = read.snapshots()[2]; + std::vector data(5); + it.meshes["E"]["x"].loadChunkRaw(data.data(), {0}, {5}); + it.close(); + REQUIRE((data == std::vector{0, 1, 2, 3, 4})); + // no guarantee which attribute version we get + REQUIRE((it.timeUnitSI() == 2.0 || it.timeUnitSI() == 1.0)); + } + { + auto it = read.snapshots()[0].open(); + std::vector data(5); + it.meshes["B"]["y"].loadChunkRaw(data.data(), {0}, {5}); + it.close(); + REQUIRE((data == std::vector{0, 1, 2, 3, 4})); + } + { + auto it = read.snapshots()[1].open(); + std::vector data(5); + it.particles["e"]["position"]["x"].loadChunkRaw( + data.data(), {0}, {5}); + it.close(); + REQUIRE((data == std::vector{0, 1, 2, 3, 4})); + } + } } auto close_and_reopen_test() -> void { - run_test([](Series &s) { return s.iterations; }); - run_test([](Series &s) { return s.writeIterations(); }); - run_test([](Series &s) { return s.snapshots(); }); + run_test( + [](Series &s) { return s.iterations; }, + {Access::READ_ONLY, Access::READ_LINEAR}); + // since these write data in a way that distributes one iteration's data + // over multiple steps, only random access read mode makes sense + run_test( + [](Series &s) { return s.writeIterations(); }, + {Access::READ_RANDOM_ACCESS}); + run_test( + [](Series &s) { return s.snapshots(); }, {Access::READ_RANDOM_ACCESS}); } #else auto close_and_reopen_test() -> void diff --git a/test/Files_SerialIO/filebased_write_test.cpp b/test/Files_SerialIO/filebased_write_test.cpp index 3c2738a9af..488abd7984 100644 --- a/test/Files_SerialIO/filebased_write_test.cpp +++ b/test/Files_SerialIO/filebased_write_test.cpp @@ -1,4 +1,5 @@ #include "SerialIOTests.hpp" +#include "openPMD/IO/Access.hpp" namespace filebased_write_test { @@ -55,7 +56,7 @@ void close_and_reopen_iterations( iteration.close(); } std::cout << "Trying to read iteration 3 out of line" << std::endl; - if (need_to_explitly_open_iterations) + if (need_to_explitly_open_iterations || access == Access::READ_ONLY) { list.snapshots()[3].open(); } @@ -65,7 +66,7 @@ void close_and_reopen_iterations( for (auto &[idx, iteration] : list.snapshots()) { std::cout << "Seeing iteration " << idx << std::endl; - if (need_to_explitly_open_iterations) + if (need_to_explitly_open_iterations || access == Access::READ_ONLY) { iteration.open(); } From b785148574a0a1bbcfb0331895f9cc6e0115c939 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 21 Feb 2024 11:23:37 +0100 Subject: [PATCH 60/93] Unused variable --- src/snapshots/StatefulIterator.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/snapshots/StatefulIterator.cpp b/src/snapshots/StatefulIterator.cpp index 7083b38e3d..ee0fdb1bfc 100644 --- a/src/snapshots/StatefulIterator.cpp +++ b/src/snapshots/StatefulIterator.cpp @@ -837,7 +837,6 @@ StatefulIterator &StatefulIterator::operator++() auto StatefulIterator::seek(Seek const &seek) -> StatefulIterator * { - auto &data = get(); std::optional res; /* * loopBody() might return an empty option to indicate a skipped iteration. From 95798b0b7f581766cc5915ab77a9a98c034b91d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 21 Feb 2024 11:49:17 +0100 Subject: [PATCH 61/93] some fixes to groupbased reopen test --- test/Files_SerialIO/close_iteration_test.cpp | 22 ++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/test/Files_SerialIO/close_iteration_test.cpp b/test/Files_SerialIO/close_iteration_test.cpp index 2a36728b27..2e05d18e81 100644 --- a/test/Files_SerialIO/close_iteration_test.cpp +++ b/test/Files_SerialIO/close_iteration_test.cpp @@ -12,10 +12,15 @@ inline void breakpoint() {} template -auto run_test(WriteIterations &&writeIterations, std::vector readModes) +auto run_test( + WriteIterations &&writeIterations, + std::string const &ext, + std::vector const &readModes) { + std::string filename = + "../samples/close_iteration_reopen_groupbased." + ext; Series series( - "../samples/close_iteration_reopen_groupbased.bp4", + filename, Access::CREATE, R"(adios2.use_group_table = true adios2.modifiable_attributes = true)"); @@ -64,7 +69,7 @@ auto run_test(WriteIterations &&writeIterations, std::vector readModes) for (auto mode : readModes) { - Series read("../samples/close_iteration_reopen_groupbased.bp4", mode); + Series read(filename, mode); { auto it = read.snapshots()[0]; std::vector data(5); @@ -111,14 +116,23 @@ auto close_and_reopen_test() -> void { run_test( [](Series &s) { return s.iterations; }, + "bp4", {Access::READ_ONLY, Access::READ_LINEAR}); // since these write data in a way that distributes one iteration's data // over multiple steps, only random access read mode makes sense run_test( [](Series &s) { return s.writeIterations(); }, + "bp4", {Access::READ_RANDOM_ACCESS}); run_test( - [](Series &s) { return s.snapshots(); }, {Access::READ_RANDOM_ACCESS}); + [](Series &s) { return s.snapshots(); }, + "bp4", + {Access::READ_RANDOM_ACCESS}); + // that doesnt matter for json tho + run_test( + [](Series &s) { return s.snapshots(); }, + "json", + {Access::READ_RANDOM_ACCESS, Access::READ_LINEAR}); } #else auto close_and_reopen_test() -> void From 00f149fb1958f5f585d5ac3b466862d2e8ea3c88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 21 Feb 2024 14:32:36 +0100 Subject: [PATCH 62/93] Filebased reopen in ADIOS2 (no READ_WRITE support yet) --- include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp | 6 + include/openPMD/IO/ADIOS/ADIOS2File.hpp | 5 +- include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp | 6 +- include/openPMD/IO/AbstractIOHandler.hpp | 4 + src/IO/ADIOS/ADIOS2File.cpp | 8 +- src/IO/ADIOS/ADIOS2IOHandler.cpp | 49 +++++++- src/Series.cpp | 16 ++- test/Files_SerialIO/close_iteration_test.cpp | 121 ++++++++++++++++++- 8 files changed, 195 insertions(+), 20 deletions(-) diff --git a/include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp b/include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp index 9cb275d339..7648cfeb20 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp @@ -66,6 +66,12 @@ namespace adios_defs Yes, No }; + + enum class OpenFileAs + { + CreateFile, + OpenFile + }; } // namespace adios_defs /* diff --git a/include/openPMD/IO/ADIOS/ADIOS2File.hpp b/include/openPMD/IO/ADIOS/ADIOS2File.hpp index 0bcdaa6131..5d519a85ad 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2File.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2File.hpp @@ -229,7 +229,10 @@ class ADIOS2File using AttributeMap_t = std::map; - ADIOS2File(ADIOS2IOHandlerImpl &impl, InvalidatableFile file); + ADIOS2File( + ADIOS2IOHandlerImpl &impl, + InvalidatableFile file, + adios_defs::OpenFileAs); ~ADIOS2File(); diff --git a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp index db3162a2da..f5a2fc12bd 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp @@ -212,7 +212,8 @@ class ADIOS2IOHandlerImpl * @brief The ADIOS2 access type to chose for Engines opened * within this instance. */ - adios2::Mode adios2AccessMode(std::string const &fullPath); + adios2::Mode + adios2AccessMode(std::string const &fullPath, adios_defs::OpenFileAs); FlushTarget m_flushTarget = FlushTarget::Disk; @@ -403,9 +404,10 @@ class ADIOS2IOHandlerImpl */ GroupOrDataset groupOrDataset(Writable *); - enum class IfFileNotOpen : bool + enum class IfFileNotOpen : char { OpenImplicitly, + CreateImplicitly, ThrowError }; diff --git a/include/openPMD/IO/AbstractIOHandler.hpp b/include/openPMD/IO/AbstractIOHandler.hpp index 1288a87b21..43a0acf2a5 100644 --- a/include/openPMD/IO/AbstractIOHandler.hpp +++ b/include/openPMD/IO/AbstractIOHandler.hpp @@ -218,6 +218,10 @@ class AbstractIOHandler break; } } + else + { + m_backendAccess = m_frontendAccess; + } m_encoding = encoding; } diff --git a/src/IO/ADIOS/ADIOS2File.cpp b/src/IO/ADIOS/ADIOS2File.cpp index ee0c1a9062..82dd33f70b 100644 --- a/src/IO/ADIOS/ADIOS2File.cpp +++ b/src/IO/ADIOS/ADIOS2File.cpp @@ -23,6 +23,7 @@ #include "openPMD/Error.hpp" #include "openPMD/IO/ADIOS/ADIOS2IOHandler.hpp" #include "openPMD/IO/AbstractIOHandler.hpp" +#include "openPMD/IterationEncoding.hpp" #include "openPMD/auxiliary/Environment.hpp" #include "openPMD/auxiliary/StringManip.hpp" @@ -159,7 +160,10 @@ void BufferedUniquePtrPut::run(ADIOS2File &ba) switchAdios2VariableType(dtype, *this, ba); } -ADIOS2File::ADIOS2File(ADIOS2IOHandlerImpl &impl, InvalidatableFile file) +ADIOS2File::ADIOS2File( + ADIOS2IOHandlerImpl &impl, + InvalidatableFile file, + adios_defs::OpenFileAs openFileAs) : m_file(impl.fullPath(std::move(file))) , m_ADIOS(impl.m_ADIOS) , m_impl(&impl) @@ -167,7 +171,7 @@ ADIOS2File::ADIOS2File(ADIOS2IOHandlerImpl &impl, InvalidatableFile file) // Declaring these members in the constructor body to avoid // initialization order hazards. Need the IO_ prefix since in some // situation there seems to be trouble with number-only IO names - m_mode = impl.adios2AccessMode(m_file); + m_mode = impl.adios2AccessMode(m_file, openFileAs); create_IO(); if (!m_IO) { diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index e6aba83a98..03a7369bb8 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -37,6 +37,7 @@ #include #include // std::tolower +#include #include #include #include @@ -650,7 +651,7 @@ void ADIOS2IOHandlerImpl::createFile( // enforce opening the file // lazy opening is deathly in parallel situations auto &fileData = - getFileData(shared_name, IfFileNotOpen::OpenImplicitly); + getFileData(shared_name, IfFileNotOpen::CreateImplicitly); if (!printedWarningsAlready.noGroupBased && m_writeAttributesFromThisRank && @@ -1509,7 +1510,8 @@ void ADIOS2IOHandlerImpl::touch( m_dirty.emplace(std::move(file)); } -adios2::Mode ADIOS2IOHandlerImpl::adios2AccessMode(std::string const &fullPath) +adios2::Mode ADIOS2IOHandlerImpl::adios2AccessMode( + std::string const &fullPath, adios_defs::OpenFileAs openFileAs) { if (m_config.json().contains("engine") && m_config["engine"].json().contains("access_mode")) @@ -1554,10 +1556,27 @@ adios2::Mode ADIOS2IOHandlerImpl::adios2AccessMode(std::string const &fullPath) switch (m_handler->m_backendAccess) { case Access::CREATE: - return adios2::Mode::Write; + switch (openFileAs) + { + + case adios_defs::OpenFileAs::CreateFile: + return adios2::Mode::Write; + case adios_defs::OpenFileAs::OpenFile: + return adios2::Mode::Append; + } + break; #if openPMD_HAS_ADIOS_2_8 case Access::READ_LINEAR: - return adios2::Mode::Read; + switch (m_handler->m_encoding) + { + + case IterationEncoding::fileBased: + return adios2::Mode::ReadRandomAccess; + case IterationEncoding::groupBased: + case IterationEncoding::variableBased: + return adios2::Mode::Read; + } + break; case Access::READ_ONLY: return adios2::Mode::ReadRandomAccess; #else @@ -1570,7 +1589,16 @@ adios2::Mode ADIOS2IOHandlerImpl::adios2AccessMode(std::string const &fullPath) auxiliary::file_exists(fullPath)) { #if openPMD_HAS_ADIOS_2_8 - return adios2::Mode::ReadRandomAccess; + switch (m_handler->m_encoding) + { + + case IterationEncoding::fileBased: + return adios2::Mode::ReadRandomAccess; + case IterationEncoding::groupBased: + case IterationEncoding::variableBased: + return adios2::Mode::Read; + } + break; #else return adios2::Mode::Read; #endif @@ -1680,7 +1708,16 @@ detail::ADIOS2File &ADIOS2IOHandlerImpl::getFileData( case IfFileNotOpen::OpenImplicitly: { auto res = m_fileData.emplace( - file, std::make_unique(*this, file)); + file, + std::make_unique( + *this, file, adios_defs::OpenFileAs::OpenFile)); + return *res.first->second; + } + case IfFileNotOpen::CreateImplicitly: { + auto res = m_fileData.emplace( + file, + std::make_unique( + *this, file, adios_defs::OpenFileAs::CreateFile)); return *res.first->second; } case IfFileNotOpen::ThrowError: diff --git a/src/Series.cpp b/src/Series.cpp index f29c417756..99564e7155 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -1775,6 +1775,12 @@ void Series::readOneIterationFileBased(std::string const &filePath) { auto &series = get(); + IOHandler()->m_encoding = IterationEncoding::fileBased; + // In this case, READ_LINEAR is implemented exclusively in the frontend + if (IOHandler()->m_backendAccess == Access::READ_LINEAR) + { + IOHandler()->m_backendAccess = Access::READ_ONLY; + } Parameter fOpen; Parameter aRead; @@ -1789,14 +1795,15 @@ void Series::readOneIterationFileBased(std::string const &filePath) aRead.name = "iterationEncoding"; IOHandler()->enqueue(IOTask(this, aRead)); IOHandler()->flush(internal::defaultFlushParams); + IterationEncoding encoding_out; if (*aRead.dtype == DT::STRING) { std::string encoding = Attribute(*aRead.resource).get(); if (encoding == "fileBased") - series.m_iterationEncoding = IterationEncoding::fileBased; + encoding_out = IterationEncoding::fileBased; else if (encoding == "groupBased") { - series.m_iterationEncoding = IterationEncoding::fileBased; + encoding_out = IterationEncoding::fileBased; std::cerr << "Series constructor called with iteration regex '%T' " "suggests loading a time series with fileBased iteration " @@ -1826,7 +1833,10 @@ void Series::readOneIterationFileBased(std::string const &filePath) error::Reason::UnexpectedContent, {}, "Unknown iterationEncoding: " + encoding); - setAttribute("iterationEncoding", encoding); + auto old_written = written(); + written() = false; + setIterationEncoding(encoding_out); + written() = old_written; } else throw std::runtime_error( diff --git a/test/Files_SerialIO/close_iteration_test.cpp b/test/Files_SerialIO/close_iteration_test.cpp index 2e05d18e81..2bb58d4e1a 100644 --- a/test/Files_SerialIO/close_iteration_test.cpp +++ b/test/Files_SerialIO/close_iteration_test.cpp @@ -12,13 +12,117 @@ inline void breakpoint() {} template -auto run_test( +auto run_test_filebased( + WriteIterations &&writeIterations, std::string const &ext) +{ + std::string filename = + "../samples/close_iteration_reopen/filebased_%T." + ext; + Series series( + filename, + /* + * @TODO + * Test READ_WRITE mode too, currently probably not working yet + */ + Access::CREATE, + R"(adios2.use_group_table = true + adios2.modifiable_attributes = true)"); + { + auto it = writeIterations(series)[0]; + auto E_x = it.meshes["E"]["x"]; + E_x.resetDataset({Datatype::INT, {5}}); + std::vector data{0, 1, 2, 3, 4}; + E_x.storeChunk(data, {0}, {5}); + it.close(); + + breakpoint(); + it.open(); + auto B_y = it.meshes["B"]["y"]; + B_y.resetDataset({Datatype::INT, {5}}); + B_y.storeChunk(data, {0}, {5}); + it.close(); + } + + { + auto it = writeIterations(series)[1]; + auto E_x = it.meshes["E"]["x"]; + E_x.resetDataset({Datatype::INT, {5}}); + std::vector data{0, 1, 2, 3, 4}; + E_x.storeChunk(data, {0}, {5}); + it.close(); + + it.open(); + auto e_position_x = it.particles["e"]["position"]["x"]; + e_position_x.resetDataset({Datatype::INT, {5}}); + e_position_x.storeChunk(data, {0}, {5}); + it.close(); + } + { + auto it = writeIterations(series)[2]; + auto E_x = it.meshes["E"]["x"]; + E_x.resetDataset({Datatype::INT, {5}}); + std::vector data{0, 1, 2, 3, 4}; + E_x.storeChunk(data, {0}, {5}); + it.close(); + + it.open(); + it.setTimeUnitSI(2.0); + it.close(); + } + series.close(); + + for (auto mode : {Access::READ_RANDOM_ACCESS, Access::READ_LINEAR}) + { + Series read(filename, mode); + { + auto it = read.snapshots()[0]; + std::vector data(5); + it.meshes["E"]["x"].loadChunkRaw(data.data(), {0}, {5}); + it.close(); + REQUIRE((data == std::vector{0, 1, 2, 3, 4})); + } + REQUIRE(read.iterations.size() == 3); + { + auto it = read.snapshots()[1]; + std::vector data(5); + it.meshes["E"]["x"].loadChunkRaw(data.data(), {0}, {5}); + it.close(); + REQUIRE((data == std::vector{0, 1, 2, 3, 4})); + } + { + auto it = read.snapshots()[2]; + std::vector data(5); + it.meshes["E"]["x"].loadChunkRaw(data.data(), {0}, {5}); + it.close(); + REQUIRE((data == std::vector{0, 1, 2, 3, 4})); + // no guarantee which attribute version we get + REQUIRE((it.timeUnitSI() == 2.0 || it.timeUnitSI() == 1.0)); + } + { + auto it = read.snapshots()[0].open(); + std::vector data(5); + it.meshes["B"]["y"].loadChunkRaw(data.data(), {0}, {5}); + it.close(); + REQUIRE((data == std::vector{0, 1, 2, 3, 4})); + } + { + auto it = read.snapshots()[1].open(); + std::vector data(5); + it.particles["e"]["position"]["x"].loadChunkRaw( + data.data(), {0}, {5}); + it.close(); + REQUIRE((data == std::vector{0, 1, 2, 3, 4})); + } + } +} + +template +auto run_test_groupbased( WriteIterations &&writeIterations, std::string const &ext, std::vector const &readModes) { std::string filename = - "../samples/close_iteration_reopen_groupbased." + ext; + "../samples/close_iteration_reopen/groupbased." + ext; Series series( filename, Access::CREATE, @@ -114,22 +218,27 @@ auto run_test( auto close_and_reopen_test() -> void { - run_test( + run_test_filebased([](Series &s) { return s.iterations; }, "bp"); + run_test_filebased([](Series &s) { return s.writeIterations(); }, "bp"); + run_test_filebased([](Series &s) { return s.snapshots(); }, "bp"); + // run_test_filebased([](Series &s) { return s.snapshots(); }, "json"); + + run_test_groupbased( [](Series &s) { return s.iterations; }, "bp4", {Access::READ_ONLY, Access::READ_LINEAR}); // since these write data in a way that distributes one iteration's data // over multiple steps, only random access read mode makes sense - run_test( + run_test_groupbased( [](Series &s) { return s.writeIterations(); }, "bp4", {Access::READ_RANDOM_ACCESS}); - run_test( + run_test_groupbased( [](Series &s) { return s.snapshots(); }, "bp4", {Access::READ_RANDOM_ACCESS}); // that doesnt matter for json tho - run_test( + run_test_groupbased( [](Series &s) { return s.snapshots(); }, "json", {Access::READ_RANDOM_ACCESS, Access::READ_LINEAR}); From c03ba192646b8fb251a679ea55059826fc45be70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 21 Feb 2024 17:40:11 +0100 Subject: [PATCH 63/93] Now supports READ_WRITE too in filebased mode --- include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp | 5 +- include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp | 1 + include/openPMD/IO/IOTask.hpp | 2 + src/IO/ADIOS/ADIOS2IOHandler.cpp | 58 +++++++++++------ src/Series.cpp | 11 +++- test/Files_SerialIO/close_iteration_test.cpp | 67 +++++++++++++++++--- 6 files changed, 112 insertions(+), 32 deletions(-) diff --git a/include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp b/include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp index 7648cfeb20..3a1abd7744 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp @@ -69,8 +69,9 @@ namespace adios_defs enum class OpenFileAs { - CreateFile, - OpenFile + Create, + Open, + Reopen }; } // namespace adios_defs diff --git a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp index f5a2fc12bd..c4e840a8b8 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp @@ -406,6 +406,7 @@ class ADIOS2IOHandlerImpl enum class IfFileNotOpen : char { + ReopenImplicitly, OpenImplicitly, CreateImplicitly, ThrowError diff --git a/include/openPMD/IO/IOTask.hpp b/include/openPMD/IO/IOTask.hpp index 731372f9e1..cc3277e15b 100644 --- a/include/openPMD/IO/IOTask.hpp +++ b/include/openPMD/IO/IOTask.hpp @@ -188,6 +188,8 @@ struct OPENPMDAPI_EXPORT Parameter } std::string name = ""; + // true <-> file was previously created and is now opened again + bool reopen = false; using ParsePreference = internal::ParsePreference; std::shared_ptr out_parsePreference = std::make_shared(ParsePreference::UpFront); diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index 03a7369bb8..95e7913606 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -931,7 +931,10 @@ void ADIOS2IOHandlerImpl::openFile( // enforce opening the file // lazy opening is deathly in parallel situations - auto &fileData = getFileData(file, IfFileNotOpen::OpenImplicitly); + auto &fileData = getFileData( + file, + parameters.reopen ? IfFileNotOpen::ReopenImplicitly + : IfFileNotOpen::OpenImplicitly); *parameters.out_parsePreference = fileData.parsePreference; m_dirty.emplace(std::move(file)); } @@ -1559,9 +1562,10 @@ adios2::Mode ADIOS2IOHandlerImpl::adios2AccessMode( switch (openFileAs) { - case adios_defs::OpenFileAs::CreateFile: + case adios_defs::OpenFileAs::Create: return adios2::Mode::Write; - case adios_defs::OpenFileAs::OpenFile: + case adios_defs::OpenFileAs::Open: + case adios_defs::OpenFileAs::Reopen: return adios2::Mode::Append; } break; @@ -1593,7 +1597,17 @@ adios2::Mode ADIOS2IOHandlerImpl::adios2AccessMode( { case IterationEncoding::fileBased: - return adios2::Mode::ReadRandomAccess; + switch (openFileAs) + { + + case adios_defs::OpenFileAs::Create: + return adios2::Mode::Write; + case adios_defs::OpenFileAs::Open: + return adios2::Mode::ReadRandomAccess; + case adios_defs::OpenFileAs::Reopen: + return adios2::Mode::Append; + } + break; case IterationEncoding::groupBased: case IterationEncoding::variableBased: return adios2::Mode::Read; @@ -1700,37 +1714,41 @@ detail::ADIOS2File &ADIOS2IOHandlerImpl::getFileData( file.valid(), "[ADIOS2] Cannot retrieve file data for a file that has " "been overwritten or deleted.") + auto openFileAs = [&]() { + using OF = adios_defs::OpenFileAs; + switch (flag) + { + case IfFileNotOpen::ReopenImplicitly: + return OF::Reopen; + case IfFileNotOpen::OpenImplicitly: + return OF::Open; + case IfFileNotOpen::CreateImplicitly: + return OF::Create; + case IfFileNotOpen::ThrowError: + break; + } + return OF{}; + }(); auto it = m_fileData.find(file); if (it == m_fileData.end()) { switch (flag) { - case IfFileNotOpen::OpenImplicitly: { - - auto res = m_fileData.emplace( - file, - std::make_unique( - *this, file, adios_defs::OpenFileAs::OpenFile)); - return *res.first->second; - } - case IfFileNotOpen::CreateImplicitly: { - auto res = m_fileData.emplace( - file, - std::make_unique( - *this, file, adios_defs::OpenFileAs::CreateFile)); - return *res.first->second; - } case IfFileNotOpen::ThrowError: throw std::runtime_error( "[ADIOS2] Requested file has not been opened yet: " + (file.fileState ? file.fileState->name : "Unknown file name")); + default: + auto res = m_fileData.emplace( + file, + std::make_unique(*this, file, openFileAs)); + return *res.first->second; } } else { return *it->second; } - throw std::runtime_error("Unreachable!"); } void ADIOS2IOHandlerImpl::dropFileData(InvalidatableFile const &file) diff --git a/src/Series.cpp b/src/Series.cpp index 99564e7155..e92634ba65 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -28,6 +28,7 @@ #include "openPMD/IO/DummyIOHandler.hpp" #include "openPMD/IO/Format.hpp" #include "openPMD/IO/IOTask.hpp" +#include "openPMD/Iteration.hpp" #include "openPMD/IterationEncoding.hpp" #include "openPMD/ThrowError.hpp" #include "openPMD/auxiliary/Date.hpp" @@ -2681,9 +2682,9 @@ auto Series::openIterationIfDirty(IterationIndex_t index, Iteration &iteration) void Series::openIteration(IterationIndex_t index, Iteration &iteration) { auto oldStatus = iteration.get().m_closed; + using CL = internal::CloseStatus; switch (oldStatus) { - using CL = internal::CloseStatus; case CL::Closed: case CL::Open: iteration.get().m_closed = CL::Open; @@ -2724,6 +2725,14 @@ void Series::openIteration(IterationIndex_t index, Iteration &iteration) // open the iteration's file again Parameter fOpen; fOpen.name = iterationFilename(index); + fOpen.reopen = oldStatus == CL::Closed && + // The filename only gets emplaced in there if we found it on the + // file system, otherwise it's generated by iterationFilename(). + // This helps us distinguish which iterations were created by us and + // which ones existed already. This is important for ADIOS2 which + // can open an iteration for Appending XOR for reading. + series.m_iterationFilenames.find(index) == + series.m_iterationFilenames.end(); IOHandler()->enqueue(IOTask(this, fOpen)); /* open base path */ diff --git a/test/Files_SerialIO/close_iteration_test.cpp b/test/Files_SerialIO/close_iteration_test.cpp index 2bb58d4e1a..092602fd05 100644 --- a/test/Files_SerialIO/close_iteration_test.cpp +++ b/test/Files_SerialIO/close_iteration_test.cpp @@ -1,5 +1,6 @@ #include "SerialIOTests.hpp" #include "openPMD/IO/Access.hpp" +#include "openPMD/auxiliary/Filesystem.hpp" #include @@ -8,21 +9,15 @@ namespace close_and_reopen_test using namespace openPMD; #if openPMD_HAVE_ADIOS2 -inline void breakpoint() -{} - template auto run_test_filebased( WriteIterations &&writeIterations, std::string const &ext) { std::string filename = "../samples/close_iteration_reopen/filebased_%T." + ext; + auxiliary::remove_directory("../samples/close_iteration_reopen"); Series series( filename, - /* - * @TODO - * Test READ_WRITE mode too, currently probably not working yet - */ Access::CREATE, R"(adios2.use_group_table = true adios2.modifiable_attributes = true)"); @@ -34,7 +29,6 @@ auto run_test_filebased( E_x.storeChunk(data, {0}, {5}); it.close(); - breakpoint(); it.open(); auto B_y = it.meshes["B"]["y"]; B_y.resetDataset({Datatype::INT, {5}}); @@ -70,6 +64,46 @@ auto run_test_filebased( } series.close(); + series = Series( + filename, + Access::READ_WRITE, + R"(adios2.use_group_table = true + adios2.modifiable_attributes = true)"); + + { + // @todo proper support for READ_WRITE in snapshots() + auto it = series.iterations[0].open(); + std::vector data(5); + it.meshes["E"]["x"].loadChunkRaw(data.data(), {0}, {5}); + it.close(); + REQUIRE((data == std::vector{0, 1, 2, 3, 4})); + } + { + auto it = series.iterations[2].open(); + std::vector data(5); + it.meshes["E"]["x"].loadChunkRaw(data.data(), {0}, {5}); + it.close(); + REQUIRE((data == std::vector{0, 1, 2, 3, 4})); + // no guarantee which attribute version we get + REQUIRE((it.timeUnitSI() == 2.0 || it.timeUnitSI() == 1.0)); + } + + { + auto it = series.iterations[3].open(); + auto E_x = it.meshes["E"]["x"]; + E_x.resetDataset({Datatype::INT, {5}}); + std::vector data{0, 1, 2, 3, 4}; + E_x.storeChunk(data, {0}, {5}); + it.close(); + + it.open(); + auto e_position_x = it.particles["e"]["position"]["x"]; + e_position_x.resetDataset({Datatype::INT, {5}}); + e_position_x.storeChunk(data, {0}, {5}); + it.close(); + } + series.close(); + for (auto mode : {Access::READ_RANDOM_ACCESS, Access::READ_LINEAR}) { Series read(filename, mode); @@ -80,7 +114,7 @@ auto run_test_filebased( it.close(); REQUIRE((data == std::vector{0, 1, 2, 3, 4})); } - REQUIRE(read.iterations.size() == 3); + REQUIRE(read.iterations.size() == 4); { auto it = read.snapshots()[1]; std::vector data(5); @@ -88,6 +122,13 @@ auto run_test_filebased( it.close(); REQUIRE((data == std::vector{0, 1, 2, 3, 4})); } + { + auto it = read.snapshots()[3]; + std::vector data(5); + it.meshes["E"]["x"].loadChunkRaw(data.data(), {0}, {5}); + it.close(); + REQUIRE((data == std::vector{0, 1, 2, 3, 4})); + } { auto it = read.snapshots()[2]; std::vector data(5); @@ -112,6 +153,14 @@ auto run_test_filebased( it.close(); REQUIRE((data == std::vector{0, 1, 2, 3, 4})); } + { + auto it = read.snapshots()[3].open(); + std::vector data(5); + it.particles["e"]["position"]["x"].loadChunkRaw( + data.data(), {0}, {5}); + it.close(); + REQUIRE((data == std::vector{0, 1, 2, 3, 4})); + } } } From 28c0516faf008cf4010f5a2015dfe2425f03af14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 21 Feb 2024 17:48:42 +0100 Subject: [PATCH 64/93] Some exceptions for unimplemented stuff --- src/Series.cpp | 2 ++ src/snapshots/ContainerImpls.cpp | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/src/Series.cpp b/src/Series.cpp index e92634ba65..0639b7f289 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -3078,6 +3078,8 @@ Snapshots Series::snapshots() break; case Access::READ_WRITE: + throw std::runtime_error( + "[Series::snapshots()] Unimplemented for READ_WRITE mode."); case Access::CREATE: case Access::APPEND: // @todo: properly distinguish here diff --git a/src/snapshots/ContainerImpls.cpp b/src/snapshots/ContainerImpls.cpp index 7e625bd1a4..480c11b660 100644 --- a/src/snapshots/ContainerImpls.cpp +++ b/src/snapshots/ContainerImpls.cpp @@ -152,6 +152,12 @@ auto StatefulSnapshotsContainer::operator[](key_type const &key) auto access = s.series.IOHandler()->m_frontendAccess; // @todo distinguish read_write + if (access == Access::READ_WRITE) + { + throw std::runtime_error( + "[StatefulSnapshotsContainer::operator[]()] Unimplemented for " + "READ_WRITE mode."); + } if (access::write(access)) { auto lastIteration = base_iterator->peekCurrentlyOpenIteration(); From 0ed110fb77a3bb2c3f740b26a9bb9c5a35726272 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 22 Feb 2024 13:28:31 +0100 Subject: [PATCH 65/93] Works in JSON and HDF5 now too --- include/openPMD/backend/Attributable.hpp | 1 - src/Series.cpp | 11 +++++++++++ src/snapshots/StatefulIterator.cpp | 17 +++++++++++++++-- test/Files_SerialIO/close_iteration_test.cpp | 20 ++++++++++++++------ 4 files changed, 40 insertions(+), 9 deletions(-) diff --git a/include/openPMD/backend/Attributable.hpp b/include/openPMD/backend/Attributable.hpp index 8117081128..12fe37819e 100644 --- a/include/openPMD/backend/Attributable.hpp +++ b/include/openPMD/backend/Attributable.hpp @@ -78,7 +78,6 @@ namespace internal */ Writable m_writable; - private: /** * The attributes defined by this Attributable. */ diff --git a/src/Series.cpp b/src/Series.cpp index 0639b7f289..19bbb688d6 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -2296,6 +2296,12 @@ void Series::readBase() "string, found " + datatypeToString(Attribute(*aRead.resource).dtype) + ")"); } + else + { + // Make sure that the meshesPath does not leak from one iteration into + // the other in file-based iteration encoding + get().m_attributes.erase("meshesPath"); + } if (std::count( aList.attributes->begin(), @@ -2328,6 +2334,11 @@ void Series::readBase() "string, found " + datatypeToString(Attribute(*aRead.resource).dtype) + ")"); } + { + // Make sure that the particlesPath does not leak from one iteration + // into the other in file-based iteration encoding + get().m_attributes.erase("particlesPath"); + } } std::string Series::iterationFilename(IterationIndex_t i) diff --git a/src/snapshots/StatefulIterator.cpp b/src/snapshots/StatefulIterator.cpp index ee0fdb1bfc..522f126f47 100644 --- a/src/snapshots/StatefulIterator.cpp +++ b/src/snapshots/StatefulIterator.cpp @@ -471,7 +471,6 @@ StatefulIterator::StatefulIterator( switch (series.iterationEncoding()) { - case IterationEncoding::fileBased: { initIteratorFilebased(); break; @@ -646,6 +645,17 @@ std::optional StatefulIterator::loopBody(Seek const &seek) */ auto maybe_current_iteration = data.currentStep.get_iteration_index(); if (maybe_current_iteration.has_value() && + // don't deactivate the iteration if it's the one that's currently + // active anyway + std::visit( + auxiliary::overloaded{ + [&](detail::seek_types::Next_t const &) { return true; }, + [&](detail::seek_types::Seek_Iteration_t const + &go_to_iteration) { + return go_to_iteration.iteration_idx != + **maybe_current_iteration; + }}, + seek) && iterations.contains(**maybe_current_iteration)) { auto ¤tIteration = iterations.at(**maybe_current_iteration); @@ -653,7 +663,10 @@ std::optional StatefulIterator::loopBody(Seek const &seek) { currentIteration.close(); } - if (series.IOHandler()->m_frontendAccess == Access::READ_LINEAR) + // sic! only erase if the Iteration was explicitly closed, don't + // implicitly sweep the iteration away from under the user + else if ( + series.IOHandler()->m_frontendAccess == Access::READ_LINEAR) { data.series.iterations.container().erase( **maybe_current_iteration); diff --git a/test/Files_SerialIO/close_iteration_test.cpp b/test/Files_SerialIO/close_iteration_test.cpp index 092602fd05..d8dcf5ea20 100644 --- a/test/Files_SerialIO/close_iteration_test.cpp +++ b/test/Files_SerialIO/close_iteration_test.cpp @@ -201,9 +201,9 @@ auto run_test_groupbased( it.close(); it.open(); - auto e_position_x = it.particles["e"]["position"]["x"]; - e_position_x.resetDataset({Datatype::INT, {5}}); - e_position_x.storeChunk(data, {0}, {5}); + auto E_y = it.meshes["E"]["y"]; + E_y.resetDataset({Datatype::INT, {5}}); + E_y.storeChunk(data, {0}, {5}); it.close(); } { @@ -257,8 +257,7 @@ auto run_test_groupbased( { auto it = read.snapshots()[1].open(); std::vector data(5); - it.particles["e"]["position"]["x"].loadChunkRaw( - data.data(), {0}, {5}); + it.meshes["E"]["y"].loadChunkRaw(data.data(), {0}, {5}); it.close(); REQUIRE((data == std::vector{0, 1, 2, 3, 4})); } @@ -270,7 +269,10 @@ auto close_and_reopen_test() -> void run_test_filebased([](Series &s) { return s.iterations; }, "bp"); run_test_filebased([](Series &s) { return s.writeIterations(); }, "bp"); run_test_filebased([](Series &s) { return s.snapshots(); }, "bp"); - // run_test_filebased([](Series &s) { return s.snapshots(); }, "json"); + run_test_filebased([](Series &s) { return s.snapshots(); }, "json"); +#if openPMD_HAVE_HDF5 + run_test_filebased([](Series &s) { return s.snapshots(); }, "h5"); +#endif run_test_groupbased( [](Series &s) { return s.iterations; }, @@ -291,6 +293,12 @@ auto close_and_reopen_test() -> void [](Series &s) { return s.snapshots(); }, "json", {Access::READ_RANDOM_ACCESS, Access::READ_LINEAR}); +#if openPMD_HAVE_HDF5 + run_test_groupbased( + [](Series &s) { return s.snapshots(); }, + "h5", + {Access::READ_RANDOM_ACCESS, Access::READ_LINEAR}); +#endif } #else auto close_and_reopen_test() -> void From 27e02bfdc59d08bb2586dd80ecf0321a0e52763a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 23 Feb 2024 17:30:58 +0100 Subject: [PATCH 66/93] CI fixes --- .../openPMD/snapshots/StatefulIterator.hpp | 34 ++++++++++++++++--- src/snapshots/ContainerImpls.cpp | 4 ++- src/snapshots/StatefulIterator.cpp | 16 ++++----- 3 files changed, 40 insertions(+), 14 deletions(-) diff --git a/include/openPMD/snapshots/StatefulIterator.hpp b/include/openPMD/snapshots/StatefulIterator.hpp index 19915291cd..ee66c03403 100644 --- a/include/openPMD/snapshots/StatefulIterator.hpp +++ b/include/openPMD/snapshots/StatefulIterator.hpp @@ -101,9 +101,20 @@ namespace detail }; template - auto map_during_t(F &&map, G &&create_new); + void map_during_t(F &&map, G &&create_new); template - auto map_during_t(F &&map); + void map_during_t(F &&map); + + // casts needed because of + // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=90943 + inline auto as_base() const -> variant_t const & + { + return *this; + } + inline auto as_base() -> variant_t & + { + return *this; + } }; namespace seek_types @@ -122,6 +133,19 @@ namespace detail using Seek_Iteration_t = seek_types::Seek_Iteration_t; static constexpr Next_t const Next{}; + + using variant_t = + std::variant; + // casts needed because of + // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=90943 + inline auto as_base() const -> variant_t const & + { + return *this; + } + inline auto as_base() -> variant_t & + { + return *this; + } }; } // namespace detail @@ -335,7 +359,7 @@ class ReadIterations namespace openPMD::detail { template -auto CurrentStep::map_during_t(F &&map, G &&create_new) +void CurrentStep::map_during_t(F &&map, G &&create_new) { std::visit( auxiliary::overloaded{ @@ -356,11 +380,11 @@ auto CurrentStep::map_during_t(F &&map, G &&create_new) this->swap(*res); } }}, - *this); + this->as_base()); } template -auto CurrentStep::map_during_t(F &&map) +void CurrentStep::map_during_t(F &&map) { map_during_t( std::forward(map), [](auto const &) { return std::nullopt; }); diff --git a/src/snapshots/ContainerImpls.cpp b/src/snapshots/ContainerImpls.cpp index 480c11b660..cea53b72f5 100644 --- a/src/snapshots/ContainerImpls.cpp +++ b/src/snapshots/ContainerImpls.cpp @@ -183,7 +183,8 @@ auto StatefulSnapshotsContainer::operator[](key_type const &key) during.iteration_idx = key; during.available_iterations_in_step = {key}; }, - [&](detail::CurrentStep::AtTheEdge where_am_i) { + [&](detail::CurrentStep::AtTheEdge where_am_i) + -> detail::CurrentStep::During_t { base_iterator->get().seen_iterations[key] = 0; switch (where_am_i) { @@ -195,6 +196,7 @@ auto StatefulSnapshotsContainer::operator[](key_type const &key) "stream is " "closed?"); } + throw std::runtime_error("Unreachable!"); }); } auto &res = s.series.iterations[key]; diff --git a/src/snapshots/StatefulIterator.cpp b/src/snapshots/StatefulIterator.cpp index 522f126f47..6c5cdd3f15 100644 --- a/src/snapshots/StatefulIterator.cpp +++ b/src/snapshots/StatefulIterator.cpp @@ -97,7 +97,7 @@ namespace detail return std::nullopt; } }}, - *this); + this->as_base()); } auto CurrentStep::get_iteration_index() -> std::optional @@ -612,7 +612,7 @@ StatefulIterator::nextStep(size_t recursion_depth) auxiliary::overloaded{ [](CurrentStep::Before_t const &) { return false; }, [](auto const &) { return true; }}, - data.currentStep); + data.currentStep.as_base()); } throw std::runtime_error("Unreachable!"); }(); @@ -655,7 +655,7 @@ std::optional StatefulIterator::loopBody(Seek const &seek) return go_to_iteration.iteration_idx != **maybe_current_iteration; }}, - seek) && + seek.as_base()) && iterations.contains(**maybe_current_iteration)) { auto ¤tIteration = iterations.at(**maybe_current_iteration); @@ -744,7 +744,7 @@ std::optional StatefulIterator::loopBody(Seek const &seek) return skipToIterationInStep( skip_to_iteration.iteration_idx); }}, - seek); + seek.as_base()); if (optionallyAStep.has_value()) { return guardReturn(optionallyAStep); @@ -771,7 +771,7 @@ std::optional StatefulIterator::loopBody(Seek const &seek) std::to_string(skip_to_iteration.iteration_idx) + "from another step not supported yet"); }}, - seek); + seek.as_base()); return guardReturn(option); } @@ -936,7 +936,7 @@ auto StatefulIterator::is_end() const -> bool return !during.iteration_idx.has_value(); }, [](CurrentStep::After_t const &) { return true; }}, - (**m_data).currentStep); + (**m_data).currentStep.as_base()); } auto StatefulIterator::assert_end_iterator() const -> void @@ -976,13 +976,13 @@ auto LegacyIteratorAdaptor::operator==(LegacyIteratorAdaptor const &other) const -> bool { return m_iterator == other.m_iterator; -}; +} auto LegacyIteratorAdaptor::operator!=(LegacyIteratorAdaptor const &other) const -> bool { return m_iterator != other.m_iterator; -}; +} ReadIterations::ReadIterations( Series series, From 99d814685f43cb84b8c9ce2d2d36e5784d14b411 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 26 Feb 2024 12:28:02 +0100 Subject: [PATCH 67/93] Virtual destructors --- include/openPMD/snapshots/ContainerImpls.hpp | 21 ++++++++++++++++ include/openPMD/snapshots/ContainerTraits.hpp | 2 ++ include/openPMD/snapshots/IteratorTraits.hpp | 3 +++ .../snapshots/RandomAccessIterator.hpp | 8 ++++++ .../openPMD/snapshots/StatefulIterator.hpp | 7 ++++++ src/snapshots/ContainerImpls.cpp | 25 +++++++++++++++++++ src/snapshots/ContainerTraits.cpp | 2 ++ src/snapshots/IteratorTraits.cpp | 7 ++++++ src/snapshots/RandomAccessIterator.cpp | 16 ++++++++++++ src/snapshots/StatefulIterator.cpp | 8 ++++++ 10 files changed, 99 insertions(+) diff --git a/include/openPMD/snapshots/ContainerImpls.hpp b/include/openPMD/snapshots/ContainerImpls.hpp index eaa589aaa4..db168b8ee5 100644 --- a/include/openPMD/snapshots/ContainerImpls.hpp +++ b/include/openPMD/snapshots/ContainerImpls.hpp @@ -20,6 +20,16 @@ class StatefulSnapshotsContainer : public AbstractSnapshotsContainer auto get() const -> StatefulIterator const *; public: + ~StatefulSnapshotsContainer() override; + + StatefulSnapshotsContainer(StatefulSnapshotsContainer const &other); + StatefulSnapshotsContainer(StatefulSnapshotsContainer &&other) noexcept; + + StatefulSnapshotsContainer & + operator=(StatefulSnapshotsContainer const &other); + StatefulSnapshotsContainer & + operator=(StatefulSnapshotsContainer &&other) noexcept; + auto currentIteration() -> std::optional override; auto currentIteration() const -> std::optional override; @@ -61,6 +71,17 @@ class RandomAccessIteratorContainer : public AbstractSnapshotsContainer RandomAccessIteratorContainer(Container cont); public: + ~RandomAccessIteratorContainer() override; + + RandomAccessIteratorContainer(RandomAccessIteratorContainer const &other); + RandomAccessIteratorContainer( + RandomAccessIteratorContainer &&other) noexcept; + + RandomAccessIteratorContainer & + operator=(RandomAccessIteratorContainer const &other); + RandomAccessIteratorContainer & + operator=(RandomAccessIteratorContainer &&other) noexcept; + auto begin() -> iterator override; auto end() -> iterator override; auto begin() const -> const_iterator override; diff --git a/include/openPMD/snapshots/ContainerTraits.hpp b/include/openPMD/snapshots/ContainerTraits.hpp index 90e2d0ee1f..a6122414fd 100644 --- a/include/openPMD/snapshots/ContainerTraits.hpp +++ b/include/openPMD/snapshots/ContainerTraits.hpp @@ -59,6 +59,8 @@ class AbstractSnapshotsContainer using reverse_iterator = OpaqueSeriesIterator; using const_reverse_iterator = OpaqueSeriesIterator; + virtual ~AbstractSnapshotsContainer() = 0; + virtual auto currentIteration() -> std::optional; virtual auto currentIteration() const -> std::optional; diff --git a/include/openPMD/snapshots/IteratorTraits.hpp b/include/openPMD/snapshots/IteratorTraits.hpp index 01ae319d3a..90c6eb8981 100644 --- a/include/openPMD/snapshots/IteratorTraits.hpp +++ b/include/openPMD/snapshots/IteratorTraits.hpp @@ -35,6 +35,7 @@ class DynamicSeriesIterator { public: using difference_type = Iteration::IterationIndex_t; + virtual ~DynamicSeriesIterator() = 0; protected: template @@ -69,6 +70,8 @@ class AbstractSeriesIterator : public DynamicSeriesIterator using difference_type = Iteration::IterationIndex_t; using value_type = value_type_in; + ~AbstractSeriesIterator() override; + // dereference // value_type const &operator*() const = 0; value_type &operator*(); diff --git a/include/openPMD/snapshots/RandomAccessIterator.hpp b/include/openPMD/snapshots/RandomAccessIterator.hpp index bc5d494137..5635bdeda6 100644 --- a/include/openPMD/snapshots/RandomAccessIterator.hpp +++ b/include/openPMD/snapshots/RandomAccessIterator.hpp @@ -53,6 +53,14 @@ class RandomAccessIterator using parent_t::operator*; using typename parent_t::value_type; + ~RandomAccessIterator() override; + + RandomAccessIterator(RandomAccessIterator const &other); + RandomAccessIterator(RandomAccessIterator &&other) noexcept; + + RandomAccessIterator &operator=(RandomAccessIterator const &other); + RandomAccessIterator &operator=(RandomAccessIterator &&other) noexcept; + value_type const &operator*() const; RandomAccessIterator &operator++(); RandomAccessIterator &operator--(); diff --git a/include/openPMD/snapshots/StatefulIterator.hpp b/include/openPMD/snapshots/StatefulIterator.hpp index ee66c03403..64c56b2e9b 100644 --- a/include/openPMD/snapshots/StatefulIterator.hpp +++ b/include/openPMD/snapshots/StatefulIterator.hpp @@ -207,6 +207,13 @@ class StatefulIterator //! construct the end() iterator explicit StatefulIterator(); + ~StatefulIterator() override; + + StatefulIterator(StatefulIterator const &other); + StatefulIterator(StatefulIterator &&other) noexcept; + + StatefulIterator &operator=(StatefulIterator const &other); + StatefulIterator &operator=(StatefulIterator &&other) noexcept; class tag_write_t {}; diff --git a/src/snapshots/ContainerImpls.cpp b/src/snapshots/ContainerImpls.cpp index cea53b72f5..6a11133f2b 100644 --- a/src/snapshots/ContainerImpls.cpp +++ b/src/snapshots/ContainerImpls.cpp @@ -27,6 +27,16 @@ StatefulSnapshotsContainer::StatefulSnapshotsContainer( std::function begin) : m_begin(std::move(begin)) {} + +StatefulSnapshotsContainer::StatefulSnapshotsContainer( + StatefulSnapshotsContainer const &other) = default; +StatefulSnapshotsContainer::StatefulSnapshotsContainer( + StatefulSnapshotsContainer &&other) noexcept = default; +StatefulSnapshotsContainer &StatefulSnapshotsContainer::operator=( + StatefulSnapshotsContainer const &other) = default; +StatefulSnapshotsContainer &StatefulSnapshotsContainer::operator=( + StatefulSnapshotsContainer &&other) noexcept = default; + auto StatefulSnapshotsContainer::get() -> StatefulIterator * { if (!m_bufferedIterator.has_value()) @@ -63,6 +73,9 @@ auto StatefulSnapshotsContainer::currentIteration() const return nullptr; } } + +StatefulSnapshotsContainer::~StatefulSnapshotsContainer() = default; + auto StatefulSnapshotsContainer::begin() -> iterator { return stateful_to_opaque(*get()); @@ -251,6 +264,18 @@ RandomAccessIteratorContainer::RandomAccessIteratorContainer( Container cont) : m_cont(std::move(cont)) {} + +RandomAccessIteratorContainer::~RandomAccessIteratorContainer() = default; + +RandomAccessIteratorContainer::RandomAccessIteratorContainer( + RandomAccessIteratorContainer const &other) = default; +RandomAccessIteratorContainer::RandomAccessIteratorContainer( + RandomAccessIteratorContainer &&other) noexcept = default; +RandomAccessIteratorContainer &RandomAccessIteratorContainer::operator=( + RandomAccessIteratorContainer const &other) = default; +RandomAccessIteratorContainer &RandomAccessIteratorContainer::operator=( + RandomAccessIteratorContainer &&other) noexcept = default; + auto RandomAccessIteratorContainer::begin() -> iterator { return OpaqueSeriesIterator( diff --git a/src/snapshots/ContainerTraits.cpp b/src/snapshots/ContainerTraits.cpp index 547b3020b5..0bf5e85a20 100644 --- a/src/snapshots/ContainerTraits.cpp +++ b/src/snapshots/ContainerTraits.cpp @@ -85,6 +85,8 @@ using value_type = template class OpaqueSeriesIterator; template class OpaqueSeriesIterator; +AbstractSnapshotsContainer::~AbstractSnapshotsContainer() = default; + auto AbstractSnapshotsContainer::currentIteration() -> std::optional { diff --git a/src/snapshots/IteratorTraits.cpp b/src/snapshots/IteratorTraits.cpp index ef18257a4a..d45490bd90 100644 --- a/src/snapshots/IteratorTraits.cpp +++ b/src/snapshots/IteratorTraits.cpp @@ -6,6 +6,9 @@ namespace openPMD { +template +DynamicSeriesIterator::~DynamicSeriesIterator() = default; + template auto DynamicSeriesIterator::dereference_operator() -> value_type & { @@ -14,6 +17,10 @@ auto DynamicSeriesIterator::dereference_operator() -> value_type & ->dereference_operator()); } +template +AbstractSeriesIterator::~AbstractSeriesIterator() = + default; + // dereference template auto AbstractSeriesIterator::operator*() -> value_type & diff --git a/src/snapshots/RandomAccessIterator.cpp b/src/snapshots/RandomAccessIterator.cpp index 816fb13487..506546ac41 100644 --- a/src/snapshots/RandomAccessIterator.cpp +++ b/src/snapshots/RandomAccessIterator.cpp @@ -6,6 +6,22 @@ inline RandomAccessIterator::RandomAccessIterator(iterator_t it) : m_it(it) {} +template +RandomAccessIterator::~RandomAccessIterator() = default; + +template +RandomAccessIterator::RandomAccessIterator( + RandomAccessIterator const &other) = default; +template +RandomAccessIterator::RandomAccessIterator( + RandomAccessIterator &&other) noexcept = default; +template +RandomAccessIterator &RandomAccessIterator::operator=( + RandomAccessIterator const &other) = default; +template +RandomAccessIterator &RandomAccessIterator::operator=( + RandomAccessIterator &&other) noexcept = default; + template auto RandomAccessIterator::operator*() const -> value_type const & { diff --git a/src/snapshots/StatefulIterator.cpp b/src/snapshots/StatefulIterator.cpp index 6c5cdd3f15..b29cd4a53c 100644 --- a/src/snapshots/StatefulIterator.cpp +++ b/src/snapshots/StatefulIterator.cpp @@ -186,6 +186,14 @@ namespace } // namespace StatefulIterator::StatefulIterator() = default; +StatefulIterator::~StatefulIterator() = default; + +StatefulIterator::StatefulIterator(StatefulIterator const &other) = default; +StatefulIterator::StatefulIterator(StatefulIterator &&other) noexcept = default; +StatefulIterator & +StatefulIterator::operator=(StatefulIterator const &other) = default; +StatefulIterator & +StatefulIterator::operator=(StatefulIterator &&other) noexcept = default; auto StatefulIterator::get() -> SharedData & { From 49787c3a9778058856f298413217c8c8e4d9997e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 26 Feb 2024 15:15:28 +0100 Subject: [PATCH 68/93] CI fixes continued --- src/IO/ADIOS/ADIOS2IOHandler.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index 95e7913606..ca4c01c97c 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -661,10 +661,12 @@ void ADIOS2IOHandlerImpl::createFile( // print this warning only in the new layout (with group table) if (m_useGroupTable.value_or(UseGroupTable::No) == UseGroupTable::Yes && - (m_engineType == "bp5" || - (openPMD_HAS_ADIOS_2_9 && - (m_engineType == "file" || m_engineType == "filestream" || - m_engineType == "bp")))) + (m_engineType == "bp5" +#if openPMD_HAS_ADIOS_2_9 + || (m_engineType == "file" || m_engineType == "filestream" || + m_engineType == "bp") +#endif + )) { std::cerr << warningADIOS2NoGroupbasedEncoding << std::endl; printedWarningsAlready.noGroupBased = true; From 3584de6def1488c532a1e1133b7632416bd80c13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 26 Feb 2024 15:46:34 +0100 Subject: [PATCH 69/93] Some fixes for noexcept specifications --- include/openPMD/backend/Attributable.hpp | 2 +- include/openPMD/backend/Container.hpp | 4 ++-- include/openPMD/snapshots/RandomAccessIterator.hpp | 7 +++++-- src/backend/Attributable.cpp | 2 +- src/snapshots/RandomAccessIterator.cpp | 9 +++++++-- 5 files changed, 16 insertions(+), 8 deletions(-) diff --git a/include/openPMD/backend/Attributable.hpp b/include/openPMD/backend/Attributable.hpp index 12fe37819e..19216183c2 100644 --- a/include/openPMD/backend/Attributable.hpp +++ b/include/openPMD/backend/Attributable.hpp @@ -219,7 +219,7 @@ class Attributable public: Attributable(); - Attributable(NoInit); + Attributable(NoInit) noexcept; virtual ~Attributable() = default; diff --git a/include/openPMD/backend/Container.hpp b/include/openPMD/backend/Container.hpp index 974b106549..629932ad57 100644 --- a/include/openPMD/backend/Container.hpp +++ b/include/openPMD/backend/Container.hpp @@ -489,7 +489,7 @@ OPENPMD_protected m_containerData = other.m_containerData; } - Container(Container &&other) : Attributable(NoInit()) + Container(Container &&other) noexcept : Attributable(NoInit()) { if (other.m_attri) { @@ -505,7 +505,7 @@ OPENPMD_protected return *this; } - Container &operator=(Container &&other) + Container &operator=(Container &&other) noexcept { if (other.m_attri) { diff --git a/include/openPMD/snapshots/RandomAccessIterator.hpp b/include/openPMD/snapshots/RandomAccessIterator.hpp index 5635bdeda6..a62c1f4ee4 100644 --- a/include/openPMD/snapshots/RandomAccessIterator.hpp +++ b/include/openPMD/snapshots/RandomAccessIterator.hpp @@ -56,10 +56,13 @@ class RandomAccessIterator ~RandomAccessIterator() override; RandomAccessIterator(RandomAccessIterator const &other); - RandomAccessIterator(RandomAccessIterator &&other) noexcept; + RandomAccessIterator(RandomAccessIterator &&other) noexcept( + noexcept(iterator_t(std::declval()))); RandomAccessIterator &operator=(RandomAccessIterator const &other); - RandomAccessIterator &operator=(RandomAccessIterator &&other) noexcept; + RandomAccessIterator & + operator=(RandomAccessIterator &&other) noexcept(noexcept( + std::declval().operator=(std::declval()))); value_type const &operator*() const; RandomAccessIterator &operator++(); diff --git a/src/backend/Attributable.cpp b/src/backend/Attributable.cpp index 5d296a28f5..5e25323dd8 100644 --- a/src/backend/Attributable.cpp +++ b/src/backend/Attributable.cpp @@ -61,7 +61,7 @@ Attributable::Attributable() } } -Attributable::Attributable(NoInit) +Attributable::Attributable(NoInit) noexcept {} Attribute Attributable::getAttribute(std::string const &key) const diff --git a/src/snapshots/RandomAccessIterator.cpp b/src/snapshots/RandomAccessIterator.cpp index 506546ac41..d0248fecff 100644 --- a/src/snapshots/RandomAccessIterator.cpp +++ b/src/snapshots/RandomAccessIterator.cpp @@ -14,13 +14,18 @@ RandomAccessIterator::RandomAccessIterator( RandomAccessIterator const &other) = default; template RandomAccessIterator::RandomAccessIterator( - RandomAccessIterator &&other) noexcept = default; + RandomAccessIterator + &&other) noexcept(noexcept(iterator_t(std::declval()))) = + default; template RandomAccessIterator &RandomAccessIterator::operator=( RandomAccessIterator const &other) = default; template RandomAccessIterator &RandomAccessIterator::operator=( - RandomAccessIterator &&other) noexcept = default; + RandomAccessIterator + &&other) noexcept(noexcept(std::declval(). + operator=(std::declval()))) = + default; template auto RandomAccessIterator::operator*() const -> value_type const & From 3dc2ea37d5aaf28768f176f8c8fdd01276b38fa3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 26 Feb 2024 18:32:00 +0100 Subject: [PATCH 70/93] Further CI Fixes --- src/snapshots/StatefulIterator.cpp | 3 ++- test/Files_SerialIO/close_iteration_test.cpp | 28 +++++++++----------- test/Files_SerialIO/filebased_write_test.cpp | 20 +++++++++++--- 3 files changed, 31 insertions(+), 20 deletions(-) diff --git a/src/snapshots/StatefulIterator.cpp b/src/snapshots/StatefulIterator.cpp index b29cd4a53c..74f24363f1 100644 --- a/src/snapshots/StatefulIterator.cpp +++ b/src/snapshots/StatefulIterator.cpp @@ -65,7 +65,8 @@ namespace detail } template - auto CurrentStep::get_variant() const -> std::optional + [[nodiscard]] auto CurrentStep::get_variant() const + -> std::optional { auto res = std::get_if(*this); if (res) diff --git a/test/Files_SerialIO/close_iteration_test.cpp b/test/Files_SerialIO/close_iteration_test.cpp index d8dcf5ea20..2863c75247 100644 --- a/test/Files_SerialIO/close_iteration_test.cpp +++ b/test/Files_SerialIO/close_iteration_test.cpp @@ -1,4 +1,5 @@ #include "SerialIOTests.hpp" +#include "openPMD/IO/ADIOS/macros.hpp" #include "openPMD/IO/Access.hpp" #include "openPMD/auxiliary/Filesystem.hpp" @@ -9,6 +10,15 @@ namespace close_and_reopen_test using namespace openPMD; #if openPMD_HAVE_ADIOS2 +constexpr char const *write_cfg = +#if openPMD_HAS_ADIOS_2_9 + R"(adios2.use_group_table = true + adios2.modifiable_attributes = true)"; +#else + R"(adios2.use_group_table = false + adios2.modifiable_attributes = false)"; +#endif + template auto run_test_filebased( WriteIterations &&writeIterations, std::string const &ext) @@ -16,11 +26,7 @@ auto run_test_filebased( std::string filename = "../samples/close_iteration_reopen/filebased_%T." + ext; auxiliary::remove_directory("../samples/close_iteration_reopen"); - Series series( - filename, - Access::CREATE, - R"(adios2.use_group_table = true - adios2.modifiable_attributes = true)"); + Series series(filename, Access::CREATE, write_cfg); { auto it = writeIterations(series)[0]; auto E_x = it.meshes["E"]["x"]; @@ -64,11 +70,7 @@ auto run_test_filebased( } series.close(); - series = Series( - filename, - Access::READ_WRITE, - R"(adios2.use_group_table = true - adios2.modifiable_attributes = true)"); + series = Series(filename, Access::READ_WRITE, write_cfg); { // @todo proper support for READ_WRITE in snapshots() @@ -172,11 +174,7 @@ auto run_test_groupbased( { std::string filename = "../samples/close_iteration_reopen/groupbased." + ext; - Series series( - filename, - Access::CREATE, - R"(adios2.use_group_table = true - adios2.modifiable_attributes = true)"); + Series series(filename, Access::CREATE, write_cfg); { auto it = writeIterations(series)[0]; auto E_x = it.meshes["E"]["x"]; diff --git a/test/Files_SerialIO/filebased_write_test.cpp b/test/Files_SerialIO/filebased_write_test.cpp index 488abd7984..0a7109ca2f 100644 --- a/test/Files_SerialIO/filebased_write_test.cpp +++ b/test/Files_SerialIO/filebased_write_test.cpp @@ -80,12 +80,24 @@ void close_and_reopen_iterations( void close_and_reopen_iterations(std::string const &filename) { close_and_reopen_iterations( - filename, Access::READ_LINEAR, "defer_iteration_parsing=false", false); + filename, + Access::READ_LINEAR, + R"({"defer_iteration_parsing":false})", + false); close_and_reopen_iterations( - filename, Access::READ_LINEAR, "defer_iteration_parsing=true", false); + filename, + Access::READ_LINEAR, + R"({"defer_iteration_parsing":true})", + false); close_and_reopen_iterations( - filename, Access::READ_ONLY, "defer_iteration_parsing=false", false); + filename, + Access::READ_ONLY, + R"({"defer_iteration_parsing":false})", + false); close_and_reopen_iterations( - filename, Access::READ_ONLY, "defer_iteration_parsing=true", true); + filename, + Access::READ_ONLY, + R"({"defer_iteration_parsing":true})", + true); } } // namespace filebased_write_test From aa2866fec07dc088638ef18c9cf9755d0b913ced Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 27 Feb 2024 17:46:18 +0100 Subject: [PATCH 71/93] CI FIXES --- test/Files_SerialIO/close_iteration_test.cpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/Files_SerialIO/close_iteration_test.cpp b/test/Files_SerialIO/close_iteration_test.cpp index 2863c75247..efaeaf712b 100644 --- a/test/Files_SerialIO/close_iteration_test.cpp +++ b/test/Files_SerialIO/close_iteration_test.cpp @@ -272,6 +272,11 @@ auto close_and_reopen_test() -> void run_test_filebased([](Series &s) { return s.snapshots(); }, "h5"); #endif + /* + * This test writes the same attribute with different values over steps, + * triggering a bug in ADIOS2 v2.7. + */ +#if openPMD_HAS_ADIOS_2_8 run_test_groupbased( [](Series &s) { return s.iterations; }, "bp4", @@ -291,12 +296,17 @@ auto close_and_reopen_test() -> void [](Series &s) { return s.snapshots(); }, "json", {Access::READ_RANDOM_ACCESS, Access::READ_LINEAR}); +#endif #if openPMD_HAVE_HDF5 run_test_groupbased( [](Series &s) { return s.snapshots(); }, "h5", {Access::READ_RANDOM_ACCESS, Access::READ_LINEAR}); #endif + run_test_groupbased( + [](Series &s) { return s.snapshots(); }, + "json", + {Access::READ_RANDOM_ACCESS, Access::READ_LINEAR}); } #else auto close_and_reopen_test() -> void From 14267e5cdc5cab858e2f7b48f6bd12d628ce4431 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 29 Feb 2024 14:32:20 +0100 Subject: [PATCH 72/93] Fixes for ADIOS2 v2.7 --- src/IO/ADIOS/ADIOS2IOHandler.cpp | 8 ++++---- test/Files_SerialIO/close_iteration_test.cpp | 13 ++++++++++--- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index ca4c01c97c..a3dada698d 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -1594,7 +1594,6 @@ adios2::Mode ADIOS2IOHandlerImpl::adios2AccessMode( if (auxiliary::directory_exists(fullPath) || auxiliary::file_exists(fullPath)) { -#if openPMD_HAS_ADIOS_2_8 switch (m_handler->m_encoding) { @@ -1605,7 +1604,11 @@ adios2::Mode ADIOS2IOHandlerImpl::adios2AccessMode( case adios_defs::OpenFileAs::Create: return adios2::Mode::Write; case adios_defs::OpenFileAs::Open: +#if openPMD_HAS_ADIOS_2_8 return adios2::Mode::ReadRandomAccess; +#else + return adios2::Mode::Read; +#endif case adios_defs::OpenFileAs::Reopen: return adios2::Mode::Append; } @@ -1615,9 +1618,6 @@ adios2::Mode ADIOS2IOHandlerImpl::adios2AccessMode( return adios2::Mode::Read; } break; -#else - return adios2::Mode::Read; -#endif } else { diff --git a/test/Files_SerialIO/close_iteration_test.cpp b/test/Files_SerialIO/close_iteration_test.cpp index efaeaf712b..f4ab333636 100644 --- a/test/Files_SerialIO/close_iteration_test.cpp +++ b/test/Files_SerialIO/close_iteration_test.cpp @@ -64,9 +64,16 @@ auto run_test_filebased( E_x.storeChunk(data, {0}, {5}); it.close(); - it.open(); - it.setTimeUnitSI(2.0); - it.close(); +#if !openPMD_HAS_ADIOS_2_8 + if (series.backend() != "ADIOS2") + { +#endif + it.open(); + it.setTimeUnitSI(2.0); + it.close(); +#if !openPMD_HAS_ADIOS_2_8 + } +#endif } series.close(); From 0139fa5d6fab19dbc22fd4af58e479862af1baf8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 5 Mar 2024 17:47:25 +0100 Subject: [PATCH 73/93] placate the intel compiler --- include/openPMD/snapshots/ContainerTraits.hpp | 4 +-- include/openPMD/snapshots/IteratorTraits.hpp | 29 ++++++++++++------- .../snapshots/RandomAccessIterator.hpp | 13 ++++++--- .../openPMD/snapshots/StatefulIterator.hpp | 4 +-- src/helper/list_series.cpp | 3 ++ src/snapshots/ContainerTraits.cpp | 5 ++++ src/snapshots/IteratorTraits.cpp | 13 ++++----- src/snapshots/RandomAccessIterator.cpp | 18 ++++++++++++ src/snapshots/StatefulIterator.cpp | 6 ++++ 9 files changed, 69 insertions(+), 26 deletions(-) diff --git a/include/openPMD/snapshots/ContainerTraits.hpp b/include/openPMD/snapshots/ContainerTraits.hpp index a6122414fd..a698961742 100644 --- a/include/openPMD/snapshots/ContainerTraits.hpp +++ b/include/openPMD/snapshots/ContainerTraits.hpp @@ -31,10 +31,10 @@ class OpaqueSeriesIterator ~OpaqueSeriesIterator(); // dereference - using parent_t::operator*; using value_type = value_type_in; - value_type const &operator*() const; + auto operator*() -> value_type &; + auto operator*() const -> value_type const &; // increment/decrement OpaqueSeriesIterator &operator++(); diff --git a/include/openPMD/snapshots/IteratorTraits.hpp b/include/openPMD/snapshots/IteratorTraits.hpp index 90c6eb8981..e4ee9a18cc 100644 --- a/include/openPMD/snapshots/IteratorTraits.hpp +++ b/include/openPMD/snapshots/IteratorTraits.hpp @@ -72,17 +72,26 @@ class AbstractSeriesIterator : public DynamicSeriesIterator ~AbstractSeriesIterator() override; + /* + * Default definitions that can be pulled in implementing child classes with + * `using` declarations. Does not work for overloaded methods due to + * compiler bugs in ICPC, hence e.g. `default_increment_operator` instead of + * `operator++`. + */ + // dereference // value_type const &operator*() const = 0; - value_type &operator*(); - value_type const *operator->() const; - value_type *operator->(); + // value_type &operator*() = 0; + auto operator->() const -> value_type const *; + auto operator->() -> value_type *; // increment/decrement // ChildClass &operator++(); // ChildClass &operator--(); - ChildClass operator++(int); - ChildClass operator--(int); + // ChildClass &operator++(int); + // ChildClass &operator--(int); + auto default_increment_operator(int) -> ChildClass; + auto default_decrement_operator(int) -> ChildClass; // comparison // bool operator==(ChildClass const &) const = 0; @@ -96,16 +105,16 @@ class AbstractSeriesIterator : public DynamicSeriesIterator using parent_t = DynamicSeriesIterator; // dereference using parent_t::dereference_operator; - value_type const &dereference_operator() const override; + auto dereference_operator() const -> value_type const & override; // increment/decrement - parent_t &increment_operator() override; - parent_t &decrement_operator() override; + auto increment_operator() -> parent_t & override; + auto decrement_operator() -> parent_t & override; // comparison - bool equality_operator(parent_t const &) const override; + auto equality_operator(parent_t const &) const -> bool override; - std::unique_ptr clone() const override; + auto clone() const -> std::unique_ptr override; private: ChildClass *this_child(); diff --git a/include/openPMD/snapshots/RandomAccessIterator.hpp b/include/openPMD/snapshots/RandomAccessIterator.hpp index a62c1f4ee4..3636d61768 100644 --- a/include/openPMD/snapshots/RandomAccessIterator.hpp +++ b/include/openPMD/snapshots/RandomAccessIterator.hpp @@ -50,7 +50,6 @@ class RandomAccessIterator iterator_t m_it; public: - using parent_t::operator*; using typename parent_t::value_type; ~RandomAccessIterator() override; @@ -64,9 +63,15 @@ class RandomAccessIterator operator=(RandomAccessIterator &&other) noexcept(noexcept( std::declval().operator=(std::declval()))); - value_type const &operator*() const; - RandomAccessIterator &operator++(); - RandomAccessIterator &operator--(); + auto operator*() -> value_type &; + auto operator*() const -> value_type const &; + + auto operator++() -> RandomAccessIterator &; + auto operator--() -> RandomAccessIterator &; + auto operator++(int) -> RandomAccessIterator; + auto operator--(int) -> RandomAccessIterator; + + using parent_t::operator!=; bool operator==(RandomAccessIterator const &other) const; }; } // namespace openPMD diff --git a/include/openPMD/snapshots/StatefulIterator.hpp b/include/openPMD/snapshots/StatefulIterator.hpp index 64c56b2e9b..05f11478a2 100644 --- a/include/openPMD/snapshots/StatefulIterator.hpp +++ b/include/openPMD/snapshots/StatefulIterator.hpp @@ -230,8 +230,8 @@ class StatefulIterator StatefulIterator(tag_write_t, Series const &); // dereference - using parent_t::operator*; - value_type const &operator*() const; + auto operator*() -> value_type &; + auto operator*() const -> value_type const &; // increment/decrement auto operator++() -> StatefulIterator &; diff --git a/src/helper/list_series.cpp b/src/helper/list_series.cpp index 89c3a45139..3e68bac039 100644 --- a/src/helper/list_series.cpp +++ b/src/helper/list_series.cpp @@ -30,6 +30,9 @@ #include #include +inline void breakpoint() +{} + namespace openPMD::helper { std::ostream &listSeries(Series &series, bool const longer, std::ostream &out) diff --git a/src/snapshots/ContainerTraits.cpp b/src/snapshots/ContainerTraits.cpp index 0bf5e85a20..e1957ddc88 100644 --- a/src/snapshots/ContainerTraits.cpp +++ b/src/snapshots/ContainerTraits.cpp @@ -39,6 +39,11 @@ OpaqueSeriesIterator::~OpaqueSeriesIterator() = default; // dereference template +auto OpaqueSeriesIterator::operator*() -> value_type & +{ + return m_internal_iterator->dereference_operator(); +} +template auto OpaqueSeriesIterator::operator*() const -> value_type const & { return m_internal_iterator->dereference_operator(); diff --git a/src/snapshots/IteratorTraits.cpp b/src/snapshots/IteratorTraits.cpp index d45490bd90..b8d18659d1 100644 --- a/src/snapshots/IteratorTraits.cpp +++ b/src/snapshots/IteratorTraits.cpp @@ -3,6 +3,7 @@ #include "openPMD/snapshots/Snapshots.hpp" #include "openPMD/snapshots/StatefulIterator.hpp" #include +#include namespace openPMD { @@ -23,12 +24,6 @@ AbstractSeriesIterator::~AbstractSeriesIterator() = // dereference template -auto AbstractSeriesIterator::operator*() -> value_type & -{ - return const_cast( - static_cast(this)->operator*()); -} -template auto AbstractSeriesIterator::operator->() const -> value_type const * { @@ -43,14 +38,16 @@ auto AbstractSeriesIterator::operator->() // increment/decrement template -ChildClass AbstractSeriesIterator::operator++(int) +ChildClass +AbstractSeriesIterator::default_increment_operator(int) { auto prev = *this_child(); this_child()->operator++(); return prev; } template -ChildClass AbstractSeriesIterator::operator--(int) +ChildClass +AbstractSeriesIterator::default_decrement_operator(int) { auto prev = *this_child(); this_child()->operator--(); diff --git a/src/snapshots/RandomAccessIterator.cpp b/src/snapshots/RandomAccessIterator.cpp index d0248fecff..689c675db7 100644 --- a/src/snapshots/RandomAccessIterator.cpp +++ b/src/snapshots/RandomAccessIterator.cpp @@ -27,6 +27,12 @@ RandomAccessIterator &RandomAccessIterator::operator=( operator=(std::declval()))) = default; +template +auto RandomAccessIterator::operator*() -> value_type & +{ + return *m_it; +} + template auto RandomAccessIterator::operator*() const -> value_type const & { @@ -47,6 +53,18 @@ auto RandomAccessIterator::operator--() -> RandomAccessIterator & return *this; } +template +auto RandomAccessIterator::operator++(int i) -> RandomAccessIterator +{ + return parent_t::default_increment_operator(i); +} + +template +auto RandomAccessIterator::operator--(int i) -> RandomAccessIterator +{ + return parent_t::default_decrement_operator(i); +} + template auto RandomAccessIterator::operator==( RandomAccessIterator const &other) const -> bool diff --git a/src/snapshots/StatefulIterator.cpp b/src/snapshots/StatefulIterator.cpp index 74f24363f1..1a6edb46fe 100644 --- a/src/snapshots/StatefulIterator.cpp +++ b/src/snapshots/StatefulIterator.cpp @@ -875,6 +875,12 @@ auto StatefulIterator::seek(Seek const &seek) -> StatefulIterator * return *res; } +auto StatefulIterator::operator*() -> value_type & +{ + return const_cast( + static_cast(this)->operator*()); +} + auto StatefulIterator::operator*() const -> value_type const & { auto &data = get(); From d815632cadec1719e9b83a6f03195006f760fdb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 5 Mar 2024 18:06:24 +0100 Subject: [PATCH 74/93] noexcept details for MSVC --- include/openPMD/snapshots/ContainerImpls.hpp | 17 ++++++++++++----- src/snapshots/ContainerImpls.cpp | 19 +++++++++++-------- 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/include/openPMD/snapshots/ContainerImpls.hpp b/include/openPMD/snapshots/ContainerImpls.hpp index db168b8ee5..8a3e3f9784 100644 --- a/include/openPMD/snapshots/ContainerImpls.hpp +++ b/include/openPMD/snapshots/ContainerImpls.hpp @@ -11,10 +11,15 @@ class StatefulSnapshotsContainer : public AbstractSnapshotsContainer { private: friend class Series; - std::function m_begin; - StatefulSnapshotsContainer(std::function begin); - std::optional m_bufferedIterator = std::nullopt; + struct Members + { + std::function m_begin; + std::optional m_bufferedIterator = std::nullopt; + }; + Members members; + + StatefulSnapshotsContainer(std::function begin); auto get() -> StatefulIterator *; auto get() const -> StatefulIterator const *; @@ -23,12 +28,14 @@ class StatefulSnapshotsContainer : public AbstractSnapshotsContainer ~StatefulSnapshotsContainer() override; StatefulSnapshotsContainer(StatefulSnapshotsContainer const &other); - StatefulSnapshotsContainer(StatefulSnapshotsContainer &&other) noexcept; + StatefulSnapshotsContainer(StatefulSnapshotsContainer &&other) noexcept( + noexcept(Members(std::declval()))); StatefulSnapshotsContainer & operator=(StatefulSnapshotsContainer const &other); StatefulSnapshotsContainer & - operator=(StatefulSnapshotsContainer &&other) noexcept; + operator=(StatefulSnapshotsContainer &&other) noexcept(noexcept( + std::declval().operator=(std::declval()))); auto currentIteration() -> std::optional override; auto currentIteration() const -> std::optional override; diff --git a/src/snapshots/ContainerImpls.cpp b/src/snapshots/ContainerImpls.cpp index 6a11133f2b..2167e65feb 100644 --- a/src/snapshots/ContainerImpls.cpp +++ b/src/snapshots/ContainerImpls.cpp @@ -25,29 +25,32 @@ namespace StatefulSnapshotsContainer::StatefulSnapshotsContainer( std::function begin) - : m_begin(std::move(begin)) + : members{std::move(begin)} {} StatefulSnapshotsContainer::StatefulSnapshotsContainer( StatefulSnapshotsContainer const &other) = default; StatefulSnapshotsContainer::StatefulSnapshotsContainer( - StatefulSnapshotsContainer &&other) noexcept = default; + StatefulSnapshotsContainer + &&other) noexcept(noexcept(Members(std::declval()))) = + default; StatefulSnapshotsContainer &StatefulSnapshotsContainer::operator=( StatefulSnapshotsContainer const &other) = default; -StatefulSnapshotsContainer &StatefulSnapshotsContainer::operator=( - StatefulSnapshotsContainer &&other) noexcept = default; +StatefulSnapshotsContainer &StatefulSnapshotsContainer:: +operator=(StatefulSnapshotsContainer &&other) noexcept(noexcept( + std::declval().operator=(std::declval()))) = default; auto StatefulSnapshotsContainer::get() -> StatefulIterator * { - if (!m_bufferedIterator.has_value()) + if (!members.m_bufferedIterator.has_value()) { - m_bufferedIterator = m_begin(); + members.m_bufferedIterator = members.m_begin(); } - return *m_bufferedIterator; + return *members.m_bufferedIterator; } auto StatefulSnapshotsContainer::get() const -> StatefulIterator const * { - return m_bufferedIterator.value_or(nullptr); + return members.m_bufferedIterator.value_or(nullptr); } auto StatefulSnapshotsContainer::currentIteration() -> std::optional From 8e9248a7848ba9f1b390ca2e212c828e07da3111 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 6 Mar 2024 11:16:56 +0100 Subject: [PATCH 75/93] Fix ulimit test --- include/openPMD/Iteration.hpp | 9 ++++++++- src/Iteration.cpp | 14 ++++++++++++++ src/helper/list_series.cpp | 9 ++++++++- test/SerialIOTest.cpp | 14 ++++++++++++++ 4 files changed, 44 insertions(+), 2 deletions(-) diff --git a/include/openPMD/Iteration.hpp b/include/openPMD/Iteration.hpp index 24078b5e86..468f524e7f 100644 --- a/include/openPMD/Iteration.hpp +++ b/include/openPMD/Iteration.hpp @@ -207,12 +207,19 @@ class Iteration : public Attributable /** * @brief Has the iteration been closed? - * A closed iteration may not (yet) be reopened. * * @return Whether the iteration has been closed. */ bool closed() const; + /** + * @brief Has the iteration been parsed yet? + If not, it will contain no structure yet. + * + * @return Whether the iteration has been parsed. + */ + bool parsed() const; + /** * @brief Has the iteration been closed by the writer? * Background: Upon calling Iteration::close(), the openPMD API diff --git a/src/Iteration.cpp b/src/Iteration.cpp index 74c4e9314b..b39db3279a 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -180,6 +180,20 @@ bool Iteration::closed() const throw std::runtime_error("Unreachable!"); } +bool Iteration::parsed() const +{ + switch (get().m_closed) + { + case CloseStatus::ParseAccessDeferred: + return false; + case CloseStatus::Open: + case CloseStatus::ClosedInFrontend: + case CloseStatus::Closed: + return true; + } + throw std::runtime_error("Unreachable!"); +} + bool Iteration::closedByWriter() const { using bool_type = unsigned char; diff --git a/src/helper/list_series.cpp b/src/helper/list_series.cpp index 3e68bac039..7782573b11 100644 --- a/src/helper/list_series.cpp +++ b/src/helper/list_series.cpp @@ -118,7 +118,10 @@ std::ostream &listSeries(Series &series, bool const longer, std::ostream &out) for (auto &[index, i] : series.snapshots()) { - i.open(); + if (!i.parsed()) + { + i.open(); + } if (longer) out << index << " "; @@ -135,6 +138,10 @@ std::ostream &listSeries(Series &series, bool const longer, std::ostream &out) [](std::pair const &p) { return p.first; }); + if (!i.closed()) + { + i.close(); + } } if (longer) diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index a5909a09fd..f299231222 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -38,6 +38,7 @@ #ifdef __unix__ #include #include +#include #include #include #include @@ -160,6 +161,13 @@ TEST_CASE("char_roundtrip", "[serial]") void write_and_read_many_iterations( std::string const &ext, bool intermittentFlushes) { +#ifdef __unix__ + struct rlimit rlim; + getrlimit(RLIMIT_NOFILE, &rlim); + auto old_soft_limit = rlim.rlim_cur; + rlim.rlim_cur = 512; + setrlimit(RLIMIT_NOFILE, &rlim); +#endif // the idea here is to trigger the maximum allowed number of file handles, // e.g., the upper limit in "ulimit -n" (default: often 1024). Once this // is reached, files should be closed automatically for open iterations @@ -172,6 +180,7 @@ void write_and_read_many_iterations( auxiliary::getEnvNum("OPENPMD_TEST_NFILES_MAX", 1030); std::string filename = "../samples/many_iterations/many_iterations_%T." + ext; + // std::cout << "WRITE " << filename << std::endl; std::vector data(10); std::iota(data.begin(), data.end(), 0.); @@ -194,6 +203,7 @@ void write_and_read_many_iterations( } // ~Series intentionally not yet called + // std::cout << "READ " << filename << std::endl; Series read( filename, Access::READ_ONLY, "{\"defer_iteration_parsing\": true}"); for (auto iteration : read.iterations) @@ -219,6 +229,10 @@ void write_and_read_many_iterations( Series list(filename, Access::READ_ONLY); helper::listSeries(list); +#ifdef __unix__ + rlim.rlim_cur = old_soft_limit; + setrlimit(RLIMIT_NOFILE, &rlim); +#endif } TEST_CASE("write_and_read_many_iterations", "[serial]") From 5fa05255ba43335624c344e7717aed274a66ffbc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 26 Mar 2024 14:42:37 +0100 Subject: [PATCH 76/93] Fix after rebase: dirtyRecursive --- src/Iteration.cpp | 2 +- src/Series.cpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Iteration.cpp b/src/Iteration.cpp index b39db3279a..4f06adbb8d 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -338,7 +338,7 @@ void Iteration::flush(internal::FlushParams const &flushParams) m.second.flush(m.first, flushParams); for (auto &species : particles) species.second.flush(species.first, flushParams); - dirty() = false; + setDirty(false); } else { diff --git a/src/Series.cpp b/src/Series.cpp index 19bbb688d6..0695180fea 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -2414,7 +2414,7 @@ AdvanceStatus Series::advance( } bool old_dirty = iteration.dirty(); - iteration.dirty() = true; // force flush() to open this + iteration.setDirty(true); // force flush() to open this switch (mode) { @@ -2432,7 +2432,7 @@ AdvanceStatus Series::advance( end, {FlushLevel::CreateOrOpenFiles}, /* flushIOHandler = */ false); - iteration.dirty() = old_dirty; + iteration.setDirty(old_dirty); break; } From 0374f315eff5dba70b2a1d1fbc0a37001a2809d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 26 Mar 2024 17:42:38 +0100 Subject: [PATCH 77/93] Fixes after rebase --- src/Series.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Series.cpp b/src/Series.cpp index 0695180fea..6c7c10568e 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -2300,7 +2300,7 @@ void Series::readBase() { // Make sure that the meshesPath does not leak from one iteration into // the other in file-based iteration encoding - get().m_attributes.erase("meshesPath"); + Attributable::get().m_attributes.erase("meshesPath"); } if (std::count( @@ -2337,7 +2337,7 @@ void Series::readBase() { // Make sure that the particlesPath does not leak from one iteration // into the other in file-based iteration encoding - get().m_attributes.erase("particlesPath"); + Attributable::get().m_attributes.erase("particlesPath"); } } From 32040acc7775208c1dbbd1d7949732294b0294ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 7 Jun 2024 15:10:41 +0200 Subject: [PATCH 78/93] remove conflict markers... --- include/openPMD/Series.hpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/include/openPMD/Series.hpp b/include/openPMD/Series.hpp index d3965ddd4d..0150022f3a 100644 --- a/include/openPMD/Series.hpp +++ b/include/openPMD/Series.hpp @@ -49,11 +49,8 @@ #include #include #include -<<<<<<< HEAD -#include -======= #include ->>>>>>> 881f29767 (Further test and implement reopening of Iterations) +#include #include // expose private and protected members for invasive testing From 7a58e281d59484ce0a269c78e2eff10e93da76dd Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 7 Jun 2024 13:11:08 +0000 Subject: [PATCH 79/93] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- include/openPMD/snapshots/StatefulIterator.hpp | 8 ++++---- src/Iteration.cpp | 5 +++-- src/Series.cpp | 8 ++++---- src/snapshots/StatefulIterator.cpp | 8 ++++---- 4 files changed, 15 insertions(+), 14 deletions(-) diff --git a/include/openPMD/snapshots/StatefulIterator.hpp b/include/openPMD/snapshots/StatefulIterator.hpp index 05f11478a2..2a51b38b60 100644 --- a/include/openPMD/snapshots/StatefulIterator.hpp +++ b/include/openPMD/snapshots/StatefulIterator.hpp @@ -91,8 +91,8 @@ namespace detail auto get_iteration_index() const -> std::optional; - auto get_iteration_index() - -> std::optional; + auto + get_iteration_index() -> std::optional; enum class AtTheEdge : bool { @@ -298,8 +298,8 @@ class StatefulIterator auto resetCurrentIterationToBegin( size_t num_skipped_iterations, std::vector current_iterations) -> void; - auto peekCurrentlyOpenIteration() const - -> std::optional; + auto + peekCurrentlyOpenIteration() const -> std::optional; auto peekCurrentlyOpenIteration() -> std::optional; auto reparse_possibly_deleted_iteration(iteration_index_t) -> void; diff --git a/src/Iteration.cpp b/src/Iteration.cpp index 4f06adbb8d..d8878f7fdc 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -718,8 +718,9 @@ auto Iteration::beginStep(bool reread) -> BeginStepStatus } auto Iteration::beginStep( - std::optional thisObject, Series &series, bool reread) - -> BeginStepStatus + std::optional thisObject, + Series &series, + bool reread) -> BeginStepStatus { BeginStepStatus res; using IE = IterationEncoding; diff --git a/src/Series.cpp b/src/Series.cpp index 6c7c10568e..a0f0896362 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -3015,8 +3015,8 @@ namespace namespace { auto make_writing_stateful_iterator( - Series const &copied_series, internal::SeriesData &series) - -> std::function + Series const &copied_series, + internal::SeriesData &series) -> std::function { if (!series.m_sharedReadIterations) { @@ -3026,8 +3026,8 @@ namespace return [ptr = series.m_sharedReadIterations.get()]() { return ptr; }; } auto make_reading_stateful_iterator( - Series copied_series, internal::SeriesData &series) - -> std::function + Series copied_series, + internal::SeriesData &series) -> std::function { return [s = std::move(copied_series), &series]() mutable { if (!series.m_sharedReadIterations) diff --git a/src/snapshots/StatefulIterator.cpp b/src/snapshots/StatefulIterator.cpp index 1a6edb46fe..60c284105f 100644 --- a/src/snapshots/StatefulIterator.cpp +++ b/src/snapshots/StatefulIterator.cpp @@ -65,8 +65,8 @@ namespace detail } template - [[nodiscard]] auto CurrentStep::get_variant() const - -> std::optional + [[nodiscard]] auto + CurrentStep::get_variant() const -> std::optional { auto res = std::get_if(*this); if (res) @@ -324,8 +324,8 @@ namespace } // namespace auto StatefulIterator::resetCurrentIterationToBegin( - size_t num_skipped_iterations, std::vector indexes) - -> void + size_t num_skipped_iterations, + std::vector indexes) -> void { auto &data = get(); data.currentStep.map_during_t( From 684168fff1b845983cd2b4ad33d617312c0c603a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 10 Jun 2024 11:23:45 +0200 Subject: [PATCH 80/93] Better defaults? --- examples/10_streaming_write.cpp | 2 +- src/Series.cpp | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/examples/10_streaming_write.cpp b/examples/10_streaming_write.cpp index 5704638403..4856fd8bfd 100644 --- a/examples/10_streaming_write.cpp +++ b/examples/10_streaming_write.cpp @@ -43,7 +43,7 @@ int main() // in streaming setups, e.g. an iteration cannot be opened again once // it has been closed. // `Series::iterations` can be directly accessed in random-access workflows. - auto iterations = series.snapshots(); + auto iterations = series.writeIterations(); for (size_t i = 0; i < 100; ++i) { Iteration iteration = iterations[i]; diff --git a/src/Series.cpp b/src/Series.cpp index a0f0896362..b2a902583b 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -3088,13 +3088,13 @@ Snapshots Series::snapshots() } break; + // Our Read-Write workflows are entirely random-access based (so + // far). case Access::READ_WRITE: - throw std::runtime_error( - "[Series::snapshots()] Unimplemented for READ_WRITE mode."); + // Stateful Iteration accessed via Series::writeIterations(). case Access::CREATE: case Access::APPEND: - // @todo: properly distinguish here - iterator_kind = IK::Stateful; + iterator_kind = IK::RandomAccess; break; } } @@ -3108,7 +3108,6 @@ Snapshots Series::snapshots() case IteratorKind::Stateful: { std::function begin; - // @todo: distinguish read/write access if (access::write(IOHandler()->m_frontendAccess)) { begin = make_writing_stateful_iterator(*this, series); From 467b90bc33c4d55398edb9b94e8dfa7e16f75aac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 10 Jun 2024 15:56:51 +0200 Subject: [PATCH 81/93] Parameterize Series::snapshots() --- CMakeLists.txt | 1 + examples/10_streaming_write.cpp | 2 +- include/openPMD/IO/Access.hpp | 2 + include/openPMD/Series.hpp | 3 +- include/openPMD/snapshots/ContainerImpls.hpp | 5 -- src/IO/Access.cpp | 27 ++++++++ src/Series.cpp | 71 +++++++++++++++++--- 7 files changed, 93 insertions(+), 18 deletions(-) create mode 100644 src/IO/Access.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index a4b80070ed..4230bfa44a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -485,6 +485,7 @@ set(IO_SOURCE src/IO/AbstractIOHandler.cpp src/IO/AbstractIOHandlerImpl.cpp src/IO/AbstractIOHandlerHelper.cpp + src/IO/Access.cpp src/IO/DummyIOHandler.cpp src/IO/IOTask.cpp src/IO/FlushParams.cpp diff --git a/examples/10_streaming_write.cpp b/examples/10_streaming_write.cpp index 4856fd8bfd..304a36c025 100644 --- a/examples/10_streaming_write.cpp +++ b/examples/10_streaming_write.cpp @@ -43,7 +43,7 @@ int main() // in streaming setups, e.g. an iteration cannot be opened again once // it has been closed. // `Series::iterations` can be directly accessed in random-access workflows. - auto iterations = series.writeIterations(); + auto iterations = series.snapshots(/* access_synchronously = */ true); for (size_t i = 0; i < 100; ++i) { Iteration iteration = iterations[i]; diff --git a/include/openPMD/IO/Access.hpp b/include/openPMD/IO/Access.hpp index eddd9f40e9..a5a2a84ae0 100644 --- a/include/openPMD/IO/Access.hpp +++ b/include/openPMD/IO/Access.hpp @@ -82,6 +82,8 @@ enum class Access APPEND //!< write new iterations to an existing series without reading }; // Access +std::ostream &operator<<(std::ostream &o, Access const &a); + namespace access { inline bool readOnly(Access access) diff --git a/include/openPMD/Series.hpp b/include/openPMD/Series.hpp index 0150022f3a..15be1be276 100644 --- a/include/openPMD/Series.hpp +++ b/include/openPMD/Series.hpp @@ -640,7 +640,8 @@ class Series : public Attributable */ ReadIterations readIterations(); - Snapshots snapshots(); + Snapshots + snapshots(std::optional access_synchronously = std::nullopt); /** * @brief Parse the Series. diff --git a/include/openPMD/snapshots/ContainerImpls.hpp b/include/openPMD/snapshots/ContainerImpls.hpp index 8a3e3f9784..04d8e874ba 100644 --- a/include/openPMD/snapshots/ContainerImpls.hpp +++ b/include/openPMD/snapshots/ContainerImpls.hpp @@ -65,11 +65,6 @@ class StatefulSnapshotsContainer : public AbstractSnapshotsContainer auto contains(key_type const &key) const -> bool override; }; -/* - * @todo how to deal with iteration::open() iteration::close() ? - * -> have it guaranteed with READ_LINEAR, not with READ_RANDOM_ACCESS, - * but need to see how to deal with write modes - */ class RandomAccessIteratorContainer : public AbstractSnapshotsContainer { private: diff --git a/src/IO/Access.cpp b/src/IO/Access.cpp new file mode 100644 index 0000000000..86f2cf9ee9 --- /dev/null +++ b/src/IO/Access.cpp @@ -0,0 +1,27 @@ +#include "openPMD/IO/Access.hpp" + +namespace openPMD +{ +std::ostream &operator<<(std::ostream &o, Access const &a) +{ + switch (a) + { + case Access::READ_RANDOM_ACCESS: + o << std::string("READ_RANDOM_ACCESS"); + break; + case Access::READ_LINEAR: + o << std::string("READ_LINEAR"); + break; + case Access::READ_WRITE: + o << std::string("READ_WRITE"); + break; + case Access::CREATE: + o << std::string("CREATE"); + break; + case Access::APPEND: + o << std::string("APPEND"); + break; + } + return o; +} +} // namespace openPMD diff --git a/src/Series.cpp b/src/Series.cpp index b2a902583b..520d90a78f 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -3044,22 +3044,56 @@ namespace } } // namespace -Snapshots Series::snapshots() +Snapshots Series::snapshots(std::optional const access_synchronously) { auto &series = get(); if (series.m_deferred_initialization.has_value()) { runDeferredInitialization(); } + auto access = IOHandler()->m_frontendAccess; + auto guard_wrong_access_specification = + [&](bool access_must_be_synchronous) { + if (!access_synchronously.has_value()) + { + return; + } + if (access_must_be_synchronous != *access_synchronously) + { + std::stringstream error; + error << "[Series::snapshots()] Specified " + << (*access_synchronously ? "synchronous" + : "non-synchronous") + << " iteration in method parameter " + "`access_synchronously`, but access type " + << access << " requires " + << (access_must_be_synchronous ? "synchronous" + : "non-synchronous") + << " iteration. Please remove the parameter, there is no " + "need to specify it under " + << access << " mode." << std::endl; + throw error::WrongAPIUsage(error.str()); + } + else + { + std::cerr + << "[Series::snapshots()] No need to explicitly specify " + "synchronous or non-synchronous access via method " + "parameter `access_synchronously` in mode '" + << access << ". Will ignore." << std::endl; + } + }; IteratorKind iterator_kind{}; { using IK = IteratorKind; - switch (IOHandler()->m_frontendAccess) + switch (access) { case Access::READ_LINEAR: + guard_wrong_access_specification(true); iterator_kind = IK::Stateful; break; case Access::READ_ONLY: + guard_wrong_access_specification(false); if (series.m_parsePreference.has_value()) { switch (series.m_parsePreference.value()) @@ -3068,12 +3102,12 @@ Snapshots Series::snapshots() iterator_kind = IK::RandomAccess; break; case internal::ParsePreference::PerStep: - std::cerr - << "[Warning] Series: Use READ_LINEAR access mode to " - "access Series that requires collective processing." - << std::endl; - iterator_kind = IK::Stateful; - break; + throw error::ReadError( + error::AffectedObject::File, + error::Reason::UnexpectedContent, + std::nullopt, + "[Series::snapshots()] Series requires collective " + "processing with READ_LINEAR access mode."); } } else if (iterationEncoding() != IterationEncoding::fileBased) @@ -3088,13 +3122,28 @@ Snapshots Series::snapshots() } break; + case Access::READ_WRITE: // Our Read-Write workflows are entirely random-access based (so // far). - case Access::READ_WRITE: - // Stateful Iteration accessed via Series::writeIterations(). + // (Might be possible to allow stateful access actually, but there's + // no real use, so keep it simple.) + guard_wrong_access_specification(false); + iterator_kind = IK::RandomAccess; + break; + case Access::CREATE: case Access::APPEND: - iterator_kind = IK::RandomAccess; + // Users can select. + if (access_synchronously.value_or( + /* random-access logic by default */ + false)) + { + iterator_kind = IK::Stateful; + } + else + { + iterator_kind = IK::RandomAccess; + } break; } } From 86d6ed38f3532f548e37e04f6dcb4ace3e73d41c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 10 Jun 2024 16:19:27 +0200 Subject: [PATCH 82/93] Use enum class for last commit --- examples/10_streaming_write.cpp | 4 +- include/openPMD/Series.hpp | 11 +++++- src/Series.cpp | 70 +++++++++++++-------------------- 3 files changed, 39 insertions(+), 46 deletions(-) diff --git a/examples/10_streaming_write.cpp b/examples/10_streaming_write.cpp index 304a36c025..5d5fbfdb15 100644 --- a/examples/10_streaming_write.cpp +++ b/examples/10_streaming_write.cpp @@ -1,3 +1,5 @@ +#include "openPMD/Series.hpp" +#include "openPMD/snapshots/Snapshots.hpp" #include #include @@ -43,7 +45,7 @@ int main() // in streaming setups, e.g. an iteration cannot be opened again once // it has been closed. // `Series::iterations` can be directly accessed in random-access workflows. - auto iterations = series.snapshots(/* access_synchronously = */ true); + auto iterations = series.snapshots(SnapshotAccess::Linear); for (size_t i = 0; i < 100; ++i) { Iteration iteration = iterations[i]; diff --git a/include/openPMD/Series.hpp b/include/openPMD/Series.hpp index 15be1be276..1c951401cd 100644 --- a/include/openPMD/Series.hpp +++ b/include/openPMD/Series.hpp @@ -640,8 +640,13 @@ class Series : public Attributable */ ReadIterations readIterations(); - Snapshots - snapshots(std::optional access_synchronously = std::nullopt); + enum class SnapshotAccess + { + RandomAccess, + Linear + }; + Snapshots snapshots( + std::optional access_synchronously = std::nullopt); /** * @brief Parse the Series. @@ -894,6 +899,8 @@ OPENPMD_private AbstractIOHandler const *IOHandler() const; }; // Series +using SnapshotAccess = Series::SnapshotAccess; + namespace debug { void printDirty(Series const &); diff --git a/src/Series.cpp b/src/Series.cpp index 520d90a78f..f3ddaa0b3b 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -3003,15 +3003,6 @@ ReadIterations Series::readIterations() std::move(res), IOHandler()->m_frontendAccess, get().m_parsePreference}; } -namespace -{ - enum class IteratorKind - { - RandomAccess, - Stateful - }; -} - namespace { auto make_writing_stateful_iterator( @@ -3044,7 +3035,8 @@ namespace } } // namespace -Snapshots Series::snapshots(std::optional const access_synchronously) +Snapshots +Series::snapshots(std::optional const access_synchronously) { auto &series = get(); if (series.m_deferred_initialization.has_value()) @@ -3053,22 +3045,24 @@ Snapshots Series::snapshots(std::optional const access_synchronously) } auto access = IOHandler()->m_frontendAccess; auto guard_wrong_access_specification = - [&](bool access_must_be_synchronous) { + [&](SnapshotAccess required_access) { if (!access_synchronously.has_value()) { - return; + return required_access; } - if (access_must_be_synchronous != *access_synchronously) + if (required_access != *access_synchronously) { std::stringstream error; error << "[Series::snapshots()] Specified " - << (*access_synchronously ? "synchronous" - : "non-synchronous") + << (*access_synchronously == SnapshotAccess::Linear + ? "linear" + : "random-access") << " iteration in method parameter " "`access_synchronously`, but access type " << access << " requires " - << (access_must_be_synchronous ? "synchronous" - : "non-synchronous") + << (required_access == SnapshotAccess::Linear + ? "linear" + : "random-access") << " iteration. Please remove the parameter, there is no " "need to specify it under " << access << " mode." << std::endl; @@ -3082,24 +3076,26 @@ Snapshots Series::snapshots(std::optional const access_synchronously) "parameter `access_synchronously` in mode '" << access << ". Will ignore." << std::endl; } + return required_access; }; - IteratorKind iterator_kind{}; + SnapshotAccess usedSnapshotAccess{}; { - using IK = IteratorKind; switch (access) { case Access::READ_LINEAR: - guard_wrong_access_specification(true); - iterator_kind = IK::Stateful; + usedSnapshotAccess = + guard_wrong_access_specification(SnapshotAccess::Linear); break; case Access::READ_ONLY: - guard_wrong_access_specification(false); + usedSnapshotAccess = + guard_wrong_access_specification(SnapshotAccess::RandomAccess); + + // Some error checks if (series.m_parsePreference.has_value()) { switch (series.m_parsePreference.value()) { case internal::ParsePreference::UpFront: - iterator_kind = IK::RandomAccess; break; case internal::ParsePreference::PerStep: throw error::ReadError( @@ -3116,45 +3112,33 @@ Snapshots Series::snapshots(std::optional const access_synchronously) "READ_ONLY mode and non-fileBased iteration encoding, but " "the backend did not set a parse preference."); } - else - { - iterator_kind = IK::RandomAccess; - } - break; case Access::READ_WRITE: // Our Read-Write workflows are entirely random-access based (so // far). // (Might be possible to allow stateful access actually, but there's // no real use, so keep it simple.) - guard_wrong_access_specification(false); - iterator_kind = IK::RandomAccess; + usedSnapshotAccess = + guard_wrong_access_specification(SnapshotAccess::RandomAccess); break; case Access::CREATE: case Access::APPEND: // Users can select. - if (access_synchronously.value_or( - /* random-access logic by default */ - false)) - { - iterator_kind = IK::Stateful; - } - else - { - iterator_kind = IK::RandomAccess; - } + usedSnapshotAccess = access_synchronously.value_or( + /* random-access logic by default */ + SnapshotAccess::RandomAccess); break; } } - switch (iterator_kind) + switch (usedSnapshotAccess) { - case IteratorKind::RandomAccess: { + case SnapshotAccess::RandomAccess: { return Snapshots(std::shared_ptr{ new RandomAccessIteratorContainer(series.iterations)}); } - case IteratorKind::Stateful: { + case SnapshotAccess::Linear: { std::function begin; if (access::write(IOHandler()->m_frontendAccess)) From 19e97f52be6246b76d51620744578015f4a5ba93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 10 Jun 2024 17:15:19 +0200 Subject: [PATCH 83/93] Add some missing minor function implementations --- src/snapshots/ContainerImpls.cpp | 62 +++++++++++++++++++++++--------- 1 file changed, 45 insertions(+), 17 deletions(-) diff --git a/src/snapshots/ContainerImpls.cpp b/src/snapshots/ContainerImpls.cpp index 2167e65feb..9e129f6d8d 100644 --- a/src/snapshots/ContainerImpls.cpp +++ b/src/snapshots/ContainerImpls.cpp @@ -137,21 +137,41 @@ bool StatefulSnapshotsContainer::empty() const } auto StatefulSnapshotsContainer::size() const -> size_t { - throw std::runtime_error("Unimplemented"); + /* + * This should return the sum over all IO steps, counting the number of + * snapshots contained in each step. This information should be tracked in + * future in order to have knowledge on where to find an Iteration once it + * was seen. + */ + throw std::runtime_error( + "[StatefulSnapshotsContainer::size()] Unimplemented"); } -auto StatefulSnapshotsContainer::at(key_type const &) const +auto StatefulSnapshotsContainer::at(key_type const &key) const -> mapped_type const & { - throw std::runtime_error( - "Item access not (yet) implemented on a stateful " - "container/iterator."); + auto it = get(); + auto current_iteration = it->peekCurrentlyOpenIteration(); + if (!current_iteration.has_value() || (*current_iteration)->first != key) + { + throw std::out_of_range( + "[StatefulSnapshotsContainer::at()] Cannot skip to a Snapshot that " + "is currently not active in a const context."); + } + return (*current_iteration)->second; } -auto StatefulSnapshotsContainer::at(key_type const &) -> mapped_type & +auto StatefulSnapshotsContainer::at(key_type const &key) -> mapped_type & { - throw std::runtime_error( - "Item access not (yet) implemented on a stateful " - "container/iterator."); + auto base_iterator = get(); + auto result = + base_iterator->seek({StatefulIterator::Seek::Seek_Iteration_t{key}}); + if (result->is_end()) + { + throw std::out_of_range( + "[StatefulSnapshotsContainer::at()] Cannot (yet) skip to " + "a Snapshot from an I/O step that is not active."); + } + return (*result)->second; } auto StatefulSnapshotsContainer::operator[](key_type const &key) @@ -167,12 +187,9 @@ auto StatefulSnapshotsContainer::operator[](key_type const &key) auto &s = shared->value(); auto access = s.series.IOHandler()->m_frontendAccess; - // @todo distinguish read_write if (access == Access::READ_WRITE) { - throw std::runtime_error( - "[StatefulSnapshotsContainer::operator[]()] Unimplemented for " - "READ_WRITE mode."); + throw std::runtime_error("Stateful iteration on a read-write Series."); } if (access::write(access)) { @@ -237,6 +254,12 @@ auto StatefulSnapshotsContainer::operator[](key_type const &key) { auto result = base_iterator->seek( {StatefulIterator::Seek::Seek_Iteration_t{key}}); + if (result->is_end()) + { + throw std::out_of_range( + "[StatefulSnapshotsContainer::operator[]()] Cannot (yet) skip " + "to a Snapshot from an I/O step that is not active."); + } return (*result)->second; } throw error::Internal("Control flow error: This should be unreachable."); @@ -244,18 +267,23 @@ auto StatefulSnapshotsContainer::operator[](key_type const &key) auto StatefulSnapshotsContainer::clear() -> void { - throw std::runtime_error("Unimplemented"); + throw std::runtime_error( + "[StatefulSnapshotsContainer::clear()] Unimplemented"); } auto StatefulSnapshotsContainer::find(key_type const &) -> iterator { - throw std::runtime_error("Unimplemented"); + throw error::WrongAPIUsage( + "[StatefulSnapshotsContainer::find] `find()` not available in stateful " + "iteration as there is only one shared iterator per Series and " + "`find()` would need to modify that."); } auto StatefulSnapshotsContainer::find(key_type const &) const -> const_iterator { throw error::WrongAPIUsage( - "[StatefulSnapshotsContainer::find] Const iteration not possible on a " - "stateful container/iterator."); + "[StatefulSnapshotsContainer::find] `find()` not available in stateful " + "iteration as there is only one shared iterator per Series and " + "`find()` would need to modify that."); } auto StatefulSnapshotsContainer::contains(key_type const &) const -> bool From ab81b91cfcce7e1682f97d66069288c1aa1f397c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 24 Jun 2024 11:06:51 +0200 Subject: [PATCH 84/93] Don't use globbing --- CMakeLists.txt | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4230bfa44a..07584778ba 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -25,6 +25,12 @@ if(POLICY CMP0127) cmake_policy(SET CMP0127 NEW) endif() +# return() should check its arguments +# https://cmake.org/cmake/help/latest/policy/CMP0140.html +if(POLICY CMP0140) + cmake_policy(SET CMP0140 NEW) +endif() + # No in-Source builds ######################################################### # @@ -853,8 +859,19 @@ if(openPMD_BUILD_TESTING) target_compile_definitions(CatchRunner PUBLIC openPMD_HAVE_MPI=1) endif() + function(additional_testing_sources test_name out_list) + if(${test_name} STREQUAL "SerialIO") + list(APPEND ${out_list} + test/Files_SerialIO/close_iteration_test.cpp + test/Files_SerialIO/filebased_write_test.cpp + ) + endif() + return(PROPAGATE ${out_list}) + endfunction() + foreach(testname ${openPMD_TEST_NAMES}) - file(GLOB ADDITIONAL_SOURCE_FILES test/Files_${testname}/*.cpp) + set(ADDITIONAL_SOURCE_FILES "") + additional_testing_sources(${testname} ADDITIONAL_SOURCE_FILES) add_executable(${testname}Tests test/${testname}Test.cpp ${ADDITIONAL_SOURCE_FILES}) target_include_directories(${testname}Tests PRIVATE test/Files_${testname}/) openpmd_cxx_required(${testname}Tests) From cc52cd11e0bf7878982c0edbe0102304f265b26f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 1 Jul 2024 17:48:45 +0200 Subject: [PATCH 85/93] Add missing include --- src/IO/Access.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/IO/Access.cpp b/src/IO/Access.cpp index 86f2cf9ee9..1112cee69c 100644 --- a/src/IO/Access.cpp +++ b/src/IO/Access.cpp @@ -1,5 +1,8 @@ #include "openPMD/IO/Access.hpp" +#include +#include + namespace openPMD { std::ostream &operator<<(std::ostream &o, Access const &a) From ffd802d9482dd2a60adcdc5f39c11b473dbaf8e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 4 Jul 2024 17:57:32 +0200 Subject: [PATCH 86/93] Better include structure, put Legacy stuff to Legacy headers --- CMakeLists.txt | 2 + include/openPMD/ReadIterations.hpp | 66 +++++++++++++++++++ include/openPMD/Series.hpp | 6 +- include/openPMD/snapshots/IteratorHelpers.hpp | 12 ++++ include/openPMD/snapshots/Snapshots.hpp | 1 - .../openPMD/snapshots/StatefulIterator.hpp | 55 ---------------- src/ReadIterations.cpp | 64 ++++++++++++++++++ src/binding/python/Series.cpp | 4 +- src/snapshots/ContainerImpls.cpp | 15 +---- src/snapshots/IteratorHelpers.cpp | 15 +++++ src/snapshots/StatefulIterator.cpp | 63 ------------------ 11 files changed, 165 insertions(+), 138 deletions(-) create mode 100644 include/openPMD/ReadIterations.hpp create mode 100644 include/openPMD/snapshots/IteratorHelpers.hpp create mode 100644 src/ReadIterations.cpp create mode 100644 src/snapshots/IteratorHelpers.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 07584778ba..46fb4d95ff 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -466,6 +466,7 @@ set(CORE_SOURCE src/ParticlePatches.cpp src/ParticleSpecies.cpp src/Record.cpp + src/ReadIterations.cpp src/RecordComponent.cpp src/Series.cpp src/version.cpp @@ -483,6 +484,7 @@ set(CORE_SOURCE src/helper/list_series.cpp src/snapshots/ContainerImpls.cpp src/snapshots/ContainerTraits.cpp + src/snapshots/IteratorHelpers.cpp src/snapshots/IteratorTraits.cpp src/snapshots/RandomAccessIterator.cpp src/snapshots/Snapshots.cpp diff --git a/include/openPMD/ReadIterations.hpp b/include/openPMD/ReadIterations.hpp new file mode 100644 index 0000000000..e6903b9704 --- /dev/null +++ b/include/openPMD/ReadIterations.hpp @@ -0,0 +1,66 @@ +#pragma once + +/* + * Legacy header. + */ + +#include "openPMD/Iteration.hpp" +#include "openPMD/Series.hpp" +#include "openPMD/snapshots/Snapshots.hpp" + +namespace openPMD +{ +class LegacyIteratorAdaptor +{ + using value_type = IndexedIteration; + using parent_t = Snapshots::iterator; + +private: + friend class ReadIterations; + Snapshots::iterator m_iterator; + LegacyIteratorAdaptor(Snapshots::iterator iterator); + +public: + value_type operator*() const; + LegacyIteratorAdaptor &operator++(); + bool operator==(LegacyIteratorAdaptor const &other) const; + bool operator!=(LegacyIteratorAdaptor const &other) const; +}; + +/** + * @brief Reading side of the streaming API. + * + * Create instance via Series::readIterations(). + * For use in a C++11-style foreach loop over iterations. + * Designed to allow reading any kind of Series, streaming and non- + * streaming alike. + * Calling Iteration::close() manually before opening the next iteration is + * encouraged and will implicitly flush all deferred IO actions. + * Otherwise, Iteration::close() will be implicitly called upon + * StatefulIterator::operator++(), i.e. upon going to the next iteration in + * the foreach loop. + * Since this is designed for streaming mode, reopening an iteration is + * not possible once it has been closed. + * + */ +class ReadIterations +{ + friend class Series; + +private: + using iterations_t = decltype(internal::SeriesData::iterations); + using iterator_t = LegacyIteratorAdaptor; + + Series m_series; + std::optional m_parsePreference; + + ReadIterations( + Series, + Access, + std::optional parsePreference); + +public: + auto begin() -> iterator_t; + static auto end() -> iterator_t; +}; +} // namespace openPMD diff --git a/include/openPMD/Series.hpp b/include/openPMD/Series.hpp index 1c951401cd..8057fc9ca5 100644 --- a/include/openPMD/Series.hpp +++ b/include/openPMD/Series.hpp @@ -907,6 +907,6 @@ namespace debug } } // namespace openPMD -// Make sure that this one is always included if Series.hpp is included, -// otherwise Series::readIterations() cannot be used -#include "openPMD/snapshots/StatefulIterator.hpp" +// Make sure that this legacy header is always included if Series.hpp is +// included, otherwise Series::readIterations() cannot be used +#include "openPMD/ReadIterations.hpp" diff --git a/include/openPMD/snapshots/IteratorHelpers.hpp b/include/openPMD/snapshots/IteratorHelpers.hpp new file mode 100644 index 0000000000..cc9c737c78 --- /dev/null +++ b/include/openPMD/snapshots/IteratorHelpers.hpp @@ -0,0 +1,12 @@ +#pragma once + +#include "openPMD/Iteration.hpp" +#include "openPMD/snapshots/StatefulIterator.hpp" + +namespace openPMD +{ +using value_type = + Container::value_type; +auto stateful_to_opaque(StatefulIterator const &it) + -> OpaqueSeriesIterator; +} // namespace openPMD diff --git a/include/openPMD/snapshots/Snapshots.hpp b/include/openPMD/snapshots/Snapshots.hpp index 822727a267..f8471a95dc 100644 --- a/include/openPMD/snapshots/Snapshots.hpp +++ b/include/openPMD/snapshots/Snapshots.hpp @@ -22,7 +22,6 @@ #include "openPMD/Iteration.hpp" #include "openPMD/snapshots/ContainerTraits.hpp" -#include "openPMD/snapshots/RandomAccessIterator.hpp" #include #include #include diff --git a/include/openPMD/snapshots/StatefulIterator.hpp b/include/openPMD/snapshots/StatefulIterator.hpp index 2a51b38b60..c3f02e9b8a 100644 --- a/include/openPMD/snapshots/StatefulIterator.hpp +++ b/include/openPMD/snapshots/StatefulIterator.hpp @@ -304,61 +304,6 @@ class StatefulIterator auto reparse_possibly_deleted_iteration(iteration_index_t) -> void; }; - -class LegacyIteratorAdaptor -{ - using value_type = IndexedIteration; - using parent_t = StatefulIterator; - -private: - friend class ReadIterations; - StatefulIterator m_iterator; - LegacyIteratorAdaptor(StatefulIterator iterator); - -public: - value_type operator*() const; - LegacyIteratorAdaptor &operator++(); - static LegacyIteratorAdaptor end(); - bool operator==(LegacyIteratorAdaptor const &other) const; - bool operator!=(LegacyIteratorAdaptor const &other) const; -}; - -/** - * @brief Reading side of the streaming API. - * - * Create instance via Series::readIterations(). - * For use in a C++11-style foreach loop over iterations. - * Designed to allow reading any kind of Series, streaming and non- - * streaming alike. - * Calling Iteration::close() manually before opening the next iteration is - * encouraged and will implicitly flush all deferred IO actions. - * Otherwise, Iteration::close() will be implicitly called upon - * StatefulIterator::operator++(), i.e. upon going to the next iteration in - * the foreach loop. - * Since this is designed for streaming mode, reopening an iteration is - * not possible once it has been closed. - * - */ -class ReadIterations -{ - friend class Series; - -private: - using iterations_t = decltype(internal::SeriesData::iterations); - using iterator_t = LegacyIteratorAdaptor; - - Series m_series; - std::optional m_parsePreference; - - ReadIterations( - Series, - Access, - std::optional parsePreference); - -public: - auto begin() -> iterator_t; - auto end() -> iterator_t; -}; } // namespace openPMD // Template definitions diff --git a/src/ReadIterations.cpp b/src/ReadIterations.cpp new file mode 100644 index 0000000000..84e1e9639a --- /dev/null +++ b/src/ReadIterations.cpp @@ -0,0 +1,64 @@ +#include "openPMD/ReadIterations.hpp" +#include "openPMD/snapshots/IteratorHelpers.hpp" +#include "openPMD/snapshots/StatefulIterator.hpp" + +namespace openPMD +{ +LegacyIteratorAdaptor::LegacyIteratorAdaptor(Snapshots::iterator iterator) + : m_iterator(std::move(iterator)) +{} + +auto LegacyIteratorAdaptor::operator*() const -> value_type +{ + return m_iterator.operator*(); +} + +auto LegacyIteratorAdaptor::operator++() -> LegacyIteratorAdaptor & +{ + ++m_iterator; + return *this; +} + +auto LegacyIteratorAdaptor::operator==(LegacyIteratorAdaptor const &other) const + -> bool +{ + return m_iterator == other.m_iterator; +} + +auto LegacyIteratorAdaptor::operator!=(LegacyIteratorAdaptor const &other) const + -> bool +{ + return m_iterator != other.m_iterator; +} + +ReadIterations::ReadIterations( + Series series, + Access access, + std::optional parsePreference) + : m_series(std::move(series)), m_parsePreference(parsePreference) +{ + auto &data = m_series.get(); + if (access == Access::READ_LINEAR && !data.m_sharedReadIterations) + { + // Open the iterator now already, so that metadata may already be read + data.m_sharedReadIterations = std::make_unique( + StatefulIterator::tag_read, m_series, m_parsePreference); + } +} + +auto ReadIterations::begin() -> iterator_t +{ + auto &series = m_series.get(); + if (!series.m_sharedReadIterations) + { + series.m_sharedReadIterations = std::make_unique( + StatefulIterator::tag_read, m_series, m_parsePreference); + } + return stateful_to_opaque(*series.m_sharedReadIterations); +} + +auto ReadIterations::end() -> iterator_t +{ + return stateful_to_opaque(StatefulIterator::end()); +} +} // namespace openPMD diff --git a/src/binding/python/Series.cpp b/src/binding/python/Series.cpp index 88c40b9640..2967438863 100644 --- a/src/binding/python/Series.cpp +++ b/src/binding/python/Series.cpp @@ -109,7 +109,7 @@ not possible once it has been closed. .def( "__next__", [](StatefulIteratorPythonAdaptor &iterator) { - if (iterator == LegacyIteratorAdaptor::end()) + if (iterator == ReadIterations::end()) { throw py::stop_iteration(); } @@ -127,7 +127,7 @@ not possible once it has been closed. ++iterator; } iterator.first_iteration = false; - if (iterator == LegacyIteratorAdaptor::end()) + if (iterator == ReadIterations::end()) { throw py::stop_iteration(); } diff --git a/src/snapshots/ContainerImpls.cpp b/src/snapshots/ContainerImpls.cpp index 9e129f6d8d..2b41b834e4 100644 --- a/src/snapshots/ContainerImpls.cpp +++ b/src/snapshots/ContainerImpls.cpp @@ -2,6 +2,7 @@ #include "openPMD/Error.hpp" #include "openPMD/IO/Access.hpp" #include "openPMD/snapshots/ContainerTraits.hpp" +#include "openPMD/snapshots/IteratorHelpers.hpp" #include "openPMD/snapshots/StatefulIterator.hpp" #include #include @@ -9,20 +10,6 @@ namespace openPMD { -namespace -{ - using value_type = - Container::value_type; - auto stateful_to_opaque(StatefulIterator const &it) - -> OpaqueSeriesIterator - { - std::unique_ptr> - internal_iterator_cloned{new StatefulIterator(it)}; - return OpaqueSeriesIterator( - std::move(internal_iterator_cloned)); - } -} // namespace - StatefulSnapshotsContainer::StatefulSnapshotsContainer( std::function begin) : members{std::move(begin)} diff --git a/src/snapshots/IteratorHelpers.cpp b/src/snapshots/IteratorHelpers.cpp new file mode 100644 index 0000000000..0621fdb71a --- /dev/null +++ b/src/snapshots/IteratorHelpers.cpp @@ -0,0 +1,15 @@ +#include "openPMD/snapshots/IteratorHelpers.hpp" + +namespace openPMD +{ +using value_type = + Container::value_type; +auto stateful_to_opaque(StatefulIterator const &it) + -> OpaqueSeriesIterator +{ + std::unique_ptr> internal_iterator_cloned{ + new StatefulIterator(it)}; + return OpaqueSeriesIterator( + std::move(internal_iterator_cloned)); +} +} // namespace openPMD diff --git a/src/snapshots/StatefulIterator.cpp b/src/snapshots/StatefulIterator.cpp index 60c284105f..e61ec9a36c 100644 --- a/src/snapshots/StatefulIterator.cpp +++ b/src/snapshots/StatefulIterator.cpp @@ -966,67 +966,4 @@ StatefulIterator::operator bool() const { return m_data->has_value(); } - -LegacyIteratorAdaptor::LegacyIteratorAdaptor(StatefulIterator iterator) - : m_iterator(std::move(iterator)) -{} - -auto LegacyIteratorAdaptor::operator*() const -> value_type -{ - return m_iterator.operator*(); -} - -auto LegacyIteratorAdaptor::operator++() -> LegacyIteratorAdaptor & -{ - ++m_iterator; - return *this; -} - -auto LegacyIteratorAdaptor::end() -> LegacyIteratorAdaptor -{ - return StatefulIterator::end(); -} - -auto LegacyIteratorAdaptor::operator==(LegacyIteratorAdaptor const &other) const - -> bool -{ - return m_iterator == other.m_iterator; -} - -auto LegacyIteratorAdaptor::operator!=(LegacyIteratorAdaptor const &other) const - -> bool -{ - return m_iterator != other.m_iterator; -} - -ReadIterations::ReadIterations( - Series series, - Access access, - std::optional parsePreference) - : m_series(std::move(series)), m_parsePreference(parsePreference) -{ - auto &data = m_series.get(); - if (access == Access::READ_LINEAR && !data.m_sharedReadIterations) - { - // Open the iterator now already, so that metadata may already be read - data.m_sharedReadIterations = std::make_unique( - StatefulIterator::tag_read, m_series, m_parsePreference); - } -} - -ReadIterations::iterator_t ReadIterations::begin() -{ - auto &series = m_series.get(); - if (!series.m_sharedReadIterations) - { - series.m_sharedReadIterations = std::make_unique( - StatefulIterator::tag_read, m_series, m_parsePreference); - } - return *series.m_sharedReadIterations; -} - -ReadIterations::iterator_t ReadIterations::end() -{ - return iterator_t::end(); -} } // namespace openPMD From 3dce4b6475c87750a384b8c0070bfe20308db5a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 5 Jul 2024 16:44:57 +0200 Subject: [PATCH 87/93] Bugfix --- src/snapshots/ContainerImpls.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/snapshots/ContainerImpls.cpp b/src/snapshots/ContainerImpls.cpp index 2b41b834e4..2a56fc89aa 100644 --- a/src/snapshots/ContainerImpls.cpp +++ b/src/snapshots/ContainerImpls.cpp @@ -60,7 +60,7 @@ auto StatefulSnapshotsContainer::currentIteration() const } else { - return nullptr; + return std::nullopt; } } From 2e77dd398c4e1efcdc2a1971a22f59cef7ba9d9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 4 Jul 2024 17:25:31 +0200 Subject: [PATCH 88/93] Documentation, cleanup --- examples/10_streaming_write.cpp | 2 +- include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp | 8 + include/openPMD/Iteration.hpp | 7 + include/openPMD/ReadIterations.hpp | 14 +- include/openPMD/Series.hpp | 86 +++++++- include/openPMD/WriteIterations.hpp | 5 + include/openPMD/openPMD.hpp | 2 +- include/openPMD/snapshots/ContainerImpls.hpp | 25 ++- include/openPMD/snapshots/ContainerTraits.hpp | 24 ++- include/openPMD/snapshots/IteratorTraits.hpp | 43 +++- .../snapshots/RandomAccessIterator.hpp | 8 + include/openPMD/snapshots/Snapshots.hpp | 70 ++++++ .../openPMD/snapshots/StatefulIterator.hpp | 119 +++++++++-- src/IO/ADIOS/ADIOS2IOHandler.cpp | 6 + src/ReadIterations.cpp | 10 +- src/Series.cpp | 66 +++--- src/helper/list_series.cpp | 10 +- src/snapshots/ContainerImpls.cpp | 43 ++-- src/snapshots/ContainerTraits.cpp | 6 - src/snapshots/StatefulIterator.cpp | 199 +++++++++--------- 20 files changed, 544 insertions(+), 209 deletions(-) create mode 100644 include/openPMD/WriteIterations.hpp diff --git a/examples/10_streaming_write.cpp b/examples/10_streaming_write.cpp index 5d5fbfdb15..fc609dcf10 100644 --- a/examples/10_streaming_write.cpp +++ b/examples/10_streaming_write.cpp @@ -45,7 +45,7 @@ int main() // in streaming setups, e.g. an iteration cannot be opened again once // it has been closed. // `Series::iterations` can be directly accessed in random-access workflows. - auto iterations = series.snapshots(SnapshotAccess::Linear); + auto iterations = series.snapshots(SnapshotWorkflow::Synchronous); for (size_t i = 0; i < 100; ++i) { Iteration iteration = iterations[i]; diff --git a/include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp b/include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp index 3a1abd7744..1b350feee1 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp @@ -67,6 +67,14 @@ namespace adios_defs No }; + /* + * Necessary to implement the `reopen` flag of + * `Parameter`. The distinction between Open and + * Reopen is necessary for Write workflows in file-based encoding. In order + * to write new data to an Iteration that was created and closed previously, + * the only applicable access mode is Append mode, ideally in conjunction + * with `SetParameter("FlattenSteps", "ON")`. + */ enum class OpenFileAs { Create, diff --git a/include/openPMD/Iteration.hpp b/include/openPMD/Iteration.hpp index 468f524e7f..ff72bf1edf 100644 --- a/include/openPMD/Iteration.hpp +++ b/include/openPMD/Iteration.hpp @@ -85,6 +85,13 @@ namespace internal * overwritten. */ CloseStatus m_closed = CloseStatus::Open; + /* + * While parsing a file-based Series, each file is opened, read, then + * closed again. Explicitly `Iteration::open()`ing a file should only be + * necessary after having explicitly closed it (or in + * defer_iteration_parsing mode). So, the parsing procedures will set + * this flag as true when closing an Iteration. + */ bool allow_reopening_implicitly = false; /** diff --git a/include/openPMD/ReadIterations.hpp b/include/openPMD/ReadIterations.hpp index e6903b9704..d393d99706 100644 --- a/include/openPMD/ReadIterations.hpp +++ b/include/openPMD/ReadIterations.hpp @@ -10,6 +10,11 @@ namespace openPMD { +/** @brief Legacy Iterator type for `Series::readIterations()` + * + * Wraps the Iterator type of `Series::snapshots()`, but has `IndexedIteration` + * as value_type instead of `std::pair`. + */ class LegacyIteratorAdaptor { using value_type = IndexedIteration; @@ -28,7 +33,12 @@ class LegacyIteratorAdaptor }; /** - * @brief Reading side of the streaming API. + * @brief Legacy class as return type for `Series::readIterations()`. + * + * This is a feature-restricted subset for the functionality of + * `Series::snapshots()`, prefer using that. The compatibility layer is needed + * due to the different value_type for `Series::readIterations()`-based + * iteration (`IterationIndex` instead of `std::pair`). * * Create instance via Series::readIterations(). * For use in a C++11-style foreach loop over iterations. @@ -39,8 +49,6 @@ class LegacyIteratorAdaptor * Otherwise, Iteration::close() will be implicitly called upon * StatefulIterator::operator++(), i.e. upon going to the next iteration in * the foreach loop. - * Since this is designed for streaming mode, reopening an iteration is - * not possible once it has been closed. * */ class ReadIterations diff --git a/include/openPMD/Series.hpp b/include/openPMD/Series.hpp index 8057fc9ca5..6cb21cb266 100644 --- a/include/openPMD/Series.hpp +++ b/include/openPMD/Series.hpp @@ -110,13 +110,29 @@ namespace internal * Due to include order, this member needs to be a pointer instead of * an optional. */ - std::unique_ptr m_sharedReadIterations; + std::unique_ptr m_sharedStatefulIterator; /** * For writing: Remember which iterations have been written in the * currently active output step. Use this later when writing the * snapshot attribute. */ std::set m_currentlyActiveIterations; + /** + * This map contains the filenames of those Iterations which were found + * on the file system upon opening the Series for reading in file-based + * encoding. It is only written to by readFileBased(). + * Other files that we create anew have names generated live by + * iterationFilename(), but files that existed previously might have + * different padding. + * This information is required for re-opening Iterations after closing. + * Since ADIOS2 has no read-write mode, this is important in our own + * READ_WRITE mode when re-opening a closed file in file-based encoding: + * A file that existed previously is re-opened in Read mode and will + * not support updating its contents. + * A file that we created anew is re-opened in Append mode to continue + * writing data to it. Using `adios2.engine.parameters.FlattenSteps = + * "ON"` is recommended in this case. + */ std::unordered_map m_iterationFilenames; /** * Needed if reading a single iteration of a file-based series. @@ -640,13 +656,62 @@ class Series : public Attributable */ ReadIterations readIterations(); - enum class SnapshotAccess + /** Parameter for Series::snapshots(), see there. + */ + enum class SnapshotWorkflow { RandomAccess, - Linear + Synchronous }; - Snapshots snapshots( - std::optional access_synchronously = std::nullopt); + + /** @brief Preferred way to access Iterations/Snapshots. Single API for all + * workflows and access modi. + * + * Two fundamental workflows for Iteration/Snapshot processing exist: + * + * 1. Random-access workflow: Iterations/Snapshots are accessed + * independently from one another. Users must take care to open() + * and close() them as needed. + * More than one Iteration can be open at the same time. + * 2. Linear/Synchronous workflow: + * The (parallel) Series has one shared state. The effect is twofold: + * (a) Advancing one iterator, e.g. via `operator++()` will advance all + * other iterators as well. + * (b) Advancing an iterator is collective, all MPI ranks must + participate. + * The workflow is generally managed more automatically, + * `Iteration::open()` is not needed, `Iteration::close()` is + * recommended, but not needed. In READ_LINEAR mode, parsed Iteration + * data is deleted upon closing (and will be reparsed upon reopening) + * for a better support of datasets with many Snapshots/Iterations. This + * happens only if user code explicitly calls Iteration::close(). + * This mode generally brings better performance than random access mode + * due to the more restricted workflow. + * For accessing some kinds of Series, Synchronous access is even + * necessary, e.g. for Streaming workflows or variable-based encoding. + * (In future, random-access of Series with variable-based encoding will + * be possible under the condition that each Iteration/Snapshot has the + * same internal structure). + * + * As a rule of thumb, the synchronous workflow should be preferred as long + * as possible. The random-access workflow should be chosen when more + * flexible interaction with Snapshots is needed. + * + * Random-vs.-Synchronous access is determined automatically + * in READ workflows: Access::READ_LINEAR uses the synchronous workflow, + * while Access::READ_ONLY and Access::READ_WRITE use the random-access + * workflow. + * + * Conversely, the Access::CREATE and Access::APPEND access modes both + * resolve to random-access by default, but can be specified to use + * Synchronous workflow if needed. + * + * @param snapshot_workflow Specify the intended workflow + * in Access::CREATE and Access::APPEND. Leave unspecified in + * other access modes as those support only one workflow each. + */ + Snapshots + snapshots(std::optional snapshot_workflow = std::nullopt); /** * @brief Parse the Series. @@ -807,6 +872,9 @@ OPENPMD_private void flushMeshesPath(); void flushParticlesPath(); void flushRankTable(); + /* Parameter `read_only_this_single_iteration` used for reopening an + * Iteration after closing it. + */ void readFileBased( std::optional read_only_this_single_iteration); void readOneIterationFileBased(std::string const &filePath); @@ -820,8 +888,10 @@ OPENPMD_private * and turn them into a warning (useful when parsing a Series, since parsing * should succeed without issue). * If true, the error will always be re-thrown (useful when using - * ReadIterations since those methods should be aware when the current - * step is broken). + * ReadIterations since those methods should be aware when the current step + * is broken). + * Parameter `read_only_this_single_iteration` used for reopening an + * Iteration after closing it. */ std::vector readGorVBased( bool do_always_throw_errors, @@ -899,7 +969,7 @@ OPENPMD_private AbstractIOHandler const *IOHandler() const; }; // Series -using SnapshotAccess = Series::SnapshotAccess; +using SnapshotWorkflow = Series::SnapshotWorkflow; namespace debug { diff --git a/include/openPMD/WriteIterations.hpp b/include/openPMD/WriteIterations.hpp new file mode 100644 index 0000000000..932a651f7f --- /dev/null +++ b/include/openPMD/WriteIterations.hpp @@ -0,0 +1,5 @@ +#pragma once + +// legacy header + +#include "openPMD/snapshots/Snapshots.hpp" diff --git a/include/openPMD/openPMD.hpp b/include/openPMD/openPMD.hpp index d29bcdd818..0773243b66 100644 --- a/include/openPMD/openPMD.hpp +++ b/include/openPMD/openPMD.hpp @@ -38,7 +38,7 @@ namespace openPMD #include "openPMD/RecordComponent.hpp" #include "openPMD/Series.hpp" #include "openPMD/UnitDimension.hpp" -#include "openPMD/snapshots/StatefulIterator.hpp" +#include "openPMD/snapshots/Snapshots.hpp" #include "openPMD/backend/Attributable.hpp" #include "openPMD/backend/Attribute.hpp" diff --git a/include/openPMD/snapshots/ContainerImpls.hpp b/include/openPMD/snapshots/ContainerImpls.hpp index 04d8e874ba..8a0189e80e 100644 --- a/include/openPMD/snapshots/ContainerImpls.hpp +++ b/include/openPMD/snapshots/ContainerImpls.hpp @@ -3,8 +3,13 @@ #include "openPMD/snapshots/ContainerTraits.hpp" #include "openPMD/snapshots/RandomAccessIterator.hpp" #include "openPMD/snapshots/StatefulIterator.hpp" + #include +/* + * Private header, not included in user code. + */ + namespace openPMD { class StatefulSnapshotsContainer : public AbstractSnapshotsContainer @@ -14,6 +19,21 @@ class StatefulSnapshotsContainer : public AbstractSnapshotsContainer struct Members { + /* + * Consider the following user code: + * > auto iterations = series.snapshots(); + * > for (auto & iteration : iterations) { ... } + * + * Here, only the for loop should actually wait for Iteration data. For + * ensuring that Iterations are not waited for too early, the + * initialization procedures are stored as a `std::function` in here and + * only called upon the right moment. Until then, m_bufferedIterator + * stays empty. + * Compare the implementation of Series::snapshots(). In there, m_begin + * is defined either by make_writing_stateful_iterator or + * make_reading_stateful_iterator. + * The iterator is resolved upon calling get() below. + */ std::function m_begin; std::optional m_bufferedIterator = std::nullopt; }; @@ -37,7 +57,7 @@ class StatefulSnapshotsContainer : public AbstractSnapshotsContainer operator=(StatefulSnapshotsContainer &&other) noexcept(noexcept( std::declval().operator=(std::declval()))); - auto currentIteration() -> std::optional override; + using AbstractSnapshotsContainer::currentIteration; auto currentIteration() const -> std::optional override; auto begin() -> iterator override; @@ -84,6 +104,9 @@ class RandomAccessIteratorContainer : public AbstractSnapshotsContainer RandomAccessIteratorContainer & operator=(RandomAccessIteratorContainer &&other) noexcept; + using AbstractSnapshotsContainer::currentIteration; + auto currentIteration() const -> std::optional override; + auto begin() -> iterator override; auto end() -> iterator override; auto begin() const -> const_iterator override; diff --git a/include/openPMD/snapshots/ContainerTraits.hpp b/include/openPMD/snapshots/ContainerTraits.hpp index a698961742..6a6d7e904d 100644 --- a/include/openPMD/snapshots/ContainerTraits.hpp +++ b/include/openPMD/snapshots/ContainerTraits.hpp @@ -2,8 +2,17 @@ #include "openPMD/Iteration.hpp" #include "openPMD/snapshots/IteratorTraits.hpp" + +/* Public header due to use of AbstractSnapshotsContainer and its iterator type + * OpaqueSeriesIterator in Snapshots class header. No direct user interaction + * required with this header. + */ + namespace openPMD { +/** Counterpart to Snapshots class: + * Iterator type that can wrap different implementations internally. + */ template class OpaqueSeriesIterator : public AbstractSeriesIterator< @@ -38,14 +47,26 @@ class OpaqueSeriesIterator // increment/decrement OpaqueSeriesIterator &operator++(); + /** Not implemented for synchronous workflow: + * Reverse iteration not possible. + */ OpaqueSeriesIterator &operator--(); + /** Not implemented for synchronous workflow: + * Post increment not possible. + */ OpaqueSeriesIterator operator++(int); + /** Not implemented for synchronous workflow: + * Reverse iteration not possible. + */ OpaqueSeriesIterator operator--(int); // comparison bool operator==(OpaqueSeriesIterator const &other) const; }; +// Internal interface used by Snapshots class for interacting with containers. +// This needs to be in a public header since the type definition is used in +// private members of the Snapshots class which itself is a public class. class AbstractSnapshotsContainer { public: @@ -62,7 +83,8 @@ class AbstractSnapshotsContainer virtual ~AbstractSnapshotsContainer() = 0; virtual auto currentIteration() -> std::optional; - virtual auto currentIteration() const -> std::optional; + virtual auto + currentIteration() const -> std::optional = 0; virtual auto begin() -> iterator = 0; virtual auto begin() const -> const_iterator = 0; diff --git a/include/openPMD/snapshots/IteratorTraits.hpp b/include/openPMD/snapshots/IteratorTraits.hpp index e4ee9a18cc..f44f3c25b1 100644 --- a/include/openPMD/snapshots/IteratorTraits.hpp +++ b/include/openPMD/snapshots/IteratorTraits.hpp @@ -22,12 +22,33 @@ #include "openPMD/Iteration.hpp" #include "openPMD/backend/Writable.hpp" + #include +/* + * Public header due to use in OpaqueSeriesIterator type which is the public + * iterator type of Snapshots class. + */ + namespace openPMD { -// Abstract class that can be used as an abstract interface to an opaque -// iterator implementation +/* + * Abstract class that can be used as an abstract interface to an opaque + * iterator implementation + * + * This has renamed operator names for two reasons: + * + * 1. Name shadowing of default implementations is too finnicky. + * 2. The return type for the actual operators should be a reference to the + * child class, but for an Interface we need a common return type. + * + * For returning a reference to the child class, we need a CRT-style template to + * know the type. That does not work for an interface. As a result, this generic + * Iterator interface is split in two parts, class DynamicSeriesIterator which + * can be used generically without specifying the child class. All methods that + * need to know the child class type are put into the CRT-style class template + * AbstractSeriesIterator below. + */ template < typename value_type = Container::value_type> @@ -54,13 +75,17 @@ class DynamicSeriesIterator virtual std::unique_ptr clone() const = 0; }; - -// Class template with default method implementations for iterators. -// No virtual classes since there is no use. -// Commented methods must be implemented by child classes. -// Implement as `class MyIterator : public AbstractSeriesIterator` -// Use `using AbstractSeriesIterator::operator-` to pull default -// implementations. +/* + * Class template with default method implementations for iterators. + * Complementary class to above class DynamicSeriesIterator, containing those + * methods that have the specific Iterator type in their type specification. See + * the documentation for DynamicSeriesIterator for more details. + * No virtual classes since there is no use. + * Commented methods must be implemented by child classes. + * Implement as `class MyIterator : public AbstractSeriesIterator` + * Use `using AbstractSeriesIterator::operator-` to pull default + * implementations. + */ template < typename ChildClass, typename value_type_in = typename ChildClass::value_type> diff --git a/include/openPMD/snapshots/RandomAccessIterator.hpp b/include/openPMD/snapshots/RandomAccessIterator.hpp index 3636d61768..1487d64aa7 100644 --- a/include/openPMD/snapshots/RandomAccessIterator.hpp +++ b/include/openPMD/snapshots/RandomAccessIterator.hpp @@ -22,7 +22,14 @@ #include "openPMD/Iteration.hpp" #include "openPMD/snapshots/IteratorTraits.hpp" + #include + +/* + * Private header not included in user code. + * Implements the Iterator interface for the random-access workflow. + */ + namespace openPMD { namespace detail @@ -47,6 +54,7 @@ class RandomAccessIterator RandomAccessIterator(iterator_t it); + /* Internal iterator */ iterator_t m_it; public: diff --git a/include/openPMD/snapshots/Snapshots.hpp b/include/openPMD/snapshots/Snapshots.hpp index f8471a95dc..7a75780dfe 100644 --- a/include/openPMD/snapshots/Snapshots.hpp +++ b/include/openPMD/snapshots/Snapshots.hpp @@ -28,6 +28,24 @@ namespace openPMD { +/** Entry point for accessing Snapshots/Iterations. Common API for different + * Snapshot accessing workflows. + * + * Refer to Series::snapshots() for a detailed description of different + * workflows and their applications. + * The API generally follows the Container/map conventions, though especially + * the synchronous workflow does not implement all of the API calls. + * + * Since the synchronous workflow is tailored to support use cases from ADIOS2 + * which do not support going back to the start (IO steps, streaming), the + * Container semantics are based on the remaining Iterations that have not yet + * been seen. Especially does this mean that Snapshots::begin() does not reset + * the Iterator back to the start, but will return the current state. There is + * restricted support for going back to past Iterations by reopening them under + * the condition that no previous ADIOS2 step needs to be activated. Note that + * an Iteration handle goes invalid after closing it, a new Iteration handle is + * acquired by Snapshots::operator[](). + */ class Snapshots { private: @@ -52,36 +70,88 @@ class Snapshots using const_reverse_iterator = AbstractSnapshotsContainer::const_reverse_iterator; + /** @brief The currently active Iteration. + * + * Mostly useful for the synchronous workflow, since that has a notion of + * global state where a specific Iteration is currently active (or no + * Iteration, in which case a nullopt is returned). For random-access + * workflow, the first Iteration in the Series is returned. + */ auto currentIteration() -> std::optional; auto currentIteration() const -> std::optional; auto begin() -> iterator; auto end() -> iterator; + /** Not implemented for synchronous workflow: Const iteration not possible. + */ auto begin() const -> const_iterator; + /** Not implemented for synchronous workflow: Const iteration not possible. + */ auto end() const -> const_iterator; + /** Not implemented for synchronous workflow: + * Reverse iteration not possible. + */ auto rbegin() -> reverse_iterator; + /** Not implemented for synchronous workflow: + * Reverse iteration not possible. + */ auto rend() -> reverse_iterator; + /** Not implemented for synchronous workflow: + * Const reverse iteration not possible. + */ auto rbegin() const -> const_reverse_iterator; + /** Not implemented for synchronous workflow: + * Const reverse iteration not possible. + */ auto rend() const -> const_reverse_iterator; + /** In synchronous workflow, this tells if there are remaining Iterations or + * not. + */ auto empty() const -> bool; + /** Not implemented in synchronous workflow due to unclear semantics (past + * Iterations should not be considered, future Iterations are not yet + * known). + */ auto size() const -> size_t; + /** Select an Iteration within the current IO step. + */ auto at(key_type const &key) const -> mapped_type const &; + /** Select an Iteration within the current IO step. + */ auto at(key_type const &key) -> mapped_type &; auto operator[](key_type const &key) -> mapped_type &; + /** Not implmented in synchronous workflow. + */ auto clear() -> void; // insert // swap + /** Not implmented in synchronous workflow. + * + * (In future: Might be implemented in terms of searching within the current + * IO step) + */ auto find(key_type const &key) -> iterator; + /** Not implmented in synchronous workflow. + * + * (In future: Might be implemented in terms of checking the current + * Iteration against the searched key and returning end() if it doesn't + * match. Anything else is not possible since the shared Iterator would have + * to be modified.) + */ auto find(key_type const &key) const -> const_iterator; + /** Implemented in terms of contains(), see there. + */ auto count(key_type const &key) const -> size_t; + /** Not implmented in synchronous workflow. + */ auto contains(key_type const &key) const -> bool; // erase diff --git a/include/openPMD/snapshots/StatefulIterator.hpp b/include/openPMD/snapshots/StatefulIterator.hpp index c3f02e9b8a..571b59bdc6 100644 --- a/include/openPMD/snapshots/StatefulIterator.hpp +++ b/include/openPMD/snapshots/StatefulIterator.hpp @@ -36,6 +36,11 @@ #include #include +/* + * Private header not included in user code. + * Implements the Iterator interface for the stateful/synchronous workflow. + */ + namespace openPMD { namespace internal @@ -45,26 +50,46 @@ namespace internal namespace detail { + /* The iterator status is either of the following: + */ namespace step_status_types { + /* No step was opened yet, the Series was just opened. + */ struct Before_t {}; + /* A step is currently active + */ struct During_t { - size_t idx; + // The index of the current Step. + size_t step_count; + // The current Iteration within the Step. + // Empty optional indicates that no Iteration is left in the current + // step for processing, i.e. a new step must be opened or the Series + // is over. std::optional iteration_idx; + // Iteration indexes that are accessible within the current step. + // These are not modified when closing an Iteration as long as the + // current IO step stays active. std::vector available_iterations_in_step; During_t( - size_t idx, + size_t step_count, std::optional iteration_idx, std::vector available_iterations_in_step); }; + /* No further data available in the Series. + */ struct After_t {}; } // namespace step_status_types + + /* This class unifies the current step status as described above into a + * std::variant with some helper functions. + */ struct CurrentStep : std::variant< step_status_types::Before_t, @@ -90,18 +115,33 @@ namespace detail auto get_variant() const -> std::optional; auto get_iteration_index() const - -> std::optional; - auto - get_iteration_index() -> std::optional; + -> std::optional; + /* Passed as first param of create_new lambda in map_during_t, so the + * lambda can make an appropriate case distinction. + */ enum class AtTheEdge : bool { Begin, End }; + /* + * Helper for a common way to access the underlying variant. + * `map` has type `auto (During_t &) -> void`, i.e. it can modify the + * `During_t` struct if the variant holds it. In other cases, + * `create_new` is called, it has the type + * `auto (AtTheEdge) -> std::optional`. + * `AtTheEdge` is used for specifying if the variant status is Begin or + * End. If the returned optional contains a value, that value is swapped + * with the current variant. + */ template void map_during_t(F &&map, G &&create_new); + + /* + * Overload where `create_new` is a no-op. + */ template void map_during_t(F &&map); @@ -117,10 +157,15 @@ namespace detail } }; + /* + * Types for telling the Iterator where to go next. + */ namespace seek_types { + /* Just give me the next Iteration */ struct Next_t {}; + /* Give me some specific Iteration */ struct Seek_Iteration_t { Iteration::IterationIndex_t iteration_idx; @@ -149,6 +194,9 @@ namespace detail }; } // namespace detail +/** Based on the logic of the former class ReadIterations, integrating into + * itself the logic of former WriteIterations. + */ class StatefulIterator : public AbstractSeriesIterator< StatefulIterator, @@ -160,8 +208,6 @@ class StatefulIterator using iteration_index_t = IndexedIteration::index_t; - using maybe_series_t = std::optional; - using CurrentStep = detail::CurrentStep; struct SharedData @@ -176,18 +222,50 @@ class StatefulIterator using step_index = size_t; + /* + * This must be a non-owning internal handle to break reference cycles. + * A non-owning handle is fine due to the usual semantics for iterator + * invalidation. + */ Series series; + /* + * No step opened yet, so initialize this with CurrentStep::Before. Look + * to the documentation of Before_t, During_t, After_t and CurrentStep + * classes above for more info. + */ CurrentStep currentStep = {CurrentStep::Before}; + /* + * Stores the parse preference optionally passed in the constructor. + * Decides if IO step logic is actually used. + */ std::optional parsePreference; + /* + * Store which Iterations we already saw and in which IO step we did. + * Currently used for eliminating repetitions when (e.g. due to + * checkpoint-restart workflows) Iterations repeat in different steps. + * + * Possible future uses: + * + * 1. Support jumping back to a previous step in order to reopen an + * Iteration previously seen. (Would require reopening files in + * ADIOS2, but so be it.) + * 2. Pre-parsing a variable-based file for repeating Iterations and + * eliminating the earlier instances of repeated Iterations (instead + * of the later instances as is done now). + */ std::unordered_map seen_iterations; - auto currentIteration() -> std::optional; - auto currentIteration() const - -> std::optional; + /* + * This returns the current value of `During_t::iteration_idx` if that + * exists. + */ + auto + currentIteration() const -> std::optional; }; /* - * The shared data is never empty, emptiness is indicated by std::optional + * The shared pointer is never empty, + * emptiness is indicated by std::optional. */ std::shared_ptr> m_data = std::make_shared>(std::nullopt); @@ -259,8 +337,15 @@ class StatefulIterator operator bool() const; - // Custom non-iterator methods - auto seek(Seek const &) -> StatefulIterator *; + /* + * Try moving this Iterator to the location specified by Seek, i.e.: + * + * 1. Either the next available Iteration + * 2. Or a specific Iteration specified by an index. + * + * A new step will be opened for this purpose if needed. + */ + auto seek(Seek const &) -> StatefulIterator &; private: std::optional nextIterationInStep(); @@ -282,11 +367,19 @@ class StatefulIterator void initIteratorFilebased(); + /* + * Called when an Iteration was just opened but entirely fails parsing. + */ void deactivateDeadIteration(iteration_index_t); void initSeriesInLinearReadMode(); void close(); + + /* When not using IO steps, the status should not be set to After_t, but be + * kept as During_t. This way, Iterations can still be opened without the + * Iterator thinking it's from a past step. + */ enum class TypeOfEndIterator : char { NoMoreSteps, diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index a3dada698d..e26ba0ccff 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -1610,6 +1610,12 @@ adios2::Mode ADIOS2IOHandlerImpl::adios2AccessMode( return adios2::Mode::Read; #endif case adios_defs::OpenFileAs::Reopen: + /* In order to write new data to an Iteration that was + * created and closed previously, the only applicable access + * mode is Append mode, ideally in conjunction with + * `SetParameter("FlattenSteps", "ON")`. + * See test/Files_SerialIO/close_iteration_test.cpp. + */ return adios2::Mode::Append; } break; diff --git a/src/ReadIterations.cpp b/src/ReadIterations.cpp index 84e1e9639a..a1cd619b9b 100644 --- a/src/ReadIterations.cpp +++ b/src/ReadIterations.cpp @@ -38,10 +38,10 @@ ReadIterations::ReadIterations( : m_series(std::move(series)), m_parsePreference(parsePreference) { auto &data = m_series.get(); - if (access == Access::READ_LINEAR && !data.m_sharedReadIterations) + if (access == Access::READ_LINEAR && !data.m_sharedStatefulIterator) { // Open the iterator now already, so that metadata may already be read - data.m_sharedReadIterations = std::make_unique( + data.m_sharedStatefulIterator = std::make_unique( StatefulIterator::tag_read, m_series, m_parsePreference); } } @@ -49,12 +49,12 @@ ReadIterations::ReadIterations( auto ReadIterations::begin() -> iterator_t { auto &series = m_series.get(); - if (!series.m_sharedReadIterations) + if (!series.m_sharedStatefulIterator) { - series.m_sharedReadIterations = std::make_unique( + series.m_sharedStatefulIterator = std::make_unique( StatefulIterator::tag_read, m_series, m_parsePreference); } - return stateful_to_opaque(*series.m_sharedReadIterations); + return stateful_to_opaque(*series.m_sharedStatefulIterator); } auto ReadIterations::end() -> iterator_t diff --git a/src/Series.cpp b/src/Series.cpp index f3ddaa0b3b..49c65bfbdf 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -2922,9 +2922,9 @@ namespace internal void SeriesData::close() { // WriteIterations gets the first shot at flushing - if (this->m_sharedReadIterations) + if (this->m_sharedStatefulIterator) { - this->m_sharedReadIterations->close(); + this->m_sharedStatefulIterator->close(); } /* * Scenario: A user calls `Series::flush()` but does not check for @@ -2997,10 +2997,11 @@ ReadIterations Series::readIterations() { // Use private constructor instead of copy constructor to avoid // object slicing - Series res; - res.setData(std::dynamic_pointer_cast(this->m_attri)); + auto series = m_attri->asInternalCopyOf(); return ReadIterations{ - std::move(res), IOHandler()->m_frontendAccess, get().m_parsePreference}; + std::move(series), + IOHandler()->m_frontendAccess, + get().m_parsePreference}; } namespace @@ -3009,34 +3010,35 @@ namespace Series const &copied_series, internal::SeriesData &series) -> std::function { - if (!series.m_sharedReadIterations) + if (!series.m_sharedStatefulIterator) { - series.m_sharedReadIterations = std::make_unique( - StatefulIterator::tag_write, copied_series); + series.m_sharedStatefulIterator = + std::make_unique( + StatefulIterator::tag_write, copied_series); } - return [ptr = series.m_sharedReadIterations.get()]() { return ptr; }; + return [ptr = series.m_sharedStatefulIterator.get()]() { return ptr; }; } auto make_reading_stateful_iterator( Series copied_series, internal::SeriesData &series) -> std::function { return [s = std::move(copied_series), &series]() mutable { - if (!series.m_sharedReadIterations) + if (!series.m_sharedStatefulIterator) { auto parse_preference = series.m_parsePreference; - series.m_sharedReadIterations = + series.m_sharedStatefulIterator = std::make_unique( StatefulIterator::tag_read, std::move(s), parse_preference); } - return series.m_sharedReadIterations.get(); + return series.m_sharedStatefulIterator.get(); }; } } // namespace Snapshots -Series::snapshots(std::optional const access_synchronously) +Series::snapshots(std::optional const snapshot_workflow) { auto &series = get(); if (series.m_deferred_initialization.has_value()) @@ -3045,22 +3047,22 @@ Series::snapshots(std::optional const access_synchronously) } auto access = IOHandler()->m_frontendAccess; auto guard_wrong_access_specification = - [&](SnapshotAccess required_access) { - if (!access_synchronously.has_value()) + [&](SnapshotWorkflow required_access) { + if (!snapshot_workflow.has_value()) { return required_access; } - if (required_access != *access_synchronously) + if (required_access != *snapshot_workflow) { std::stringstream error; error << "[Series::snapshots()] Specified " - << (*access_synchronously == SnapshotAccess::Linear + << (*snapshot_workflow == SnapshotWorkflow::Synchronous ? "linear" : "random-access") << " iteration in method parameter " - "`access_synchronously`, but access type " + "`snapshot_workflow`, but access type " << access << " requires " - << (required_access == SnapshotAccess::Linear + << (required_access == SnapshotWorkflow::Synchronous ? "linear" : "random-access") << " iteration. Please remove the parameter, there is no " @@ -3073,22 +3075,22 @@ Series::snapshots(std::optional const access_synchronously) std::cerr << "[Series::snapshots()] No need to explicitly specify " "synchronous or non-synchronous access via method " - "parameter `access_synchronously` in mode '" + "parameter `snapshot_workflow` in mode '" << access << ". Will ignore." << std::endl; } return required_access; }; - SnapshotAccess usedSnapshotAccess{}; + SnapshotWorkflow usedSnapshotWorkflow{}; { switch (access) { case Access::READ_LINEAR: - usedSnapshotAccess = - guard_wrong_access_specification(SnapshotAccess::Linear); + usedSnapshotWorkflow = + guard_wrong_access_specification(SnapshotWorkflow::Synchronous); break; case Access::READ_ONLY: - usedSnapshotAccess = - guard_wrong_access_specification(SnapshotAccess::RandomAccess); + usedSnapshotWorkflow = guard_wrong_access_specification( + SnapshotWorkflow::RandomAccess); // Some error checks if (series.m_parsePreference.has_value()) @@ -3118,27 +3120,27 @@ Series::snapshots(std::optional const access_synchronously) // far). // (Might be possible to allow stateful access actually, but there's // no real use, so keep it simple.) - usedSnapshotAccess = - guard_wrong_access_specification(SnapshotAccess::RandomAccess); + usedSnapshotWorkflow = guard_wrong_access_specification( + SnapshotWorkflow::RandomAccess); break; case Access::CREATE: case Access::APPEND: // Users can select. - usedSnapshotAccess = access_synchronously.value_or( + usedSnapshotWorkflow = snapshot_workflow.value_or( /* random-access logic by default */ - SnapshotAccess::RandomAccess); + SnapshotWorkflow::RandomAccess); break; } } - switch (usedSnapshotAccess) + switch (usedSnapshotWorkflow) { - case SnapshotAccess::RandomAccess: { + case SnapshotWorkflow::RandomAccess: { return Snapshots(std::shared_ptr{ new RandomAccessIteratorContainer(series.iterations)}); } - case SnapshotAccess::Linear: { + case SnapshotWorkflow::Synchronous: { std::function begin; if (access::write(IOHandler()->m_frontendAccess)) diff --git a/src/helper/list_series.cpp b/src/helper/list_series.cpp index 7782573b11..415f8e4743 100644 --- a/src/helper/list_series.cpp +++ b/src/helper/list_series.cpp @@ -30,9 +30,6 @@ #include #include -inline void breakpoint() -{} - namespace openPMD::helper { std::ostream &listSeries(Series &series, bool const longer, std::ostream &out) @@ -118,6 +115,8 @@ std::ostream &listSeries(Series &series, bool const longer, std::ostream &out) for (auto &[index, i] : series.snapshots()) { + // Necessary only if Series was opened in READ_RANDOM_ACCESS mode + // with `defer_iteration_parsing = true`. if (!i.parsed()) { i.open(); @@ -138,10 +137,7 @@ std::ostream &listSeries(Series &series, bool const longer, std::ostream &out) [](std::pair const &p) { return p.first; }); - if (!i.closed()) - { - i.close(); - } + i.close(); } if (longer) diff --git a/src/snapshots/ContainerImpls.cpp b/src/snapshots/ContainerImpls.cpp index 2a56fc89aa..eb0bc54572 100644 --- a/src/snapshots/ContainerImpls.cpp +++ b/src/snapshots/ContainerImpls.cpp @@ -39,18 +39,7 @@ auto StatefulSnapshotsContainer::get() const -> StatefulIterator const * { return members.m_bufferedIterator.value_or(nullptr); } -auto StatefulSnapshotsContainer::currentIteration() - -> std::optional -{ - if (auto it = get(); it) - { - return it->peekCurrentlyOpenIteration(); - } - else - { - return nullptr; - } -} + auto StatefulSnapshotsContainer::currentIteration() const -> std::optional { @@ -150,15 +139,15 @@ auto StatefulSnapshotsContainer::at(key_type const &key) const auto StatefulSnapshotsContainer::at(key_type const &key) -> mapped_type & { auto base_iterator = get(); - auto result = + auto &result = base_iterator->seek({StatefulIterator::Seek::Seek_Iteration_t{key}}); - if (result->is_end()) + if (result.is_end()) { throw std::out_of_range( "[StatefulSnapshotsContainer::at()] Cannot (yet) skip to " "a Snapshot from an I/O step that is not active."); } - return (*result)->second; + return result->second; } auto StatefulSnapshotsContainer::operator[](key_type const &key) @@ -198,8 +187,9 @@ auto StatefulSnapshotsContainer::operator[](key_type const &key) { s.currentStep.map_during_t( [&](detail::CurrentStep::During_t &during) { - ++during.idx; - base_iterator->get().seen_iterations[key] = during.idx; + ++during.step_count; + base_iterator->get().seen_iterations[key] = + during.step_count; during.iteration_idx = key; during.available_iterations_in_step = {key}; }, @@ -239,15 +229,15 @@ auto StatefulSnapshotsContainer::operator[](key_type const &key) } else if (access::read(access)) { - auto result = base_iterator->seek( + auto &result = base_iterator->seek( {StatefulIterator::Seek::Seek_Iteration_t{key}}); - if (result->is_end()) + if (result.is_end()) { throw std::out_of_range( "[StatefulSnapshotsContainer::operator[]()] Cannot (yet) skip " "to a Snapshot from an I/O step that is not active."); } - return (*result)->second; + return result->second; } throw error::Internal("Control flow error: This should be unreachable."); } @@ -294,6 +284,19 @@ RandomAccessIteratorContainer &RandomAccessIteratorContainer::operator=( RandomAccessIteratorContainer &RandomAccessIteratorContainer::operator=( RandomAccessIteratorContainer &&other) noexcept = default; +auto RandomAccessIteratorContainer::currentIteration() const + -> std::optional +{ + if (auto begin = m_cont.begin(); begin != m_cont.end()) + { + return std::make_optional(&*begin); + } + else + { + return std::nullopt; + } +} + auto RandomAccessIteratorContainer::begin() -> iterator { return OpaqueSeriesIterator( diff --git a/src/snapshots/ContainerTraits.cpp b/src/snapshots/ContainerTraits.cpp index e1957ddc88..01fa7b53cb 100644 --- a/src/snapshots/ContainerTraits.cpp +++ b/src/snapshots/ContainerTraits.cpp @@ -106,12 +106,6 @@ auto AbstractSnapshotsContainer::currentIteration() return std::nullopt; } } -auto AbstractSnapshotsContainer::currentIteration() const - -> std::optional -{ - return std::nullopt; -} - auto AbstractSnapshotsContainer::at(key_type const &key) -> mapped_type & { return const_cast( diff --git a/src/snapshots/StatefulIterator.cpp b/src/snapshots/StatefulIterator.cpp index e61ec9a36c..42e17f34c1 100644 --- a/src/snapshots/StatefulIterator.cpp +++ b/src/snapshots/StatefulIterator.cpp @@ -26,6 +26,7 @@ #include "openPMD/Iteration.hpp" #include "openPMD/Series.hpp" #include "openPMD/auxiliary/Variant.hpp" +#include "openPMD/backend/ParsePreference.hpp" #include #include @@ -40,11 +41,11 @@ namespace openPMD namespace detail { step_status_types::During_t::During_t( - size_t idx_in, + size_t step_count_in, std::optional iteration_idx_in, std::vector available_iterations_in_step_in) - : idx(idx_in) + : step_count(step_count_in) , iteration_idx(iteration_idx_in) , available_iterations_in_step( std::move(available_iterations_in_step_in)) @@ -80,40 +81,18 @@ namespace detail } auto CurrentStep::get_iteration_index() const - -> std::optional + -> std::optional { - using res_t = std::optional; + using res_t = std::optional; return std::visit( auxiliary::overloaded{ [](auto const &) -> res_t { return std::nullopt; }, [](During_t const &during) -> res_t { - if (during.iteration_idx.has_value()) - { - return std::make_optional< - Iteration::IterationIndex_t const *>( - &*during.iteration_idx); - } - else - { - return std::nullopt; - } + return during.iteration_idx; }}, this->as_base()); } - auto CurrentStep::get_iteration_index() - -> std::optional - { - auto res = - static_cast(this)->get_iteration_index(); - if (res.has_value()) - { - return const_cast(*res); - } - else - { - return std::nullopt; - } - } + } // namespace detail StatefulIterator::SharedData::~SharedData() @@ -137,8 +116,8 @@ StatefulIterator::SharedData::~SharedData() if (IOHandler && current_iteration.has_value() && IOHandler && IOHandler->m_lastFlushSuccessful) { - auto lastIterationIndex = **current_iteration; - if (!series.iterations.contains(**current_iteration)) + auto lastIterationIndex = *current_iteration; + if (!series.iterations.contains(*current_iteration)) { return; } @@ -150,13 +129,8 @@ StatefulIterator::SharedData::~SharedData() } } -auto StatefulIterator::SharedData::currentIteration() - -> std::optional -{ - return currentStep.get_iteration_index(); -} auto StatefulIterator::SharedData::currentIteration() const - -> std::optional + -> std::optional { return currentStep.get_iteration_index(); } @@ -330,9 +304,9 @@ auto StatefulIterator::resetCurrentIterationToBegin( auto &data = get(); data.currentStep.map_during_t( [&](CurrentStep::During_t &during) { - during.idx += num_skipped_iterations; + during.step_count += num_skipped_iterations; restrict_to_unseen_iterations( - indexes, data.seen_iterations, during.idx); + indexes, data.seen_iterations, during.step_count); during.available_iterations_in_step = std::move(indexes); if (during.available_iterations_in_step.empty()) { @@ -382,9 +356,7 @@ auto StatefulIterator::peekCurrentlyOpenIteration() const { return std::nullopt; } - // Iteration ¤tIteration = - // s.series.iterations.at(*s.currentIteration); - auto currentIteration = s.series.iterations.find(**maybeCurrentIteration); + auto currentIteration = s.series.iterations.find(*maybeCurrentIteration); if (currentIteration == s.series.iterations.end()) { return std::nullopt; @@ -441,15 +413,13 @@ StatefulIterator::StatefulIterator(tag_write_t, Series const &series_in) auto &data = get(); /* * Since the iterator is stored in - * internal::SeriesData::m_sharedReadIterations, + * internal::SeriesData::m_sharedStatefulIterator, * we need to use a non-owning Series instance here for tie-breaking * purposes. * This is ok due to the usual C++ iterator invalidation workflows * (deleting the original container invalidates the iterator). */ - data.series = Series(); - data.series.setData(std::shared_ptr( - series_in.m_series.get(), [](auto const *) {})); + data.series = series_in.m_attri->asInternalCopyOf(); } StatefulIterator::StatefulIterator( @@ -462,15 +432,14 @@ StatefulIterator::StatefulIterator( data.parsePreference = parsePreference; /* * Since the iterator is stored in - * internal::SeriesData::m_sharedReadIterations, + * internal::SeriesData::m_sharedStatefulIterator, * we need to use a non-owning Series instance here for tie-breaking * purposes. * This is ok due to the usual C++ iterator invalidation workflows * (deleting the original container invalidates the iterator). */ - data.series = Series(); - data.series.setData(std::shared_ptr( - series_in.m_series.get(), [](auto const *) {})); + data.series = series_in.m_attri->asInternalCopyOf(); + auto &series = data.series; if (series.IOHandler()->m_frontendAccess == Access::READ_LINEAR && series.iterations.empty()) @@ -488,7 +457,7 @@ StatefulIterator::StatefulIterator( case IterationEncoding::variableBased: if (!seek({Seek::Next})) { - this->assert_end_iterator(); + throw std::runtime_error("Must not happen"); } break; } @@ -635,7 +604,7 @@ StatefulIterator::nextStep(size_t recursion_depth) if (auto during = data.currentStep.get_variant(); during.has_value()) { - ++(*during)->idx; + ++(*during)->step_count; } resetCurrentIterationToBegin( recursion_depth, std::move(current_iterations)); @@ -662,27 +631,39 @@ std::optional StatefulIterator::loopBody(Seek const &seek) [&](detail::seek_types::Seek_Iteration_t const &go_to_iteration) { return go_to_iteration.iteration_idx != - **maybe_current_iteration; + *maybe_current_iteration; }}, seek.as_base()) && - iterations.contains(**maybe_current_iteration)) + // Iteration might have previously been deleted + iterations.contains(*maybe_current_iteration)) { - auto ¤tIteration = iterations.at(**maybe_current_iteration); + auto ¤tIteration = iterations.at(*maybe_current_iteration); if (!currentIteration.closed()) { currentIteration.close(); } - // sic! only erase if the Iteration was explicitly closed, don't - // implicitly sweep the iteration away from under the user + // sic! only erase if the Iteration was explicitly closed from user + // code, don't implicitly sweep the iteration away from under the + // user else if ( series.IOHandler()->m_frontendAccess == Access::READ_LINEAR) { data.series.iterations.container().erase( - **maybe_current_iteration); + *maybe_current_iteration); } } } + /* + * Some checks and postprocessing: + * + * 1. Do some correctness checks. + * 2. If no data or the end Iterator is returned, then do nothing. + * 3. If the current step has no further data, then end the step and return + * a nullopt. + * 4. If an Iteration is successfully returned, check its close status, + * maybe open it or throw an error if an unexpected status is found. + */ auto guardReturn = [&series, &iterations, &data]( auto const &option) -> std::optional { @@ -703,50 +684,49 @@ std::optional StatefulIterator::loopBody(Seek const &seek) series.advance(AdvanceMode::ENDSTEP); return std::nullopt; } - auto ¤t_iteration = **maybe_current_iteration; - // If we had the iteration already, then it's either not there at all - // (because old iterations are deleted in linear access mode), - // or it's still there but closed in random-access mode + auto ¤t_iteration = *maybe_current_iteration; - if (iterations.contains(current_iteration)) + if (!iterations.contains(current_iteration)) { - auto iteration = iterations.at(current_iteration); - switch (iteration.get().m_closed) - { - case internal::CloseStatus::ParseAccessDeferred: - try - { - iterations.at(current_iteration).open(); - } - catch (error::ReadError const &err) - { - std::cerr << "Cannot read iteration '" << current_iteration - << "' and will skip it due to read error:\n" - << err.what() << std::endl; - option.value()->deactivateDeadIteration(current_iteration); - return std::nullopt; - } - [[fallthrough]]; - case internal::CloseStatus::Open: - return option; - case internal::CloseStatus::Closed: - case internal::CloseStatus::ClosedInFrontend: - throw error::Internal( - "[StatefulIterator] Next step returned an iteration that " - "is already closed, should have opened it."); - } - throw std::runtime_error("Unreachable!"); + throw error::Internal( + "[StatefulIterator::loopBody] An iteration index was returned, " + "but the corresponding Iteration does not exist."); } - else + + auto iteration = iterations.at(current_iteration); + switch (iteration.get().m_closed) { - // we had this iteration already, skip it - series.advance(AdvanceMode::ENDSTEP); - return std::nullopt; + case internal::CloseStatus::ParseAccessDeferred: + try + { + iterations.at(current_iteration).open(); + } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read iteration '" << current_iteration + << "' and will skip it due to read error:\n" + << err.what() << std::endl; + option.value()->deactivateDeadIteration(current_iteration); + return std::nullopt; + } + [[fallthrough]]; + case internal::CloseStatus::Open: + return option; + case internal::CloseStatus::Closed: + case internal::CloseStatus::ClosedInFrontend: + throw error::Internal( + "[StatefulIterator] Next step returned an iteration that " + "is already closed, should have opened it."); } + throw std::runtime_error("Unreachable!"); }; + /* + * First check if the requested Iteration can be found within the current + * step. + */ { - std::optional optionallyAStep = std::visit( + std::optional optionallyAnIteration = std::visit( auxiliary::overloaded{ [&](Seek::Next_t) { return nextIterationInStep(); }, [&](Seek::Seek_Iteration_t const &skip_to_iteration) { @@ -754,22 +734,37 @@ std::optional StatefulIterator::loopBody(Seek const &seek) skip_to_iteration.iteration_idx); }}, seek.as_base()); - if (optionallyAStep.has_value()) + if (optionallyAnIteration.has_value()) { - return guardReturn(optionallyAStep); + return guardReturn(optionallyAnIteration); } } - // The currently active iterations have been exhausted. - // Now see if there are further iterations to be found. + /* + * The current step does not contain the requested data. In file-based + * iteration encoding or in UpFront parse mode (i.e. no steps), this means + * that we've reached the end now. + */ - if (series.iterationEncoding() == IterationEncoding::fileBased) + if (series.iterationEncoding() == IterationEncoding::fileBased || + (data.parsePreference == internal::ParsePreference::UpFront && + // If the step status is still Begin, the Series is currently being + // initiated. In group-/variable-based encoding, a step needs to be + // begun at least once. This is because a Series with + // ParsePreference::UpFront is modeled as a Series with one big step + // that contains all Iterations. + std::holds_alternative( + data.currentStep.as_base()))) { // this one is handled above, stream is over once it proceeds to here this->turn_into_end_iterator(TypeOfEndIterator::NoMoreIterationsInStep); return {this}; } + /* + * The currently active iterations have been exhausted. + * Now see if there are further iterations to be found in next step. + */ auto option = std::visit( auxiliary::overloaded{ [&](Seek::Next_t) { return nextStep(/* recursion_depth = */ 0); }, @@ -854,10 +849,10 @@ void StatefulIterator::deactivateDeadIteration(iteration_index_t index) StatefulIterator &StatefulIterator::operator++() { - return *seek({Seek::Next}); + return seek({Seek::Next}); } -auto StatefulIterator::seek(Seek const &seek) -> StatefulIterator * +auto StatefulIterator::seek(Seek const &seek) -> StatefulIterator & { std::optional res; /* @@ -872,7 +867,7 @@ auto StatefulIterator::seek(Seek const &seek) -> StatefulIterator * res = loopBody(seek); } while (!res.has_value()); - return *res; + return **res; } auto StatefulIterator::operator*() -> value_type & @@ -889,7 +884,7 @@ auto StatefulIterator::operator*() const -> value_type const & auto iterator = static_cast( data.series.iterations) - .find(**cur); + .find(*cur); return iterator.operator*(); } else From 09c0a64b68b98d75bf8fe3cbeb5a72c7618cb19b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 8 Jul 2024 17:08:41 +0200 Subject: [PATCH 89/93] Add check_recursive_include script --- check_recursive_include.py | 50 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100755 check_recursive_include.py diff --git a/check_recursive_include.py b/check_recursive_include.py new file mode 100755 index 0000000000..e5b37cb6e9 --- /dev/null +++ b/check_recursive_include.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 + +import re +import subprocess +import sys + + +def track_includes(start_file): + remove_start = re.compile("include/") + + def clean(file): + return re.sub(remove_start, "", file) + + unseen_files = [start_file] + res = {} + # import ipdb + # ipdb.set_trace(context=30) + while unseen_files: + current_file = unseen_files[0] + del unseen_files[0] + + cmd = ["grep", "-Rl", clean(current_file), "include/"] + try: + next_files = subprocess.check_output(cmd) + lines = [line for line in next_files.decode().splitlines() if line] + except subprocess.CalledProcessError: + lines = [] + + res[current_file] = lines + for line in lines: + if line not in res: + unseen_files.append(line) + return res + + +if __name__ == "__main__": + + remove_start = re.compile("include/(openPMD/)?") + remove_slash = re.compile("/|\\.") + + def clean(file): + return re.sub(remove_slash, "_", re.sub(remove_start, "", file)) + + res = track_includes(sys.argv[1]) + print("digraph {") + for key, values in res.items(): + key = clean(key) + for target in values: + print("\t{} -> {};".format(key, clean(target))) + print("}") From de4b6626b584f9f3f12059d3cf9765b75780e02c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 11 Jul 2024 15:27:59 +0200 Subject: [PATCH 90/93] Fixes after rebase --- src/Series.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Series.cpp b/src/Series.cpp index 49c65bfbdf..453442ef7c 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -1835,9 +1835,9 @@ void Series::readOneIterationFileBased(std::string const &filePath) {}, "Unknown iterationEncoding: " + encoding); auto old_written = written(); - written() = false; + setWritten(false, Attributable::EnqueueAsynchronously::No); setIterationEncoding(encoding_out); - written() = old_written; + setWritten(old_written, Attributable::EnqueueAsynchronously::Yes); } else throw std::runtime_error( From 655c932141631b1a2adafe4a1523e080adfaa199 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 16 Jul 2024 17:05:00 +0200 Subject: [PATCH 91/93] Fix bug that hindered files from being properly closed --- src/Series.cpp | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/Series.cpp b/src/Series.cpp index 453442ef7c..17269feb63 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -2492,11 +2492,8 @@ AdvanceStatus Series::advance( switch (series.m_iterationEncoding) { case IE::fileBased: { - if (itData.m_closed != internal::CloseStatus::Closed) - { - Parameter fClose; - IOHandler()->enqueue(IOTask(&iteration, std::move(fClose))); - } + Parameter fClose; + IOHandler()->enqueue(IOTask(&iteration, std::move(fClose))); itData.m_closed = internal::CloseStatus::Closed; break; } From 009901062ead4ce126fe091cec4a98b86546fcb5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 17 Jul 2024 16:08:07 +0200 Subject: [PATCH 92/93] Will this fix the Windows CI errors I dont think so --- test/Files_SerialIO/filebased_write_test.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/Files_SerialIO/filebased_write_test.cpp b/test/Files_SerialIO/filebased_write_test.cpp index 0a7109ca2f..ba2780a71e 100644 --- a/test/Files_SerialIO/filebased_write_test.cpp +++ b/test/Files_SerialIO/filebased_write_test.cpp @@ -5,11 +5,11 @@ namespace filebased_write_test { using namespace openPMD; -void close_and_reopen_iterations( +auto close_and_reopen_iterations( const std::string &filename, openPMD::Access access, std::string const &json_config, - bool need_to_explitly_open_iterations) + bool need_to_explitly_open_iterations) -> void { Series list(filename, access, json_config); @@ -77,7 +77,7 @@ void close_and_reopen_iterations( } } -void close_and_reopen_iterations(std::string const &filename) +auto close_and_reopen_iterations(std::string const &filename) -> void { close_and_reopen_iterations( filename, From bfc9897c151b133da0381dc1e4bdcb46da0f7dcb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 18 Jul 2024 11:47:57 +0200 Subject: [PATCH 93/93] Use macro instead of function Proper return() is supported beginning with CMake 3.25 only --- CMakeLists.txt | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 46fb4d95ff..20636f9fd8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -25,12 +25,6 @@ if(POLICY CMP0127) cmake_policy(SET CMP0127 NEW) endif() -# return() should check its arguments -# https://cmake.org/cmake/help/latest/policy/CMP0140.html -if(POLICY CMP0140) - cmake_policy(SET CMP0140 NEW) -endif() - # No in-Source builds ######################################################### # @@ -861,15 +855,14 @@ if(openPMD_BUILD_TESTING) target_compile_definitions(CatchRunner PUBLIC openPMD_HAVE_MPI=1) endif() - function(additional_testing_sources test_name out_list) + macro(additional_testing_sources test_name out_list) if(${test_name} STREQUAL "SerialIO") list(APPEND ${out_list} test/Files_SerialIO/close_iteration_test.cpp test/Files_SerialIO/filebased_write_test.cpp ) endif() - return(PROPAGATE ${out_list}) - endfunction() + endmacro() foreach(testname ${openPMD_TEST_NAMES}) set(ADDITIONAL_SOURCE_FILES "")