diff --git a/include/openPMD/IO/ADIOS/ADIOS1IOHandler.hpp b/include/openPMD/IO/ADIOS/ADIOS1IOHandler.hpp index 7491c122f3..98cbdcb86e 100644 --- a/include/openPMD/IO/ADIOS/ADIOS1IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS1IOHandler.hpp @@ -49,7 +49,7 @@ class OPENPMDAPI_EXPORT ADIOS1IOHandler : public AbstractIOHandler return "ADIOS1"; } - std::future flush() override; + std::future flush(internal::FlushParams const &) override; void enqueue(IOTask const &) override; @@ -71,7 +71,7 @@ class OPENPMDAPI_EXPORT ADIOS1IOHandler : public AbstractIOHandler return "DUMMY_ADIOS1"; } - std::future flush() override; + std::future flush(internal::FlushParams const &) override; private: std::unique_ptr m_impl; diff --git a/include/openPMD/IO/ADIOS/ADIOS1IOHandlerImpl.hpp b/include/openPMD/IO/ADIOS/ADIOS1IOHandlerImpl.hpp index e749e2b71c..d7e0d921e9 100644 --- a/include/openPMD/IO/ADIOS/ADIOS1IOHandlerImpl.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS1IOHandlerImpl.hpp @@ -48,7 +48,7 @@ class OPENPMDAPI_EXPORT ADIOS1IOHandlerImpl : public AbstractIOHandlerImpl virtual void init(); - std::future flush() override; + std::future flush(); void createFile(Writable *, Parameter const &) override; diff --git a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp index c5935ac9cf..f789c9aca9 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp @@ -140,7 +140,7 @@ class ADIOS2IOHandlerImpl ~ADIOS2IOHandlerImpl() override; - std::future flush() override; + std::future flush(internal::FlushParams const &); void createFile(Writable *, Parameter const &) override; @@ -1323,7 +1323,7 @@ class ADIOS2IOHandler : public AbstractIOHandler // we must not throw in a destructor try { - this->flush(); + this->flush(internal::defaultFlushParams); } catch (std::exception const &ex) { @@ -1362,6 +1362,6 @@ class ADIOS2IOHandler : public AbstractIOHandler return "ADIOS2"; } - std::future flush() override; + std::future flush(internal::FlushParams const &) override; }; // ADIOS2IOHandler } // namespace openPMD diff --git a/include/openPMD/IO/ADIOS/ParallelADIOS1IOHandler.hpp b/include/openPMD/IO/ADIOS/ParallelADIOS1IOHandler.hpp index 41cb38cb77..20ee1458a5 100644 --- a/include/openPMD/IO/ADIOS/ParallelADIOS1IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ParallelADIOS1IOHandler.hpp @@ -52,7 +52,7 @@ class OPENPMDAPI_EXPORT ParallelADIOS1IOHandler : public AbstractIOHandler return "MPI_ADIOS1"; } - std::future flush() override; + std::future flush(internal::FlushParams const &) override; #if openPMD_HAVE_ADIOS1 void enqueue(IOTask const &) override; #endif diff --git a/include/openPMD/IO/ADIOS/ParallelADIOS1IOHandlerImpl.hpp b/include/openPMD/IO/ADIOS/ParallelADIOS1IOHandlerImpl.hpp index 6d001cb9fd..8e37d6872c 100644 --- a/include/openPMD/IO/ADIOS/ParallelADIOS1IOHandlerImpl.hpp +++ b/include/openPMD/IO/ADIOS/ParallelADIOS1IOHandlerImpl.hpp @@ -50,7 +50,7 @@ class OPENPMDAPI_EXPORT ParallelADIOS1IOHandlerImpl virtual void init(); - std::future flush() override; + std::future flush(); void createFile(Writable *, Parameter const &) override; diff --git a/include/openPMD/IO/AbstractIOHandler.hpp b/include/openPMD/IO/AbstractIOHandler.hpp index dafa896a76..a7239a3375 100644 --- a/include/openPMD/IO/AbstractIOHandler.hpp +++ b/include/openPMD/IO/AbstractIOHandler.hpp @@ -61,7 +61,9 @@ class unsupported_data_error : public std::runtime_error * @brief Determine what items should be flushed upon Series::flush() * */ -enum class FlushLevel : unsigned char +// do not write `enum class FlushLevel : unsigned char` here since NVHPC +// does not compile it correctly +enum class FlushLevel { /** * Flush operation that was triggered by user code. @@ -84,9 +86,31 @@ enum class FlushLevel : unsigned char * CREATE_DATASET tasks. * Attributes may or may not be flushed yet. */ - SkeletonOnly + SkeletonOnly, + /** + * Only creates/opens files, nothing more + */ + CreateOrOpenFiles }; +namespace internal +{ + /** + * Parameters recursively passed through the openPMD hierarchy when + * flushing. + * + */ + struct FlushParams + { + FlushLevel flushLevel = FlushLevel::InternalFlush; + }; + + /* + * To be used for reading + */ + constexpr FlushParams defaultFlushParams{}; +} // namespace internal + /** Interface for communicating between logical and physically persistent data. * * Input and output operations are channeled through a task queue that is @@ -123,7 +147,7 @@ class AbstractIOHandler * @return Future indicating the completion state of the operation for * backends that decide to implement this operation asynchronously. */ - virtual std::future flush() = 0; + virtual std::future flush(internal::FlushParams const &) = 0; /** The currently used backend */ virtual std::string backendName() const = 0; @@ -132,7 +156,6 @@ class AbstractIOHandler Access const m_backendAccess; Access const m_frontendAccess; std::queue m_work; - FlushLevel m_flushLevel = FlushLevel::InternalFlush; }; // AbstractIOHandler } // namespace openPMD diff --git a/include/openPMD/IO/AbstractIOHandlerImpl.hpp b/include/openPMD/IO/AbstractIOHandlerImpl.hpp index 01aeb4e9fc..170cf4b81a 100644 --- a/include/openPMD/IO/AbstractIOHandlerImpl.hpp +++ b/include/openPMD/IO/AbstractIOHandlerImpl.hpp @@ -40,7 +40,7 @@ class AbstractIOHandlerImpl virtual ~AbstractIOHandlerImpl() = default; - virtual std::future flush() + std::future flush() { using namespace auxiliary; diff --git a/include/openPMD/IO/DummyIOHandler.hpp b/include/openPMD/IO/DummyIOHandler.hpp index 8a84bb0919..9a4f3c3852 100644 --- a/include/openPMD/IO/DummyIOHandler.hpp +++ b/include/openPMD/IO/DummyIOHandler.hpp @@ -44,6 +44,6 @@ class DummyIOHandler : public AbstractIOHandler /** No-op consistent with the IOHandler interface to enable library use * without IO. */ - std::future flush() override; + std::future flush(internal::FlushParams const &) override; }; // DummyIOHandler } // namespace openPMD diff --git a/include/openPMD/IO/HDF5/HDF5IOHandler.hpp b/include/openPMD/IO/HDF5/HDF5IOHandler.hpp index 59bbffd4c5..da1f1ff130 100644 --- a/include/openPMD/IO/HDF5/HDF5IOHandler.hpp +++ b/include/openPMD/IO/HDF5/HDF5IOHandler.hpp @@ -43,7 +43,7 @@ class HDF5IOHandler : public AbstractIOHandler return "HDF5"; } - std::future flush() override; + std::future flush(internal::FlushParams const &) override; private: std::unique_ptr m_impl; diff --git a/include/openPMD/IO/HDF5/ParallelHDF5IOHandler.hpp b/include/openPMD/IO/HDF5/ParallelHDF5IOHandler.hpp index fd94d87232..cc660464fc 100644 --- a/include/openPMD/IO/HDF5/ParallelHDF5IOHandler.hpp +++ b/include/openPMD/IO/HDF5/ParallelHDF5IOHandler.hpp @@ -49,7 +49,7 @@ class ParallelHDF5IOHandler : public AbstractIOHandler return "MPI_HDF5"; } - std::future flush() override; + std::future flush(internal::FlushParams const &) override; private: std::unique_ptr m_impl; diff --git a/include/openPMD/IO/JSON/JSONIOHandler.hpp b/include/openPMD/IO/JSON/JSONIOHandler.hpp index 1c1302bb55..37b00fa165 100644 --- a/include/openPMD/IO/JSON/JSONIOHandler.hpp +++ b/include/openPMD/IO/JSON/JSONIOHandler.hpp @@ -38,7 +38,7 @@ class JSONIOHandler : public AbstractIOHandler return "JSON"; } - std::future flush() override; + std::future flush(internal::FlushParams const &) override; private: JSONIOHandlerImpl m_impl; diff --git a/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp index f0927b36b7..23ac579401 100644 --- a/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp +++ b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp @@ -210,7 +210,7 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl void listAttributes(Writable *, Parameter &) override; - std::future flush() override; + std::future flush(); private: using FILEHANDLE = std::fstream; diff --git a/include/openPMD/Iteration.hpp b/include/openPMD/Iteration.hpp index 18afb6e0e1..3dad0772f1 100644 --- a/include/openPMD/Iteration.hpp +++ b/include/openPMD/Iteration.hpp @@ -177,22 +177,26 @@ class Iteration : public LegacyAttributable * containing this iteration. */ std::string filename; + bool beginStep = false; }; - void flushFileBased(std::string const &, uint64_t); - void flushGroupBased(uint64_t); - void flushVariableBased(uint64_t); - void flush(); + void flushFileBased( + std::string const &, uint64_t, internal::FlushParams const &); + void flushGroupBased(uint64_t, internal::FlushParams const &); + void flushVariableBased(uint64_t, internal::FlushParams const &); + void flush(internal::FlushParams const &); void deferParseAccess(DeferredParseAccess); /* - * Control flow for read(), readFileBased(), readGroupBased() and - * read_impl(): - * read() is called as the entry point. File-based and group-based + * Control flow for runDeferredParseAccess(), readFileBased(), + * readGroupBased() and read_impl(): + * runDeferredParseAccess() is called as the entry point. + * File-based and group-based * iteration layouts need to be parsed slightly differently: * In file-based iteration layout, each iteration's file also contains * attributes for the /data group. In group-based layout, those have * already been parsed during opening of the Series. - * Hence, read() will call either readFileBased() or readGroupBased() to + * Hence, runDeferredParseAccess() will call either readFileBased() or + * readGroupBased() to * allow for those different control flows. * Finally, read_impl() is called which contains the common parsing * logic for an iteration. @@ -201,10 +205,10 @@ class Iteration : public LegacyAttributable * Calling it on an Iteration not yet parsed is an error. * */ - void read(); void reread(std::string const &path); - void readFileBased(std::string filePath, std::string const &groupPath); - void readGorVBased(std::string const &groupPath); + void readFileBased( + std::string filePath, std::string const &groupPath, bool beginStep); + void readGorVBased(std::string const &groupPath, bool beginStep); void read_impl(std::string const &groupPath); /** @@ -261,7 +265,7 @@ class Iteration : public LegacyAttributable * * @return AdvanceStatus */ - AdvanceStatus beginStep(); + AdvanceStatus beginStep(bool reread); /** * @brief End an IO step on the IO file (or file-like object) diff --git a/include/openPMD/Mesh.hpp b/include/openPMD/Mesh.hpp index 1dac4760ee..17ce9373de 100644 --- a/include/openPMD/Mesh.hpp +++ b/include/openPMD/Mesh.hpp @@ -228,7 +228,8 @@ class Mesh : public BaseRecord private: Mesh(); - void flush_impl(std::string const &) override; + void + flush_impl(std::string const &, internal::FlushParams const &) override; void read() override; }; // Mesh diff --git a/include/openPMD/ParticleSpecies.hpp b/include/openPMD/ParticleSpecies.hpp index fc80960ca5..0257cd474f 100644 --- a/include/openPMD/ParticleSpecies.hpp +++ b/include/openPMD/ParticleSpecies.hpp @@ -43,7 +43,7 @@ class ParticleSpecies : public Container ParticleSpecies(); void read(); - void flush(std::string const &) override; + void flush(std::string const &, internal::FlushParams const &) override; /** * @brief Check recursively whether this ParticleSpecies is dirty. diff --git a/include/openPMD/Record.hpp b/include/openPMD/Record.hpp index 10bf0a5666..4f7ee51c28 100644 --- a/include/openPMD/Record.hpp +++ b/include/openPMD/Record.hpp @@ -50,7 +50,8 @@ class Record : public BaseRecord private: Record(); - void flush_impl(std::string const &) override; + void + flush_impl(std::string const &, internal::FlushParams const &) override; void read() override; }; // Record diff --git a/include/openPMD/RecordComponent.hpp b/include/openPMD/RecordComponent.hpp index c5d6fb41ee..f1c8cae451 100644 --- a/include/openPMD/RecordComponent.hpp +++ b/include/openPMD/RecordComponent.hpp @@ -250,7 +250,7 @@ class RecordComponent : public BaseRecordComponent std::shared_ptr m_hasBeenExtended = std::make_shared(false); private: - void flush(std::string const &); + void flush(std::string const &, internal::FlushParams const &); virtual void read(); /** diff --git a/include/openPMD/RecordComponent.tpp b/include/openPMD/RecordComponent.tpp index 92ec2beb38..46ef527311 100644 --- a/include/openPMD/RecordComponent.tpp +++ b/include/openPMD/RecordComponent.tpp @@ -276,7 +276,7 @@ RecordComponent::storeChunk( Offset o, Extent e, F && createBuffer ) * Flush the openPMD hierarchy to the backend without flushing any actual * data yet. */ - seriesFlush( FlushLevel::SkeletonOnly ); + seriesFlush({FlushLevel::SkeletonOnly}); size_t size = 1; for( auto ext : e ) @@ -303,16 +303,16 @@ RecordComponent::storeChunk( Offset o, Extent e, F && createBuffer ) getBufferView.offset = o; getBufferView.extent = e; getBufferView.dtype = getDatatype(); - IOHandler()->enqueue( IOTask( this, getBufferView ) ); - IOHandler()->flush(); + IOHandler()->enqueue(IOTask(this, getBufferView)); + IOHandler()->flush(internal::defaultFlushParams); auto &out = *getBufferView.out; - if( !out.backendManagedBuffer ) + if (!out.backendManagedBuffer) { - auto data = std::forward< F >( createBuffer )( size ); - out.ptr = static_cast< void * >( data.get() ); - storeChunk( std::move( data ), std::move( o ), std::move( e ) ); + auto data = std::forward(createBuffer)(size); + out.ptr = static_cast(data.get()); + storeChunk(std::move(data), std::move(o), std::move(e)); } - return DynamicMemoryView< T >{ std::move( getBufferView ), size, *this }; + return DynamicMemoryView{std::move(getBufferView), size, *this}; } template< typename T > diff --git a/include/openPMD/Series.hpp b/include/openPMD/Series.hpp index 2ff8b74ce3..8cce096c9d 100644 --- a/include/openPMD/Series.hpp +++ b/include/openPMD/Series.hpp @@ -408,16 +408,19 @@ class SeriesInterface : public AttributableInterface * * @param begin Start of the range of iterations to flush. * @param end End of the range of iterations to flush. - * @param level Flush level, as documented in AbstractIOHandler.hpp. + * @param flushParams Flush params, as documented in AbstractIOHandler.hpp. * @param flushIOHandler Tasks will always be enqueued to the backend. * If this flag is true, tasks will be flushed to the backend. */ std::future flush_impl( iterations_iterator begin, iterations_iterator end, - FlushLevel level, + internal::FlushParams flushParams, bool flushIOHandler = true); - void flushFileBased(iterations_iterator begin, iterations_iterator end); + void flushFileBased( + iterations_iterator begin, + iterations_iterator end, + internal::FlushParams flushParams); /* * Group-based and variable-based iteration layouts share a lot of logic * (realistically, the variable-based iteration layout only throws out @@ -425,7 +428,10 @@ class SeriesInterface : public AttributableInterface * As a convention, methods that deal with both layouts are called * .*GorVBased, short for .*GroupOrVariableBased */ - void flushGorVBased(iterations_iterator begin, iterations_iterator end); + void flushGorVBased( + iterations_iterator begin, + iterations_iterator end, + internal::FlushParams flushParams); void flushMeshesPath(); void flushParticlesPath(); void readFileBased(); diff --git a/include/openPMD/Span.hpp b/include/openPMD/Span.hpp index 64b6086943..11325aea52 100644 --- a/include/openPMD/Span.hpp +++ b/include/openPMD/Span.hpp @@ -123,7 +123,7 @@ class DynamicMemoryView // might need to update m_recordComponent.IOHandler()->enqueue( IOTask(&m_recordComponent, m_param)); - m_recordComponent.IOHandler()->flush(); + m_recordComponent.IOHandler()->flush(internal::defaultFlushParams); } return Span{static_cast(m_param.out->ptr), m_size}; } diff --git a/include/openPMD/backend/Attributable.hpp b/include/openPMD/backend/Attributable.hpp index d3b8411aff..9ae989e090 100644 --- a/include/openPMD/backend/Attributable.hpp +++ b/include/openPMD/backend/Attributable.hpp @@ -264,9 +264,9 @@ class AttributableInterface Iteration &containingIteration(); /** @} */ - void seriesFlush(FlushLevel); + void seriesFlush(internal::FlushParams); - void flushAttributes(); + void flushAttributes(internal::FlushParams const &); enum ReadMode { /** diff --git a/include/openPMD/backend/BaseRecord.hpp b/include/openPMD/backend/BaseRecord.hpp index 7d4ffbdaa8..4e7e5d70bb 100644 --- a/include/openPMD/backend/BaseRecord.hpp +++ b/include/openPMD/backend/BaseRecord.hpp @@ -95,8 +95,9 @@ class BaseRecord : public Container std::shared_ptr m_containsScalar; private: - void flush(std::string const &) final; - virtual void flush_impl(std::string const &) = 0; + void flush(std::string const &, internal::FlushParams const &) final; + virtual void + flush_impl(std::string const &, internal::FlushParams const &) = 0; virtual void read() = 0; /** @@ -200,7 +201,7 @@ BaseRecord::erase(key_type const &key) Parameter dDelete; dDelete.name = "."; this->IOHandler()->enqueue(IOTask(&rc, dDelete)); - this->IOHandler()->flush(); + this->IOHandler()->flush(internal::defaultFlushParams); } res = Container::erase(key); } @@ -230,7 +231,7 @@ BaseRecord::erase(iterator res) Parameter dDelete; dDelete.name = "."; this->IOHandler()->enqueue(IOTask(&rc, dDelete)); - this->IOHandler()->flush(); + this->IOHandler()->flush(internal::defaultFlushParams); } ret = Container::erase(res); } @@ -265,7 +266,7 @@ inline void BaseRecord::readBase() aRead.name = "unitDimension"; this->IOHandler()->enqueue(IOTask(this, aRead)); - this->IOHandler()->flush(); + this->IOHandler()->flush(internal::defaultFlushParams); if (*aRead.dtype == DT::ARR_DBL_7) this->setAttribute( "unitDimension", @@ -290,7 +291,7 @@ inline void BaseRecord::readBase() aRead.name = "timeOffset"; this->IOHandler()->enqueue(IOTask(this, aRead)); - this->IOHandler()->flush(); + this->IOHandler()->flush(internal::defaultFlushParams); if (*aRead.dtype == DT::FLOAT) this->setAttribute( "timeOffset", Attribute(*aRead.resource).template get()); @@ -303,7 +304,8 @@ inline void BaseRecord::readBase() } template -inline void BaseRecord::flush(std::string const &name) +inline void BaseRecord::flush( + std::string const &name, internal::FlushParams const &flushParams) { if (!this->written() && this->empty()) throw std::runtime_error( @@ -311,7 +313,7 @@ inline void BaseRecord::flush(std::string const &name) "RecordComponents: " + name); - this->flush_impl(name); + this->flush_impl(name, flushParams); // flush_impl must take care to correctly set the dirty() flag so this // method doesn't do it } diff --git a/include/openPMD/backend/Container.hpp b/include/openPMD/backend/Container.hpp index 82038a6eaf..3ed44d4f04 100644 --- a/include/openPMD/backend/Container.hpp +++ b/include/openPMD/backend/Container.hpp @@ -344,7 +344,7 @@ class Container : public LegacyAttributable Parameter pDelete; pDelete.path = "."; IOHandler()->enqueue(IOTask(&res->second, pDelete)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); } return m_container->erase(key); } @@ -361,7 +361,7 @@ class Container : public LegacyAttributable Parameter pDelete; pDelete.path = "."; IOHandler()->enqueue(IOTask(&res->second, pDelete)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); } return m_container->erase(res); } @@ -389,7 +389,8 @@ class Container : public LegacyAttributable m_container->clear(); } - virtual void flush(std::string const &path) + virtual void + flush(std::string const &path, internal::FlushParams const &flushParams) { if (!written()) { @@ -398,7 +399,7 @@ class Container : public LegacyAttributable IOHandler()->enqueue(IOTask(this, pCreate)); } - flushAttributes(); + flushAttributes(flushParams); } std::shared_ptr m_container; diff --git a/include/openPMD/backend/PatchRecord.hpp b/include/openPMD/backend/PatchRecord.hpp index 27ddb0ba96..84d180bac5 100644 --- a/include/openPMD/backend/PatchRecord.hpp +++ b/include/openPMD/backend/PatchRecord.hpp @@ -41,7 +41,8 @@ class PatchRecord : public BaseRecord private: PatchRecord() = default; - void flush_impl(std::string const &) override; + void + flush_impl(std::string const &, internal::FlushParams const &) override; void read() override; }; // PatchRecord } // namespace openPMD diff --git a/include/openPMD/backend/PatchRecordComponent.hpp b/include/openPMD/backend/PatchRecordComponent.hpp index 935ab7a92b..87b9f625b5 100644 --- a/include/openPMD/backend/PatchRecordComponent.hpp +++ b/include/openPMD/backend/PatchRecordComponent.hpp @@ -66,7 +66,7 @@ class PatchRecordComponent : public BaseRecordComponent OPENPMD_private: PatchRecordComponent(); - void flush(std::string const &); + void flush(std::string const &, internal::FlushParams const &); void read(); std::shared_ptr > m_chunks; diff --git a/include/openPMD/backend/Writable.hpp b/include/openPMD/backend/Writable.hpp index 243dce135a..114e2f7566 100644 --- a/include/openPMD/backend/Writable.hpp +++ b/include/openPMD/backend/Writable.hpp @@ -107,16 +107,16 @@ class Writable final void seriesFlush(); OPENPMD_private: - void seriesFlush( FlushLevel ); + void seriesFlush(internal::FlushParams); /* * These members need to be shared pointers since distinct instances of * Writable may share them. */ - std::shared_ptr abstractFilePosition; - std::shared_ptr IOHandler; - internal::AttributableData *attributable; - Writable *parent; - bool dirty; + std::shared_ptr abstractFilePosition = nullptr; + std::shared_ptr IOHandler = nullptr; + internal::AttributableData *attributable = nullptr; + Writable *parent = nullptr; + bool dirty = true; /** * If parent is not null, then this is a vector of keys such that: * &(*parent)[key_1]...[key_n] == this @@ -139,6 +139,6 @@ class Writable final * Writable and its meaning within the current dataset. * */ - bool written; + bool written = false; }; } // namespace openPMD diff --git a/src/IO/ADIOS/ADIOS1IOHandler.cpp b/src/IO/ADIOS/ADIOS1IOHandler.cpp index 863f8b2c95..cafe9f8750 100644 --- a/src/IO/ADIOS/ADIOS1IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS1IOHandler.cpp @@ -329,7 +329,7 @@ ADIOS1IOHandler::ADIOS1IOHandler(std::string path, Access at) ADIOS1IOHandler::~ADIOS1IOHandler() = default; -std::future ADIOS1IOHandler::flush() +std::future ADIOS1IOHandler::flush(internal::FlushParams const &) { return m_impl->flush(); } @@ -436,7 +436,7 @@ ADIOS1IOHandler::ADIOS1IOHandler(std::string path, Access at) ADIOS1IOHandler::~ADIOS1IOHandler() = default; -std::future ADIOS1IOHandler::flush() +std::future ADIOS1IOHandler::flush(internal::FlushParams const &) { return std::future(); } diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index a155486851..53df53b865 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -234,7 +234,8 @@ std::string ADIOS2IOHandlerImpl::fileSuffix() const } } -std::future ADIOS2IOHandlerImpl::flush() +std::future +ADIOS2IOHandlerImpl::flush(internal::FlushParams const &flushParams) { auto res = AbstractIOHandlerImpl::flush(); for (auto &p : m_fileData) @@ -242,7 +243,7 @@ std::future ADIOS2IOHandlerImpl::flush() if (m_dirty.find(p.first) != m_dirty.end()) { p.second->flush( - m_handler->m_flushLevel, /* writeAttributes = */ false); + flushParams.flushLevel, /* writeAttributes = */ false); } else { @@ -2623,6 +2624,7 @@ namespace detail case FlushLevel::InternalFlush: case FlushLevel::SkeletonOnly: + case FlushLevel::CreateOrOpenFiles: /* * Tasks have been given to ADIOS2, but we don't flush them * yet. So, move everything to m_alreadyEnqueued to avoid @@ -2876,9 +2878,10 @@ ADIOS2IOHandler::ADIOS2IOHandler( , m_impl{this, std::move(options), std::move(engineType)} {} -std::future ADIOS2IOHandler::flush() +std::future +ADIOS2IOHandler::flush(internal::FlushParams const &flushParams) { - return m_impl.flush(); + return m_impl.flush(flushParams); } #else // openPMD_HAVE_ADIOS2 @@ -2896,7 +2899,7 @@ ADIOS2IOHandler::ADIOS2IOHandler( : AbstractIOHandler(std::move(path), at) {} -std::future ADIOS2IOHandler::flush() +std::future ADIOS2IOHandler::flush(internal::FlushParams const &) { return std::future(); } diff --git a/src/IO/ADIOS/ParallelADIOS1IOHandler.cpp b/src/IO/ADIOS/ParallelADIOS1IOHandler.cpp index 7713e207cb..722d25238c 100644 --- a/src/IO/ADIOS/ParallelADIOS1IOHandler.cpp +++ b/src/IO/ADIOS/ParallelADIOS1IOHandler.cpp @@ -353,7 +353,7 @@ ParallelADIOS1IOHandler::ParallelADIOS1IOHandler( ParallelADIOS1IOHandler::~ParallelADIOS1IOHandler() = default; -std::future ParallelADIOS1IOHandler::flush() +std::future ParallelADIOS1IOHandler::flush(internal::FlushParams const &) { return m_impl->flush(); } @@ -480,7 +480,7 @@ ParallelADIOS1IOHandler::ParallelADIOS1IOHandler(std::string path, Access at) ParallelADIOS1IOHandler::~ParallelADIOS1IOHandler() = default; -std::future ParallelADIOS1IOHandler::flush() +std::future ParallelADIOS1IOHandler::flush(internal::FlushParams const &) { return std::future(); } diff --git a/src/IO/DummyIOHandler.cpp b/src/IO/DummyIOHandler.cpp index f10cd50ac9..308f584ce4 100644 --- a/src/IO/DummyIOHandler.cpp +++ b/src/IO/DummyIOHandler.cpp @@ -32,7 +32,7 @@ DummyIOHandler::DummyIOHandler(std::string path, Access at) void DummyIOHandler::enqueue(IOTask const &) {} -std::future DummyIOHandler::flush() +std::future DummyIOHandler::flush(internal::FlushParams const &) { return std::future(); } diff --git a/src/IO/HDF5/HDF5IOHandler.cpp b/src/IO/HDF5/HDF5IOHandler.cpp index dc642fa209..10b7651f8f 100644 --- a/src/IO/HDF5/HDF5IOHandler.cpp +++ b/src/IO/HDF5/HDF5IOHandler.cpp @@ -2285,7 +2285,7 @@ HDF5IOHandler::HDF5IOHandler(std::string path, Access at, nlohmann::json config) HDF5IOHandler::~HDF5IOHandler() = default; -std::future HDF5IOHandler::flush() +std::future HDF5IOHandler::flush(internal::FlushParams const &) { return m_impl->flush(); } @@ -2299,7 +2299,7 @@ HDF5IOHandler::HDF5IOHandler( HDF5IOHandler::~HDF5IOHandler() = default; -std::future HDF5IOHandler::flush() +std::future HDF5IOHandler::flush(internal::FlushParams const &) { return std::future(); } diff --git a/src/IO/HDF5/ParallelHDF5IOHandler.cpp b/src/IO/HDF5/ParallelHDF5IOHandler.cpp index df251c7c96..7557054f41 100644 --- a/src/IO/HDF5/ParallelHDF5IOHandler.cpp +++ b/src/IO/HDF5/ParallelHDF5IOHandler.cpp @@ -54,7 +54,7 @@ ParallelHDF5IOHandler::ParallelHDF5IOHandler( ParallelHDF5IOHandler::~ParallelHDF5IOHandler() = default; -std::future ParallelHDF5IOHandler::flush() +std::future ParallelHDF5IOHandler::flush(internal::FlushParams const &) { return m_impl->flush(); } @@ -196,7 +196,7 @@ ParallelHDF5IOHandler::ParallelHDF5IOHandler( ParallelHDF5IOHandler::~ParallelHDF5IOHandler() = default; -std::future ParallelHDF5IOHandler::flush() +std::future ParallelHDF5IOHandler::flush(internal::FlushParams const &) { return std::future(); } diff --git a/src/IO/JSON/JSONIOHandler.cpp b/src/IO/JSON/JSONIOHandler.cpp index 158c5454ed..15d18194c7 100644 --- a/src/IO/JSON/JSONIOHandler.cpp +++ b/src/IO/JSON/JSONIOHandler.cpp @@ -29,7 +29,7 @@ JSONIOHandler::JSONIOHandler(std::string path, Access at) : AbstractIOHandler{path, at}, m_impl{JSONIOHandlerImpl{this}} {} -std::future JSONIOHandler::flush() +std::future JSONIOHandler::flush(internal::FlushParams const &) { return m_impl.flush(); } diff --git a/src/Iteration.cpp b/src/Iteration.cpp index 360c17d8c9..7026dd90e3 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -128,7 +128,7 @@ Iteration &Iteration::close(bool _flush) auto end = begin; ++end; - s->flush_impl(begin, end, FlushLevel::UserFlush); + s->flush_impl(begin, end, {FlushLevel::UserFlush}); } } else @@ -148,13 +148,13 @@ Iteration &Iteration::open() if (*m_closed == CloseStatus::ParseAccessDeferred) { *m_closed = CloseStatus::Open; + runDeferredParseAccess(); } - runDeferredParseAccess(); internal::SeriesInternal *s = &retrieveSeries(); // figure out my iteration number auto begin = s->indexOf(*this); s->openIteration(begin->first, *this); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); return *this; } @@ -191,7 +191,10 @@ bool Iteration::closedByWriter() const } } -void Iteration::flushFileBased(std::string const &filename, uint64_t i) +void Iteration::flushFileBased( + std::string const &filename, + uint64_t i, + internal::FlushParams const &flushParams) { /* Find the root point [Series] of this file, * meshesPath and particlesPath are stored there */ @@ -227,7 +230,7 @@ void Iteration::flushFileBased(std::string const &filename, uint64_t i) fOpen.name = filename; fOpen.encoding = IterationEncoding::fileBased; IOHandler()->enqueue(IOTask(s, fOpen)); - flush(); + flush(flushParams); return; } @@ -237,10 +240,20 @@ void Iteration::flushFileBased(std::string const &filename, uint64_t i) s->openIteration(i, *this); } - flush(); + switch (flushParams.flushLevel) + { + case FlushLevel::CreateOrOpenFiles: + break; + case FlushLevel::SkeletonOnly: + case FlushLevel::InternalFlush: + case FlushLevel::UserFlush: + flush(flushParams); + break; + } } -void Iteration::flushGroupBased(uint64_t i) +void Iteration::flushGroupBased( + uint64_t i, internal::FlushParams const &flushParams) { if (!written()) { @@ -250,10 +263,20 @@ void Iteration::flushGroupBased(uint64_t i) IOHandler()->enqueue(IOTask(this, pCreate)); } - flush(); + switch (flushParams.flushLevel) + { + case FlushLevel::CreateOrOpenFiles: + break; + case FlushLevel::SkeletonOnly: + case FlushLevel::InternalFlush: + case FlushLevel::UserFlush: + flush(flushParams); + break; + } } -void Iteration::flushVariableBased(uint64_t i) +void Iteration::flushVariableBased( + uint64_t i, internal::FlushParams const &flushParams) { if (!written()) { @@ -264,17 +287,26 @@ void Iteration::flushVariableBased(uint64_t i) this->setAttribute("snapshot", i); } - flush(); + switch (flushParams.flushLevel) + { + case FlushLevel::CreateOrOpenFiles: + break; + case FlushLevel::SkeletonOnly: + case FlushLevel::InternalFlush: + case FlushLevel::UserFlush: + flush(flushParams); + break; + } } -void Iteration::flush() +void Iteration::flush(internal::FlushParams const &flushParams) { if (IOHandler()->m_frontendAccess == Access::READ_ONLY) { for (auto &m : meshes) - m.second.flush(m.first); + m.second.flush(m.first, flushParams); for (auto &species : particles) - species.second.flush(species.first); + species.second.flush(species.first, flushParams); } else { @@ -289,9 +321,9 @@ void Iteration::flush() s->setMeshesPath("meshes/"); s->flushMeshesPath(); } - meshes.flush(s->meshesPath()); + meshes.flush(s->meshesPath(), flushParams); for (auto &m : meshes) - m.second.flush(m.first); + m.second.flush(m.first, flushParams); } else { @@ -305,16 +337,16 @@ void Iteration::flush() s->setParticlesPath("particles/"); s->flushParticlesPath(); } - particles.flush(s->particlesPath()); + particles.flush(s->particlesPath(), flushParams); for (auto &species : particles) - species.second.flush(species.first); + species.second.flush(species.first, flushParams); } else { particles.dirty() = false; } - flushAttributes(); + flushAttributes(flushParams); } } @@ -324,25 +356,6 @@ void Iteration::deferParseAccess(DeferredParseAccess dr) auxiliary::makeOption(std::move(dr)); } -void Iteration::read() -{ - if (!m_deferredParseAccess->has_value()) - { - return; - } - auto const &deferred = m_deferredParseAccess->get(); - if (deferred.fileBased) - { - readFileBased(deferred.filename, deferred.path); - } - else - { - readGorVBased(deferred.path); - } - // reset this thing - *m_deferredParseAccess = auxiliary::Option(); -} - void Iteration::reread(std::string const &path) { if (m_deferredParseAccess->has_value()) @@ -355,8 +368,15 @@ void Iteration::reread(std::string const &path) } void Iteration::readFileBased( - std::string filePath, std::string const &groupPath) + std::string filePath, std::string const &groupPath, bool doBeginStep) { + if (doBeginStep) + { + /* + * beginStep() must take care to open files + */ + beginStep(/* reread = */ false); + } auto &series = retrieveSeries(); series.readOneIterationFileBased(filePath); @@ -366,9 +386,15 @@ void Iteration::readFileBased( read_impl(groupPath); } -void Iteration::readGorVBased(std::string const &groupPath) +void Iteration::readGorVBased(std::string const &groupPath, bool doBeginStep) { - + if (doBeginStep) + { + /* + * beginStep() must take care to open files + */ + beginStep(/* reread = */ false); + } read_impl(groupPath); } @@ -383,7 +409,7 @@ void Iteration::read_impl(std::string const &groupPath) aRead.name = "dt"; IOHandler()->enqueue(IOTask(this, aRead)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); if (*aRead.dtype == DT::FLOAT) setDt(Attribute(*aRead.resource).get()); else if (*aRead.dtype == DT::DOUBLE) @@ -395,7 +421,7 @@ void Iteration::read_impl(std::string const &groupPath) aRead.name = "time"; IOHandler()->enqueue(IOTask(this, aRead)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); if (*aRead.dtype == DT::FLOAT) setTime(Attribute(*aRead.resource).get()); else if (*aRead.dtype == DT::DOUBLE) @@ -407,7 +433,7 @@ void Iteration::read_impl(std::string const &groupPath) aRead.name = "timeUnitSI"; IOHandler()->enqueue(IOTask(this, aRead)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); if (*aRead.dtype == DT::DOUBLE) setTimeUnitSI(Attribute(*aRead.resource).get()); else @@ -425,7 +451,7 @@ void Iteration::read_impl(std::string const &groupPath) if (version == "1.0.0" || version == "1.0.1") { IOHandler()->enqueue(IOTask(this, pList)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); hasMeshes = std::count( pList.paths->begin(), pList.paths->end(), @@ -454,7 +480,7 @@ void Iteration::read_impl(std::string const &groupPath) /* obtain all non-scalar meshes */ IOHandler()->enqueue(IOTask(&meshes, pList)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); Parameter aList; for (auto const &mesh_name : *pList.paths) @@ -464,7 +490,7 @@ void Iteration::read_impl(std::string const &groupPath) aList.attributes->clear(); IOHandler()->enqueue(IOTask(&m, pOpen)); IOHandler()->enqueue(IOTask(&m, aList)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); auto att_begin = aList.attributes->begin(); auto att_end = aList.attributes->end(); @@ -475,7 +501,7 @@ void Iteration::read_impl(std::string const &groupPath) MeshRecordComponent &mrc = m[MeshRecordComponent::SCALAR]; mrc.parent() = m.parent(); IOHandler()->enqueue(IOTask(&mrc, pOpen)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); *mrc.m_isConstant = true; } m.read(); @@ -484,7 +510,7 @@ void Iteration::read_impl(std::string const &groupPath) /* obtain all scalar meshes */ Parameter dList; IOHandler()->enqueue(IOTask(&meshes, dList)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); Parameter dOpen; for (auto const &mesh_name : *dList.datasets) @@ -492,11 +518,11 @@ void Iteration::read_impl(std::string const &groupPath) Mesh &m = map[mesh_name]; dOpen.name = mesh_name; IOHandler()->enqueue(IOTask(&m, dOpen)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); MeshRecordComponent &mrc = m[MeshRecordComponent::SCALAR]; mrc.parent() = m.parent(); IOHandler()->enqueue(IOTask(&mrc, dOpen)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); mrc.written() = false; mrc.resetDataset(Dataset(*dOpen.dtype, *dOpen.extent)); mrc.written() = true; @@ -518,7 +544,7 @@ void Iteration::read_impl(std::string const &groupPath) /* obtain all particle species */ pList.paths->clear(); IOHandler()->enqueue(IOTask(&particles, pList)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); auto map = particles.eraseStaleEntries(); for (auto const &species_name : *pList.paths) @@ -526,7 +552,7 @@ void Iteration::read_impl(std::string const &groupPath) ParticleSpecies &p = map[species_name]; pOpen.path = species_name; IOHandler()->enqueue(IOTask(&p, pOpen)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); p.read(); } } @@ -538,7 +564,7 @@ void Iteration::read_impl(std::string const &groupPath) readAttributes(ReadMode::FullyReread); } -AdvanceStatus Iteration::beginStep() +AdvanceStatus Iteration::beginStep(bool reread) { using IE = IterationEncoding; auto &series = retrieveSeries(); @@ -563,7 +589,8 @@ AdvanceStatus Iteration::beginStep() } // re-read -> new datasets might be available - if ((series.iterationEncoding() == IE::groupBased || + if (reread && + (series.iterationEncoding() == IE::groupBased || series.iterationEncoding() == IE::variableBased) && (this->IOHandler()->m_frontendAccess == Access::READ_ONLY || this->IOHandler()->m_frontendAccess == Access::READ_WRITE)) @@ -677,18 +704,36 @@ void Iteration::runDeferredParseAccess() { return; } + + if (!m_deferredParseAccess->has_value()) + { + return; + } + auto const &deferred = m_deferredParseAccess->get(); + auto oldAccess = IOHandler()->m_frontendAccess; auto newAccess = const_cast(&IOHandler()->m_frontendAccess); *newAccess = Access::READ_WRITE; try { - read(); + if (deferred.fileBased) + { + readFileBased(deferred.filename, deferred.path, deferred.beginStep); + } + else + { + readGorVBased(deferred.path, deferred.beginStep); + } } catch (...) { + // reset this thing + *m_deferredParseAccess = auxiliary::Option(); *newAccess = oldAccess; throw; } + // reset this thing + *m_deferredParseAccess = auxiliary::Option(); *newAccess = oldAccess; } diff --git a/src/Mesh.cpp b/src/Mesh.cpp index 7f468d8c3c..35c1e4d61a 100644 --- a/src/Mesh.cpp +++ b/src/Mesh.cpp @@ -213,12 +213,13 @@ template Mesh &Mesh::setTimeOffset(double); template Mesh &Mesh::setTimeOffset(float); -void Mesh::flush_impl(std::string const &name) +void Mesh::flush_impl( + std::string const &name, internal::FlushParams const &flushParams) { if (IOHandler()->m_frontendAccess == Access::READ_ONLY) { for (auto &comp : *this) - comp.second.flush(comp.first); + comp.second.flush(comp.first, flushParams); } else { @@ -228,8 +229,8 @@ void Mesh::flush_impl(std::string const &name) { MeshRecordComponent &mrc = at(RecordComponent::SCALAR); mrc.parent() = parent(); - mrc.flush(name); - IOHandler()->flush(); + mrc.flush(name, flushParams); + IOHandler()->flush(flushParams); writable().abstractFilePosition = mrc.writable().abstractFilePosition; written() = true; @@ -248,7 +249,7 @@ void Mesh::flush_impl(std::string const &name) { for (auto &comp : *this) { - comp.second.flush(name); + comp.second.flush(name, flushParams); writable().abstractFilePosition = comp.second.writable().abstractFilePosition; } @@ -256,10 +257,10 @@ void Mesh::flush_impl(std::string const &name) else { for (auto &comp : *this) - comp.second.flush(comp.first); + comp.second.flush(comp.first, flushParams); } - flushAttributes(); + flushAttributes(flushParams); } } @@ -272,7 +273,7 @@ void Mesh::read() aRead.name = "geometry"; IOHandler()->enqueue(IOTask(this, aRead)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); if (*aRead.dtype == DT::STRING) { std::string tmpGeometry = Attribute(*aRead.resource).get(); @@ -293,7 +294,7 @@ void Mesh::read() aRead.name = "dataOrder"; IOHandler()->enqueue(IOTask(this, aRead)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); if (*aRead.dtype == DT::CHAR) setDataOrder( static_cast(Attribute(*aRead.resource).get())); @@ -313,7 +314,7 @@ void Mesh::read() aRead.name = "axisLabels"; IOHandler()->enqueue(IOTask(this, aRead)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); if (*aRead.dtype == DT::VEC_STRING || *aRead.dtype == DT::STRING) setAxisLabels( Attribute(*aRead.resource).get >()); @@ -323,7 +324,7 @@ void Mesh::read() aRead.name = "gridSpacing"; IOHandler()->enqueue(IOTask(this, aRead)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); Attribute a = Attribute(*aRead.resource); if (*aRead.dtype == DT::VEC_FLOAT || *aRead.dtype == DT::FLOAT) setGridSpacing(a.get >()); @@ -338,7 +339,7 @@ void Mesh::read() aRead.name = "gridGlobalOffset"; IOHandler()->enqueue(IOTask(this, aRead)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); if (*aRead.dtype == DT::VEC_DOUBLE || *aRead.dtype == DT::DOUBLE) setGridGlobalOffset( Attribute(*aRead.resource).get >()); @@ -348,7 +349,7 @@ void Mesh::read() aRead.name = "gridUnitSI"; IOHandler()->enqueue(IOTask(this, aRead)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); if (*aRead.dtype == DT::DOUBLE) setGridUnitSI(Attribute(*aRead.resource).get()); else @@ -364,7 +365,7 @@ void Mesh::read() { Parameter pList; IOHandler()->enqueue(IOTask(this, pList)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); Parameter pOpen; for (auto const &component : *pList.paths) @@ -378,7 +379,7 @@ void Mesh::read() Parameter dList; IOHandler()->enqueue(IOTask(this, dList)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); Parameter dOpen; for (auto const &component : *dList.datasets) @@ -386,7 +387,7 @@ void Mesh::read() MeshRecordComponent &rc = map[component]; dOpen.name = component; IOHandler()->enqueue(IOTask(&rc, dOpen)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); rc.written() = false; rc.resetDataset(Dataset(*dOpen.dtype, *dOpen.extent)); rc.written() = true; diff --git a/src/ParticlePatches.cpp b/src/ParticlePatches.cpp index 952aff37ac..76017bbf94 100644 --- a/src/ParticlePatches.cpp +++ b/src/ParticlePatches.cpp @@ -35,7 +35,7 @@ void ParticlePatches::read() { Parameter pList; IOHandler()->enqueue(IOTask(this, pList)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); Parameter pOpen; for (auto const &record_name : *pList.paths) @@ -48,7 +48,7 @@ void ParticlePatches::read() Parameter dList; IOHandler()->enqueue(IOTask(this, dList)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); Parameter dOpen; for (auto const &component_name : *dList.datasets) @@ -65,7 +65,7 @@ void ParticlePatches::read() dOpen.name = component_name; IOHandler()->enqueue(IOTask(&pr, dOpen)); IOHandler()->enqueue(IOTask(&prc, dOpen)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); if (determineDatatype() != *dOpen.dtype) throw std::runtime_error( diff --git a/src/ParticleSpecies.cpp b/src/ParticleSpecies.cpp index af15806f6b..3e85bb546e 100644 --- a/src/ParticleSpecies.cpp +++ b/src/ParticleSpecies.cpp @@ -38,7 +38,7 @@ void ParticleSpecies::read() /* obtain all non-scalar records */ Parameter pList; IOHandler()->enqueue(IOTask(this, pList)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); auto map = eraseStaleEntries(); @@ -61,7 +61,7 @@ void ParticleSpecies::read() aList.attributes->clear(); IOHandler()->enqueue(IOTask(&r, pOpen)); IOHandler()->enqueue(IOTask(&r, aList)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); auto att_begin = aList.attributes->begin(); auto att_end = aList.attributes->end(); @@ -73,7 +73,7 @@ void ParticleSpecies::read() RecordComponent &rc = scalarMap[RecordComponent::SCALAR]; rc.parent() = r.parent(); IOHandler()->enqueue(IOTask(&rc, pOpen)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); *rc.m_isConstant = true; } r.read(); @@ -90,7 +90,7 @@ void ParticleSpecies::read() /* obtain all scalar records */ Parameter dList; IOHandler()->enqueue(IOTask(this, dList)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); Parameter dOpen; for (auto const &record_name : *dList.datasets) @@ -100,12 +100,12 @@ void ParticleSpecies::read() Record &r = map[record_name]; dOpen.name = record_name; IOHandler()->enqueue(IOTask(&r, dOpen)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); auto scalarMap = r.eraseStaleEntries(); RecordComponent &rc = scalarMap[RecordComponent::SCALAR]; rc.parent() = r.parent(); IOHandler()->enqueue(IOTask(&rc, dOpen)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); rc.written() = false; rc.resetDataset(Dataset(*dOpen.dtype, *dOpen.extent)); rc.written() = true; @@ -138,14 +138,15 @@ namespace } } // namespace -void ParticleSpecies::flush(std::string const &path) +void ParticleSpecies::flush( + std::string const &path, internal::FlushParams const &flushParams) { if (IOHandler()->m_frontendAccess == Access::READ_ONLY) { for (auto &record : *this) - record.second.flush(record.first); + record.second.flush(record.first, flushParams); for (auto &patch : particlePatches) - patch.second.flush(patch.first); + patch.second.flush(patch.first, flushParams); } else { @@ -156,16 +157,16 @@ void ParticleSpecies::flush(std::string const &path) if (it != end()) it->second.setUnitDimension({{UnitDimension::L, 1}}); - Container::flush(path); + Container::flush(path, flushParams); for (auto &record : *this) - record.second.flush(record.first); + record.second.flush(record.first, flushParams); if (flushParticlePatches(particlePatches)) { - particlePatches.flush("particlePatches"); + particlePatches.flush("particlePatches", flushParams); for (auto &patch : particlePatches) - patch.second.flush(patch.first); + patch.second.flush(patch.first, flushParams); } } } diff --git a/src/ReadIterations.cpp b/src/ReadIterations.cpp index 799234785b..08dc4f9545 100644 --- a/src/ReadIterations.cpp +++ b/src/ReadIterations.cpp @@ -62,7 +62,7 @@ SeriesIterator::SeriesIterator(Series series) : m_series(std::move(series)) * the step after parsing the file is ok. */ openIteration(); - status = it->second.beginStep(); + status = it->second.beginStep(/* reread = */ true); break; case IterationEncoding::groupBased: case IterationEncoding::variableBased: @@ -71,7 +71,7 @@ SeriesIterator::SeriesIterator(Series series) : m_series(std::move(series)) * access to the file until now. Better to begin a step right away, * otherwise we might get another step's data. */ - status = it->second.beginStep(); + status = it->second.beginStep(/* reread = */ true); openIteration(); break; } @@ -106,7 +106,8 @@ SeriesIterator &SeriesIterator::operator++() case IE::variableBased: { // since we are in group-based iteration layout, it does not // matter which iteration we begin a step upon - AdvanceStatus status = currentIteration.beginStep(); + AdvanceStatus status{}; + status = currentIteration.beginStep(/* reread = */ true); if (status == AdvanceStatus::OVER) { *this = end(); @@ -141,7 +142,8 @@ SeriesIterator &SeriesIterator::operator++() using IE = IterationEncoding; case IE::fileBased: { auto &iteration = series.iterations[m_currentIteration]; - AdvanceStatus status = iteration.beginStep(); + AdvanceStatus status{}; + status = iteration.beginStep(/* reread = */ true); if (status == AdvanceStatus::OVER) { *this = end(); diff --git a/src/Record.cpp b/src/Record.cpp index 78705845bb..3f50ef510c 100644 --- a/src/Record.cpp +++ b/src/Record.cpp @@ -43,12 +43,13 @@ Record &Record::setUnitDimension(std::map const &udim) return *this; } -void Record::flush_impl(std::string const &name) +void Record::flush_impl( + std::string const &name, internal::FlushParams const &flushParams) { if (IOHandler()->m_frontendAccess == Access::READ_ONLY) { for (auto &comp : *this) - comp.second.flush(comp.first); + comp.second.flush(comp.first, flushParams); } else { @@ -58,8 +59,8 @@ void Record::flush_impl(std::string const &name) { RecordComponent &rc = at(RecordComponent::SCALAR); rc.parent() = parent(); - rc.flush(name); - IOHandler()->flush(); + rc.flush(name, flushParams); + IOHandler()->flush(flushParams); writable().abstractFilePosition = rc.writable().abstractFilePosition; written() = true; @@ -78,7 +79,7 @@ void Record::flush_impl(std::string const &name) { for (auto &comp : *this) { - comp.second.flush(name); + comp.second.flush(name, flushParams); writable().abstractFilePosition = comp.second.writable().abstractFilePosition; } @@ -86,10 +87,10 @@ void Record::flush_impl(std::string const &name) else { for (auto &comp : *this) - comp.second.flush(comp.first); + comp.second.flush(comp.first, flushParams); } - flushAttributes(); + flushAttributes(flushParams); } } @@ -104,7 +105,7 @@ void Record::read() { Parameter pList; IOHandler()->enqueue(IOTask(this, pList)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); Parameter pOpen; for (auto const &component : *pList.paths) @@ -118,7 +119,7 @@ void Record::read() Parameter dList; IOHandler()->enqueue(IOTask(this, dList)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); Parameter dOpen; for (auto const &component : *dList.datasets) @@ -126,7 +127,7 @@ void Record::read() RecordComponent &rc = (*this)[component]; dOpen.name = component; IOHandler()->enqueue(IOTask(&rc, dOpen)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); rc.written() = false; rc.resetDataset(Dataset(*dOpen.dtype, *dOpen.extent)); rc.written() = true; diff --git a/src/RecordComponent.cpp b/src/RecordComponent.cpp index 2368f02a7c..47390c3ba5 100644 --- a/src/RecordComponent.cpp +++ b/src/RecordComponent.cpp @@ -170,9 +170,10 @@ bool RecordComponent::empty() const return *m_isEmpty; } -void RecordComponent::flush(std::string const &name) +void RecordComponent::flush( + std::string const &name, internal::FlushParams const &flushParams) { - if (IOHandler()->m_flushLevel == FlushLevel::SkeletonOnly) + if (flushParams.flushLevel == FlushLevel::SkeletonOnly) { *this->m_name = name; return; @@ -249,7 +250,7 @@ void RecordComponent::flush(std::string const &name) m_chunks->pop(); } - flushAttributes(); + flushAttributes(flushParams); } } @@ -267,7 +268,7 @@ void RecordComponent::readBase() { aRead.name = "value"; IOHandler()->enqueue(IOTask(this, aRead)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); Attribute a(*aRead.resource); DT dtype = *aRead.dtype; @@ -332,7 +333,7 @@ void RecordComponent::readBase() aRead.name = "shape"; IOHandler()->enqueue(IOTask(this, aRead)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); a = Attribute(*aRead.resource); Extent e; @@ -358,7 +359,7 @@ void RecordComponent::readBase() aRead.name = "unitSI"; IOHandler()->enqueue(IOTask(this, aRead)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); if (*aRead.dtype == DT::DOUBLE) setUnitSI(Attribute(*aRead.resource).get()); else diff --git a/src/Series.cpp b/src/Series.cpp index 4123f3d067..2745207ce9 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -353,7 +353,7 @@ void SeriesInterface::flush() flush_impl( series.iterations.begin(), series.iterations.end(), - FlushLevel::UserFlush); + {FlushLevel::UserFlush}); } std::unique_ptr @@ -521,10 +521,9 @@ void SeriesInterface::initDefaults(IterationEncoding ie) std::future SeriesInterface::flush_impl( iterations_iterator begin, iterations_iterator end, - FlushLevel level, + internal::FlushParams flushParams, bool flushIOHandler) { - IOHandler()->m_flushLevel = level; auto &series = get(); series.m_lastFlushSuccessful = true; try @@ -533,35 +532,33 @@ std::future SeriesInterface::flush_impl( { using IE = IterationEncoding; case IE::fileBased: - flushFileBased(begin, end); + flushFileBased(begin, end, flushParams); break; case IE::groupBased: case IE::variableBased: - flushGorVBased(begin, end); + flushGorVBased(begin, end, flushParams); break; } if (flushIOHandler) { - auto res = IOHandler()->flush(); - IOHandler()->m_flushLevel = FlushLevel::InternalFlush; - return res; + return IOHandler()->flush(flushParams); } else { - IOHandler()->m_flushLevel = FlushLevel::InternalFlush; return {}; } } catch (...) { - IOHandler()->m_flushLevel = FlushLevel::InternalFlush; series.m_lastFlushSuccessful = false; throw; } } void SeriesInterface::flushFileBased( - iterations_iterator begin, iterations_iterator end) + iterations_iterator begin, + iterations_iterator end, + internal::FlushParams flushParams) { auto &series = get(); if (end == begin) @@ -576,7 +573,7 @@ void SeriesInterface::flushFileBased( { using IO = IterationOpened; case IO::HasBeenOpened: - it->second.flush(); + it->second.flush(flushParams); break; case IO::RemainsClosed: break; @@ -592,7 +589,7 @@ void SeriesInterface::flushFileBased( } // Phase 3 - IOHandler()->flush(); + IOHandler()->flush(flushParams); } else { @@ -613,12 +610,13 @@ void SeriesInterface::flushFileBased( dirty() |= it->second.dirty(); std::string filename = iterationFilename(it->first); - it->second.flushFileBased(filename, it->first); + it->second.flushFileBased(filename, it->first, flushParams); series.iterations.flush( - auxiliary::replace_first(basePath(), "%T/", "")); + auxiliary::replace_first(basePath(), "%T/", ""), + flushParams); - flushAttributes(); + flushAttributes(flushParams); break; } case IO::RemainsClosed: @@ -635,7 +633,7 @@ void SeriesInterface::flushFileBased( } // Phase 3 - IOHandler()->flush(); + IOHandler()->flush(flushParams); /* reset the dirty bit for every iteration (i.e. file) * otherwise only the first iteration will have updates attributes @@ -647,7 +645,9 @@ void SeriesInterface::flushFileBased( } void SeriesInterface::flushGorVBased( - iterations_iterator begin, iterations_iterator end) + iterations_iterator begin, + iterations_iterator end, + internal::FlushParams flushParams) { auto &series = get(); if (IOHandler()->m_frontendAccess == Access::READ_ONLY) @@ -658,7 +658,7 @@ void SeriesInterface::flushGorVBased( { using IO = IterationOpened; case IO::HasBeenOpened: - it->second.flush(); + it->second.flush(flushParams); break; case IO::RemainsClosed: break; @@ -673,7 +673,7 @@ void SeriesInterface::flushGorVBased( } // Phase 3 - IOHandler()->flush(); + IOHandler()->flush(flushParams); } else { @@ -686,7 +686,7 @@ void SeriesInterface::flushGorVBased( } series.iterations.flush( - auxiliary::replace_first(basePath(), "%T/", "")); + auxiliary::replace_first(basePath(), "%T/", ""), flushParams); for (auto it = begin; it != end; ++it) { @@ -703,10 +703,10 @@ void SeriesInterface::flushGorVBased( { using IE = IterationEncoding; case IE::groupBased: - it->second.flushGroupBased(it->first); + it->second.flushGroupBased(it->first, flushParams); break; case IE::variableBased: - it->second.flushVariableBased(it->first); + it->second.flushVariableBased(it->first, flushParams); break; default: throw std::runtime_error( @@ -726,8 +726,8 @@ void SeriesInterface::flushGorVBased( } } - flushAttributes(); - IOHandler()->flush(); + flushAttributes(flushParams); + IOHandler()->flush(flushParams); } } @@ -801,7 +801,7 @@ void SeriesInterface::readFileBased() iteration.runDeferredParseAccess(); Parameter fClose; iteration.IOHandler()->enqueue(IOTask(&iteration, fClose)); - iteration.IOHandler()->flush(); + iteration.IOHandler()->flush(internal::defaultFlushParams); *iteration.m_closed = Iteration::CloseStatus::ClosedTemporarily; }; if (series.m_parseLazily) @@ -847,7 +847,7 @@ void SeriesInterface::readOneIterationFileBased(std::string const &filePath) fOpen.name = filePath; IOHandler()->enqueue(IOTask(this, fOpen)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); series.iterations.parent() = getWritable(this); readBase(); @@ -855,7 +855,7 @@ void SeriesInterface::readOneIterationFileBased(std::string const &filePath) using DT = Datatype; aRead.name = "iterationEncoding"; IOHandler()->enqueue(IOTask(this, aRead)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); if (*aRead.dtype == DT::STRING) { std::string encoding = Attribute(*aRead.resource).get(); @@ -892,7 +892,7 @@ void SeriesInterface::readOneIterationFileBased(std::string const &filePath) aRead.name = "iterationFormat"; IOHandler()->enqueue(IOTask(this, aRead)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); if (*aRead.dtype == DT::STRING) { written() = false; @@ -922,7 +922,7 @@ void SeriesInterface::readGorVBased(bool do_init) fOpen.name = series.m_name; fOpen.encoding = iterationEncoding(); IOHandler()->enqueue(IOTask(this, fOpen)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); if (do_init) { @@ -932,7 +932,7 @@ void SeriesInterface::readGorVBased(bool do_init) Parameter aRead; aRead.name = "iterationEncoding"; IOHandler()->enqueue(IOTask(this, aRead)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); if (*aRead.dtype == DT::STRING) { std::string encoding = @@ -966,7 +966,7 @@ void SeriesInterface::readGorVBased(bool do_init) aRead.name = "iterationFormat"; IOHandler()->enqueue(IOTask(this, aRead)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); if (*aRead.dtype == DT::STRING) { written() = false; @@ -994,11 +994,12 @@ void SeriesInterface::readGorVBased(bool do_init) /* obtain all paths inside the basepath (i.e. all iterations) */ Parameter pList; IOHandler()->enqueue(IOTask(&series.iterations, pList)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); auto readSingleIteration = [&series, &pOpen, this]( - uint64_t index, std::string path, bool guardAgainstRereading) { + uint64_t index, std::string path, bool guardAgainstRereading, + bool beginStep) { if (series.iterations.contains(index)) { // maybe re-read @@ -1020,7 +1021,7 @@ void SeriesInterface::readGorVBased(bool do_init) { // parse for the first time, resp. delay the parsing process Iteration &i = series.iterations[index]; - i.deferParseAccess({path, index, false, ""}); + i.deferParseAccess({path, index, false, "", beginStep}); if (!series.m_parseLazily) { i.runDeferredParseAccess(); @@ -1043,7 +1044,12 @@ void SeriesInterface::readGorVBased(bool do_init) for (auto const &it : *pList.paths) { uint64_t index = std::stoull(it); - readSingleIteration(index, it, true); + /* + * For now: parse a Series in RandomAccess mode. + * (beginStep = false) + * A streaming read mode might come in a future API addition. + */ + readSingleIteration(index, it, true, false); } break; case IterationEncoding::variableBased: { @@ -1052,7 +1058,11 @@ void SeriesInterface::readGorVBased(bool do_init) { index = series.iterations.getAttribute("snapshot").get(); } - readSingleIteration(index, "", false); + /* + * Variable-based iteration encoding relies on steps, so parsing must + * happen after opening the first step. + */ + readSingleIteration(index, "", false, true); break; } } @@ -1066,7 +1076,7 @@ void SeriesInterface::readBase() aRead.name = "openPMD"; IOHandler()->enqueue(IOTask(this, aRead)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); if (*aRead.dtype == DT::STRING) setOpenPMD(Attribute(*aRead.resource).get()); else @@ -1074,7 +1084,7 @@ void SeriesInterface::readBase() aRead.name = "openPMDextension"; IOHandler()->enqueue(IOTask(this, aRead)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); if (*aRead.dtype == determineDatatype()) setOpenPMDextension(Attribute(*aRead.resource).get()); else @@ -1083,7 +1093,7 @@ void SeriesInterface::readBase() aRead.name = "basePath"; IOHandler()->enqueue(IOTask(this, aRead)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); if (*aRead.dtype == DT::STRING) setAttribute("basePath", Attribute(*aRead.resource).get()); else @@ -1092,14 +1102,14 @@ void SeriesInterface::readBase() Parameter aList; IOHandler()->enqueue(IOTask(this, aList)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); if (std::count( aList.attributes->begin(), aList.attributes->end(), "meshesPath") == 1) { aRead.name = "meshesPath"; IOHandler()->enqueue(IOTask(this, aRead)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); if (*aRead.dtype == DT::STRING) { /* allow setting the meshes path after completed IO */ @@ -1123,7 +1133,7 @@ void SeriesInterface::readBase() { aRead.name = "particlesPath"; IOHandler()->enqueue(IOTask(this, aRead)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); if (*aRead.dtype == DT::STRING) { /* allow setting the meshes path after completed IO */ @@ -1195,6 +1205,7 @@ AdvanceStatus SeriesInterface::advance( iterations_iterator begin, Iteration &iteration) { + constexpr internal::FlushParams flushParams = {FlushLevel::UserFlush}; auto &series = get(); auto end = begin; ++end; @@ -1212,7 +1223,24 @@ AdvanceStatus SeriesInterface::advance( *iteration.m_closed = Iteration::CloseStatus::Open; } - flush_impl(begin, end, FlushLevel::UserFlush, /* flushIOHandler = */ false); + switch (mode) + { + case AdvanceMode::ENDSTEP: + flush_impl(begin, end, flushParams, /* flushIOHandler = */ false); + break; + case AdvanceMode::BEGINSTEP: + /* + * When beginning a step, there is nothing to flush yet. + * Data is not written in between steps. + * So only make sure that files are accessed. + */ + flush_impl( + begin, + end, + {FlushLevel::CreateOrOpenFiles}, + /* flushIOHandler = */ false); + break; + } if (oldCloseStatus == Iteration::CloseStatus::ClosedInFrontend) { @@ -1284,17 +1312,7 @@ AdvanceStatus SeriesInterface::advance( // We cannot call SeriesInterface::flush now, since the IO handler is still // filled from calling flush(Group|File)based, but has not been emptied yet // Do that manually - IOHandler()->m_flushLevel = FlushLevel::UserFlush; - try - { - IOHandler()->flush(); - } - catch (...) - { - IOHandler()->m_flushLevel = FlushLevel::InternalFlush; - throw; - } - IOHandler()->m_flushLevel = FlushLevel::InternalFlush; + IOHandler()->flush(flushParams); return *param.status; } diff --git a/src/WriteIterations.cpp b/src/WriteIterations.cpp index 2fdfeba577..6fb9745a45 100644 --- a/src/WriteIterations.cpp +++ b/src/WriteIterations.cpp @@ -68,7 +68,7 @@ WriteIterations::mapped_type &WriteIterations::operator[](key_type &&key) auto &res = shared->iterations[std::move(key)]; if (res.getStepStatus() == StepStatus::NoStep) { - res.beginStep(); + res.beginStep(/* reread = */ false); res.setStepStatus(StepStatus::DuringStep); } return res; diff --git a/src/backend/Attributable.cpp b/src/backend/Attributable.cpp index f0ab4fbc81..79bde3a9dc 100644 --- a/src/backend/Attributable.cpp +++ b/src/backend/Attributable.cpp @@ -64,7 +64,7 @@ bool AttributableInterface::deleteAttribute(std::string const &key) Parameter aDelete; aDelete.name = key; IOHandler()->enqueue(IOTask(this, aDelete)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); attri.m_attributes.erase(it); return true; } @@ -210,16 +210,22 @@ auto AttributableInterface::myPath() const -> MyPath return res; } -void AttributableInterface::seriesFlush(FlushLevel level) +void Attributable::seriesFlush(internal::FlushParams flushParams) { - writable().seriesFlush(level); + writable().seriesFlush(flushParams); } -void AttributableInterface::flushAttributes() +void Attributable::flushAttributes(internal::FlushParams const &flushParams) { - if (IOHandler()->m_flushLevel == FlushLevel::SkeletonOnly) + switch (flushParams.flushLevel) { + case FlushLevel::SkeletonOnly: + case FlushLevel::CreateOrOpenFiles: return; + case FlushLevel::InternalFlush: + case FlushLevel::UserFlush: + // pass + break; } if (dirty()) { @@ -241,7 +247,7 @@ void AttributableInterface::readAttributes(ReadMode mode) auto &attri = get(); Parameter aList; IOHandler()->enqueue(IOTask(this, aList)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); std::vector written_attributes = attributes(); /* std::set_difference requires sorted ranges */ @@ -281,7 +287,7 @@ void AttributableInterface::readAttributes(ReadMode mode) IOHandler()->enqueue(IOTask(this, aRead)); try { - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); } catch (unsupported_data_error const &e) { diff --git a/src/backend/BaseRecordComponent.cpp b/src/backend/BaseRecordComponent.cpp index e17e652dea..0ec5acdc8e 100644 --- a/src/backend/BaseRecordComponent.cpp +++ b/src/backend/BaseRecordComponent.cpp @@ -65,7 +65,7 @@ ChunkTable BaseRecordComponent::availableChunks() Parameter param; IOTask task(this, param); IOHandler()->enqueue(task); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); return std::move(*param.chunks); } } // namespace openPMD diff --git a/src/backend/MeshRecordComponent.cpp b/src/backend/MeshRecordComponent.cpp index aa5afcce34..9602868a07 100644 --- a/src/backend/MeshRecordComponent.cpp +++ b/src/backend/MeshRecordComponent.cpp @@ -34,7 +34,7 @@ void MeshRecordComponent::read() aRead.name = "position"; IOHandler()->enqueue(IOTask(this, aRead)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); Attribute a = Attribute(*aRead.resource); if (*aRead.dtype == DT::VEC_FLOAT || *aRead.dtype == DT::FLOAT) setPosition(a.get >()); diff --git a/src/backend/PatchRecord.cpp b/src/backend/PatchRecord.cpp index 9c94241338..f31b135b62 100644 --- a/src/backend/PatchRecord.cpp +++ b/src/backend/PatchRecord.cpp @@ -36,19 +36,21 @@ PatchRecord::setUnitDimension(std::map const &udim) return *this; } -void PatchRecord::flush_impl(std::string const &path) +void PatchRecord::flush_impl( + std::string const &path, internal::FlushParams const &flushParams) { if (this->find(RecordComponent::SCALAR) == this->end()) { if (IOHandler()->m_frontendAccess != Access::READ_ONLY) Container::flush( - path); // warning (clang-tidy-10): bugprone-parent-virtual-call + path, flushParams); // warning (clang-tidy-10): + // bugprone-parent-virtual-call for (auto &comp : *this) - comp.second.flush(comp.first); + comp.second.flush(comp.first, flushParams); } else - this->operator[](RecordComponent::SCALAR).flush(path); - if (IOHandler()->m_flushLevel == FlushLevel::UserFlush) + this->operator[](RecordComponent::SCALAR).flush(path, flushParams); + if (flushParams.flushLevel == FlushLevel::UserFlush) { this->dirty() = false; } @@ -59,7 +61,7 @@ void PatchRecord::read() Parameter aRead; aRead.name = "unitDimension"; IOHandler()->enqueue(IOTask(this, aRead)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); if (*aRead.dtype == Datatype::ARR_DBL_7 || *aRead.dtype == Datatype::VEC_DOUBLE) @@ -72,7 +74,7 @@ void PatchRecord::read() Parameter dList; IOHandler()->enqueue(IOTask(this, dList)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); Parameter dOpen; for (auto const &component_name : *dList.datasets) @@ -80,7 +82,7 @@ void PatchRecord::read() PatchRecordComponent &prc = (*this)[component_name]; dOpen.name = component_name; IOHandler()->enqueue(IOTask(&prc, dOpen)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); /* allow all attributes to be set */ prc.written() = false; prc.resetDataset(Dataset(*dOpen.dtype, *dOpen.extent)); diff --git a/src/backend/PatchRecordComponent.cpp b/src/backend/PatchRecordComponent.cpp index b01a208d57..3ed031f52b 100644 --- a/src/backend/PatchRecordComponent.cpp +++ b/src/backend/PatchRecordComponent.cpp @@ -67,7 +67,8 @@ PatchRecordComponent::PatchRecordComponent() setUnitSI(1); } -void PatchRecordComponent::flush(std::string const &name) +void PatchRecordComponent::flush( + std::string const &name, internal::FlushParams const &flushParams) { if (IOHandler()->m_frontendAccess == Access::READ_ONLY) { @@ -98,7 +99,7 @@ void PatchRecordComponent::flush(std::string const &name) m_chunks->pop(); } - flushAttributes(); + flushAttributes(flushParams); } } @@ -108,7 +109,7 @@ void PatchRecordComponent::read() aRead.name = "unitSI"; IOHandler()->enqueue(IOTask(this, aRead)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); if (*aRead.dtype == Datatype::DOUBLE) setUnitSI(Attribute(*aRead.resource).get()); else diff --git a/src/backend/Writable.cpp b/src/backend/Writable.cpp index c53e30bd77..0ce92ed003 100644 --- a/src/backend/Writable.cpp +++ b/src/backend/Writable.cpp @@ -24,25 +24,19 @@ namespace openPMD { -Writable::Writable(internal::AttributableData *a) - : abstractFilePosition{nullptr} - , IOHandler{nullptr} - , attributable{a} - , parent{nullptr} - , dirty{true} - , written{false} +Writable::Writable(internal::AttributableData *a) : attributable{a} {} void Writable::seriesFlush() { - seriesFlush(FlushLevel::UserFlush); + seriesFlush({FlushLevel::UserFlush}); } -void Writable::seriesFlush(FlushLevel level) +void Writable::seriesFlush(internal::FlushParams flushParams) { auto &series = AttributableInterface(attributable).retrieveSeries(); series.flush_impl( - series.iterations.begin(), series.iterations.end(), level); + series.iterations.begin(), series.iterations.end(), flushParams); } } // namespace openPMD diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index 85326b2891..3958a1558b 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -4516,6 +4516,12 @@ void variableBasedSeries(std::string const &file) std::vector data(1000, i); E_x.storeChunk(data, {0}, {1000}); + if (i > 2) + { + iteration.setAttribute( + "iteration_is_larger_than_two", "it truly is"); + } + // this tests changing extents and dimensionalities // across iterations auto E_y = iteration.meshes["E"]["y"]; @@ -4549,6 +4555,18 @@ void variableBasedSeries(std::string const &file) size_t last_iteration_index = 0; for (auto iteration : readSeries.readIterations()) { + if (iteration.iterationIndex > 2) + { + REQUIRE( + iteration.getAttribute("iteration_is_larger_than_two") + .get() == "it truly is"); + } + else + { + REQUIRE_FALSE(iteration.containsAttribute( + "iteration_is_larger_than_two")); + } + auto E_x = iteration.meshes["E"]["x"]; REQUIRE(E_x.getDimensionality() == 1); REQUIRE(E_x.getExtent()[0] == extent);