Backport: fix-open-iteration

include/openPMD/IO/ADIOS/ADIOS1IOHandler.hpp

@@ -49,7 +49,7 @@ class OPENPMDAPI_EXPORT ADIOS1IOHandler : public AbstractIOHandler
         return "ADIOS1";
     }
 
-    std::future<void> flush() override;
+    std::future<void> flush(internal::FlushParams const &) override;
 
     void enqueue(IOTask const &) override;
 
@@ -71,7 +71,7 @@ class OPENPMDAPI_EXPORT ADIOS1IOHandler : public AbstractIOHandler
         return "DUMMY_ADIOS1";
     }
 
-    std::future<void> flush() override;
+    std::future<void> flush(internal::FlushParams const &) override;
 
 private:
     std::unique_ptr<ADIOS1IOHandlerImpl> m_impl;

include/openPMD/IO/ADIOS/ADIOS1IOHandlerImpl.hpp

@@ -48,7 +48,7 @@ class OPENPMDAPI_EXPORT ADIOS1IOHandlerImpl : public AbstractIOHandlerImpl
 
     virtual void init();
 
-    std::future<void> flush() override;
+    std::future<void> flush();
 
     void
     createFile(Writable *, Parameter<Operation::CREATE_FILE> const &) override;

include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp

@@ -140,7 +140,7 @@ class ADIOS2IOHandlerImpl
 
     ~ADIOS2IOHandlerImpl() override;
 
-    std::future<void> flush() override;
+    std::future<void> flush(internal::FlushParams const &);
 
     void
     createFile(Writable *, Parameter<Operation::CREATE_FILE> const &) override;
@@ -1323,7 +1323,7 @@ class ADIOS2IOHandler : public AbstractIOHandler
         // we must not throw in a destructor
         try
         {
-            this->flush();
+            this->flush(internal::defaultFlushParams);
         }
         catch (std::exception const &ex)
         {
@@ -1362,6 +1362,6 @@ class ADIOS2IOHandler : public AbstractIOHandler
         return "ADIOS2";
     }
 
-    std::future<void> flush() override;
+    std::future<void> flush(internal::FlushParams const &) override;
 }; // ADIOS2IOHandler
 } // namespace openPMD

include/openPMD/IO/ADIOS/ParallelADIOS1IOHandler.hpp

@@ -52,7 +52,7 @@ class OPENPMDAPI_EXPORT ParallelADIOS1IOHandler : public AbstractIOHandler
         return "MPI_ADIOS1";
     }
 
-    std::future<void> flush() override;
+    std::future<void> flush(internal::FlushParams const &) override;
 #if openPMD_HAVE_ADIOS1
     void enqueue(IOTask const &) override;
 #endif

include/openPMD/IO/ADIOS/ParallelADIOS1IOHandlerImpl.hpp

@@ -50,7 +50,7 @@ class OPENPMDAPI_EXPORT ParallelADIOS1IOHandlerImpl
 
     virtual void init();
 
-    std::future<void> flush() override;
+    std::future<void> flush();
 
     void
     createFile(Writable *, Parameter<Operation::CREATE_FILE> const &) override;

include/openPMD/IO/AbstractIOHandler.hpp

@@ -61,7 +61,9 @@ class unsupported_data_error : public std::runtime_error
  * @brief Determine what items should be flushed upon Series::flush()
  *
  */
-enum class FlushLevel : unsigned char
+// do not write `enum class FlushLevel : unsigned char` here since NVHPC
+// does not compile it correctly
+enum class FlushLevel
 {
     /**
      * Flush operation that was triggered by user code.
@@ -84,9 +86,31 @@ enum class FlushLevel : unsigned char
      * CREATE_DATASET tasks.
      * Attributes may or may not be flushed yet.
      */
-    SkeletonOnly
+    SkeletonOnly,
+    /**
+     * Only creates/opens files, nothing more
+     */
+    CreateOrOpenFiles
 };
 
+namespace internal
+{
+    /**
+     * Parameters recursively passed through the openPMD hierarchy when
+     * flushing.
+     *
+     */
+    struct FlushParams
+    {
+        FlushLevel flushLevel = FlushLevel::InternalFlush;
+    };
+
+    /*
+     * To be used for reading
+     */
+    constexpr FlushParams defaultFlushParams{};
+} // namespace internal
+
 /** Interface for communicating between logical and physically persistent data.
  *
  * Input and output operations are channeled through a task queue that is
@@ -123,7 +147,7 @@ class AbstractIOHandler
      * @return  Future indicating the completion state of the operation for
      * backends that decide to implement this operation asynchronously.
      */
-    virtual std::future<void> flush() = 0;
+    virtual std::future<void> flush(internal::FlushParams const &) = 0;
 
     /** The currently used backend */
     virtual std::string backendName() const = 0;
@@ -132,7 +156,6 @@ class AbstractIOHandler
     Access const m_backendAccess;
     Access const m_frontendAccess;
     std::queue<IOTask> m_work;
-    FlushLevel m_flushLevel = FlushLevel::InternalFlush;
 }; // AbstractIOHandler
 
 } // namespace openPMD

include/openPMD/IO/AbstractIOHandlerImpl.hpp

@@ -40,7 +40,7 @@ class AbstractIOHandlerImpl
 
     virtual ~AbstractIOHandlerImpl() = default;
 
-    virtual std::future<void> flush()
+    std::future<void> flush()
     {
         using namespace auxiliary;
 

include/openPMD/IO/DummyIOHandler.hpp

@@ -44,6 +44,6 @@ class DummyIOHandler : public AbstractIOHandler
     /** No-op consistent with the IOHandler interface to enable library use
      * without IO.
      */
-    std::future<void> flush() override;
+    std::future<void> flush(internal::FlushParams const &) override;
 }; // DummyIOHandler
 } // namespace openPMD

include/openPMD/IO/HDF5/HDF5IOHandler.hpp

@@ -43,7 +43,7 @@ class HDF5IOHandler : public AbstractIOHandler
         return "HDF5";
     }
 
-    std::future<void> flush() override;
+    std::future<void> flush(internal::FlushParams const &) override;
 
 private:
     std::unique_ptr<HDF5IOHandlerImpl> m_impl;

include/openPMD/IO/HDF5/ParallelHDF5IOHandler.hpp

@@ -49,7 +49,7 @@ class ParallelHDF5IOHandler : public AbstractIOHandler
         return "MPI_HDF5";
     }
 
-    std::future<void> flush() override;
+    std::future<void> flush(internal::FlushParams const &) override;
 
 private:
     std::unique_ptr<ParallelHDF5IOHandlerImpl> m_impl;

include/openPMD/IO/JSON/JSONIOHandler.hpp

@@ -38,7 +38,7 @@ class JSONIOHandler : public AbstractIOHandler
         return "JSON";
     }
 
-    std::future<void> flush() override;
+    std::future<void> flush(internal::FlushParams const &) override;
 
 private:
     JSONIOHandlerImpl m_impl;

include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp

@@ -210,7 +210,7 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl
 
     void listAttributes(Writable *, Parameter<Operation::LIST_ATTS> &) override;
 
-    std::future<void> flush() override;
+    std::future<void> flush();
 
 private:
     using FILEHANDLE = std::fstream;

include/openPMD/Iteration.hpp

@@ -177,22 +177,26 @@ class Iteration : public LegacyAttributable
          * containing this iteration.
          */
         std::string filename;
+        bool beginStep = false;
     };
 
-    void flushFileBased(std::string const &, uint64_t);
-    void flushGroupBased(uint64_t);
-    void flushVariableBased(uint64_t);
-    void flush();
+    void flushFileBased(
+        std::string const &, uint64_t, internal::FlushParams const &);
+    void flushGroupBased(uint64_t, internal::FlushParams const &);
+    void flushVariableBased(uint64_t, internal::FlushParams const &);
+    void flush(internal::FlushParams const &);
     void deferParseAccess(DeferredParseAccess);
     /*
-     * Control flow for read(), readFileBased(), readGroupBased() and
-     * read_impl():
-     * read() is called as the entry point. File-based and group-based
+     * Control flow for runDeferredParseAccess(), readFileBased(),
+     * readGroupBased() and read_impl():
+     * runDeferredParseAccess() is called as the entry point.
+     * File-based and group-based
      * iteration layouts need to be parsed slightly differently:
      * In file-based iteration layout, each iteration's file also contains
      * attributes for the /data group. In group-based layout, those have
      * already been parsed during opening of the Series.
-     * Hence, read() will call either readFileBased() or readGroupBased() to
+     * Hence, runDeferredParseAccess() will call either readFileBased() or
+     * readGroupBased() to
      * allow for those different control flows.
      * Finally, read_impl() is called which contains the common parsing
      * logic for an iteration.
@@ -201,10 +205,10 @@ class Iteration : public LegacyAttributable
      * Calling it on an Iteration not yet parsed is an error.
      *
      */
-    void read();
     void reread(std::string const &path);
-    void readFileBased(std::string filePath, std::string const &groupPath);
-    void readGorVBased(std::string const &groupPath);
+    void readFileBased(
+        std::string filePath, std::string const &groupPath, bool beginStep);
+    void readGorVBased(std::string const &groupPath, bool beginStep);
     void read_impl(std::string const &groupPath);
 
     /**
@@ -261,7 +265,7 @@ class Iteration : public LegacyAttributable
      *
      * @return AdvanceStatus
      */
-    AdvanceStatus beginStep();
+    AdvanceStatus beginStep(bool reread);
 
     /**
      * @brief End an IO step on the IO file (or file-like object)

include/openPMD/Mesh.hpp

@@ -228,7 +228,8 @@ class Mesh : public BaseRecord<MeshRecordComponent>
 private:
     Mesh();
 
-    void flush_impl(std::string const &) override;
+    void
+    flush_impl(std::string const &, internal::FlushParams const &) override;
     void read() override;
 }; // Mesh
 

include/openPMD/ParticleSpecies.hpp

@@ -43,7 +43,7 @@ class ParticleSpecies : public Container<Record>
     ParticleSpecies();
 
     void read();
-    void flush(std::string const &) override;
+    void flush(std::string const &, internal::FlushParams const &) override;
 
     /**
      * @brief Check recursively whether this ParticleSpecies is dirty.

include/openPMD/Record.hpp

@@ -50,7 +50,8 @@ class Record : public BaseRecord<RecordComponent>
 private:
     Record();
 
-    void flush_impl(std::string const &) override;
+    void
+    flush_impl(std::string const &, internal::FlushParams const &) override;
     void read() override;
 }; // Record
 

include/openPMD/RecordComponent.hpp

@@ -250,7 +250,7 @@ class RecordComponent : public BaseRecordComponent
     std::shared_ptr<bool> m_hasBeenExtended = std::make_shared<bool>(false);
 
 private:
-    void flush(std::string const &);
+    void flush(std::string const &, internal::FlushParams const &);
     virtual void read();
 
     /**

include/openPMD/RecordComponent.tpp

@@ -276,7 +276,7 @@ RecordComponent::storeChunk( Offset o, Extent e, F && createBuffer )
      * Flush the openPMD hierarchy to the backend without flushing any actual
      * data yet.
      */
-    seriesFlush( FlushLevel::SkeletonOnly );
+    seriesFlush({FlushLevel::SkeletonOnly});
 
     size_t size = 1;
     for( auto ext : e )
@@ -303,16 +303,16 @@ RecordComponent::storeChunk( Offset o, Extent e, F && createBuffer )
     getBufferView.offset = o;
     getBufferView.extent = e;
     getBufferView.dtype = getDatatype();
-    IOHandler()->enqueue( IOTask( this, getBufferView ) );
-    IOHandler()->flush();
+    IOHandler()->enqueue(IOTask(this, getBufferView));
+    IOHandler()->flush(internal::defaultFlushParams);
     auto &out = *getBufferView.out;
-    if( !out.backendManagedBuffer )
+    if (!out.backendManagedBuffer)
     {
-        auto data = std::forward< F >( createBuffer )( size );
-        out.ptr = static_cast< void * >( data.get() );
-        storeChunk( std::move( data ), std::move( o ), std::move( e ) );
+        auto data = std::forward<F>(createBuffer)(size);
+        out.ptr = static_cast<void *>(data.get());
+        storeChunk(std::move(data), std::move(o), std::move(e));
     }
-    return DynamicMemoryView< T >{ std::move( getBufferView ), size, *this };
+    return DynamicMemoryView<T>{std::move(getBufferView), size, *this};
 }
 
 template< typename T >

include/openPMD/Series.hpp

@@ -408,24 +408,30 @@ class SeriesInterface : public AttributableInterface
      *
      * @param begin Start of the range of iterations to flush.
      * @param end End of the range of iterations to flush.
-     * @param level Flush level, as documented in AbstractIOHandler.hpp.
+     * @param flushParams Flush params, as documented in AbstractIOHandler.hpp.
      * @param flushIOHandler Tasks will always be enqueued to the backend.
      *     If this flag is true, tasks will be flushed to the backend.
      */
     std::future<void> flush_impl(
         iterations_iterator begin,
         iterations_iterator end,
-        FlushLevel level,
+        internal::FlushParams flushParams,
         bool flushIOHandler = true);
-    void flushFileBased(iterations_iterator begin, iterations_iterator end);
+    void flushFileBased(
+        iterations_iterator begin,
+        iterations_iterator end,
+        internal::FlushParams flushParams);
     /*
      * Group-based and variable-based iteration layouts share a lot of logic
      * (realistically, the variable-based iteration layout only throws out
      *  one layer in the hierarchy).
      * As a convention, methods that deal with both layouts are called
      * .*GorVBased, short for .*GroupOrVariableBased
      */
-    void flushGorVBased(iterations_iterator begin, iterations_iterator end);
+    void flushGorVBased(
+        iterations_iterator begin,
+        iterations_iterator end,
+        internal::FlushParams flushParams);
     void flushMeshesPath();
     void flushParticlesPath();
     void readFileBased();

include/openPMD/Span.hpp

@@ -123,7 +123,7 @@ class DynamicMemoryView
             // might need to update
             m_recordComponent.IOHandler()->enqueue(
                 IOTask(&m_recordComponent, m_param));
-            m_recordComponent.IOHandler()->flush();
+            m_recordComponent.IOHandler()->flush(internal::defaultFlushParams);
         }
         return Span<T>{static_cast<T *>(m_param.out->ptr), m_size};
     }

include/openPMD/backend/Attributable.hpp

@@ -264,9 +264,9 @@ class AttributableInterface
     Iteration &containingIteration();
     /** @} */
 
-    void seriesFlush(FlushLevel);
+    void seriesFlush(internal::FlushParams);
 
-    void flushAttributes();
+    void flushAttributes(internal::FlushParams const &);
     enum ReadMode
     {
         /**