[WIP] Unify Random-Access API and Streaming API into Series::snapshots()

CMakeLists.txt

@@ -459,12 +459,11 @@ set(CORE_SOURCE
         src/Mesh.cpp
         src/ParticlePatches.cpp
         src/ParticleSpecies.cpp
-        src/ReadIterations.cpp
         src/Record.cpp
+        src/ReadIterations.cpp
         src/RecordComponent.cpp
         src/Series.cpp
         src/version.cpp
-        src/WriteIterations.cpp
         src/auxiliary/Date.cpp
         src/auxiliary/Filesystem.cpp
         src/auxiliary/JSON.cpp
@@ -476,11 +475,19 @@ set(CORE_SOURCE
         src/backend/PatchRecordComponent.cpp
         src/backend/Writable.cpp
         src/benchmark/mpi/OneDimensionalBlockSlicer.cpp
-        src/helper/list_series.cpp)
+        src/helper/list_series.cpp
+        src/snapshots/ContainerImpls.cpp
+        src/snapshots/ContainerTraits.cpp
+        src/snapshots/IteratorHelpers.cpp
+        src/snapshots/IteratorTraits.cpp
+        src/snapshots/RandomAccessIterator.cpp
+        src/snapshots/Snapshots.cpp
+        src/snapshots/StatefulIterator.cpp)
 set(IO_SOURCE
         src/IO/AbstractIOHandler.cpp
         src/IO/AbstractIOHandlerImpl.cpp
         src/IO/AbstractIOHandlerHelper.cpp
+        src/IO/Access.cpp
         src/IO/DummyIOHandler.cpp
         src/IO/IOTask.cpp
         src/IO/FlushParams.cpp
@@ -848,8 +855,20 @@ if(openPMD_BUILD_TESTING)
         target_compile_definitions(CatchRunner PUBLIC openPMD_HAVE_MPI=1)
     endif()
 
+    macro(additional_testing_sources test_name out_list)
+        if(${test_name} STREQUAL "SerialIO")
+            list(APPEND ${out_list}
+                test/Files_SerialIO/close_iteration_test.cpp
+                test/Files_SerialIO/filebased_write_test.cpp
+            )
+        endif()
+    endmacro()
+
     foreach(testname ${openPMD_TEST_NAMES})
-        add_executable(${testname}Tests test/${testname}Test.cpp)
+        set(ADDITIONAL_SOURCE_FILES "")
+        additional_testing_sources(${testname} ADDITIONAL_SOURCE_FILES)
+        add_executable(${testname}Tests test/${testname}Test.cpp ${ADDITIONAL_SOURCE_FILES})
+        target_include_directories(${testname}Tests PRIVATE test/Files_${testname}/)
         openpmd_cxx_required(${testname}Tests)
         set_target_properties(${testname}Tests PROPERTIES
             COMPILE_PDB_NAME ${testname}Tests

check_recursive_include.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+
+import re
+import subprocess
+import sys
+
+
+def track_includes(start_file):
+    remove_start = re.compile("include/")
+
+    def clean(file):
+        return re.sub(remove_start, "", file)
+
+    unseen_files = [start_file]
+    res = {}
+    # import ipdb
+    # ipdb.set_trace(context=30)
+    while unseen_files:
+        current_file = unseen_files[0]
+        del unseen_files[0]
+
+        cmd = ["grep", "-Rl", clean(current_file), "include/"]
+        try:
+            next_files = subprocess.check_output(cmd)
+            lines = [line for line in next_files.decode().splitlines() if line]
+        except subprocess.CalledProcessError:
+            lines = []
+
+        res[current_file] = lines
+        for line in lines:
+            if line not in res:
+                unseen_files.append(line)
+    return res
+
+
+if __name__ == "__main__":
+
+    remove_start = re.compile("include/(openPMD/)?")
+    remove_slash = re.compile("/|\\.")
+
+    def clean(file):
+        return re.sub(remove_slash, "_", re.sub(remove_start, "", file))
+
+    res = track_includes(sys.argv[1])
+    print("digraph {")
+    for key, values in res.items():
+        key = clean(key)
+        for target in values:
+            print("\t{} -> {};".format(key, clean(target)))
+    print("}")

examples/10_streaming_read.cpp

@@ -34,10 +34,9 @@ int main()
     // in streaming setups, e.g. an iteration cannot be opened again once
     // it has been closed.
     // `Series::iterations` can be directly accessed in random-access workflows.
-    for (IndexedIteration iteration : series.readIterations())
+    for (auto &[index, iteration] : series.snapshots())
     {
-        std::cout << "Current iteration: " << iteration.iterationIndex
-                  << std::endl;
+        std::cout << "Current iteration: " << index << std::endl;
         Record electronPositions = iteration.particles["e"]["position"];
         std::array<RecordComponent::shared_ptr_dataset_types, 3> loadedChunks;
         std::array<Extent, 3> extents;

examples/10_streaming_write.cpp

@@ -1,3 +1,5 @@
+#include "openPMD/Series.hpp"
+#include "openPMD/snapshots/Snapshots.hpp"
 #include <openPMD/openPMD.hpp>
 
 #include <algorithm>
@@ -43,7 +45,7 @@ int main()
     // in streaming setups, e.g. an iteration cannot be opened again once
     // it has been closed.
     // `Series::iterations` can be directly accessed in random-access workflows.
-    WriteIterations iterations = series.writeIterations();
+    auto iterations = series.snapshots(SnapshotWorkflow::Synchronous);
     for (size_t i = 0; i < 100; ++i)
     {
         Iteration iteration = iterations[i];

examples/2_read_serial.cpp

@@ -34,13 +34,13 @@ int main()
     cout << "Read a Series with openPMD standard version " << series.openPMD()
          << '\n';
 
-    cout << "The Series contains " << series.iterations.size()
+    cout << "The Series contains " << series.snapshots().size()
          << " iterations:";
-    for (auto const &i : series.iterations)
+    for (auto const &i : series.snapshots())
         cout << "\n\t" << i.first;
     cout << '\n';
 
-    Iteration i = series.iterations[100];
+    Iteration i = series.snapshots()[100];
     cout << "Iteration 100 contains " << i.meshes.size() << " meshes:";
     for (auto const &m : i.meshes)
         cout << "\n\t" << m.first;

include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp

@@ -66,6 +66,21 @@ namespace adios_defs
         Yes,
         No
     };
+
+    /*
+     * Necessary to implement the `reopen` flag of
+     * `Parameter<Operation::OPEN_FILE>`. The distinction between Open and
+     * Reopen is necessary for Write workflows in file-based encoding. In order
+     * to write new data to an Iteration that was created and closed previously,
+     * the only applicable access mode is Append mode, ideally in conjunction
+     * with `SetParameter("FlattenSteps", "ON")`.
+     */
+    enum class OpenFileAs
+    {
+        Create,
+        Open,
+        Reopen
+    };
 } // namespace adios_defs
 
 /*

include/openPMD/IO/ADIOS/ADIOS2File.hpp

@@ -229,7 +229,10 @@ class ADIOS2File
 
     using AttributeMap_t = std::map<std::string, adios2::Params>;
 
-    ADIOS2File(ADIOS2IOHandlerImpl &impl, InvalidatableFile file);
+    ADIOS2File(
+        ADIOS2IOHandlerImpl &impl,
+        InvalidatableFile file,
+        adios_defs::OpenFileAs);
 
     ~ADIOS2File();
 

include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp

@@ -212,7 +212,8 @@ class ADIOS2IOHandlerImpl
      * @brief The ADIOS2 access type to chose for Engines opened
      * within this instance.
      */
-    adios2::Mode adios2AccessMode(std::string const &fullPath);
+    adios2::Mode
+    adios2AccessMode(std::string const &fullPath, adios_defs::OpenFileAs);
 
     FlushTarget m_flushTarget = FlushTarget::Disk;
 
@@ -403,9 +404,11 @@ class ADIOS2IOHandlerImpl
      */
     GroupOrDataset groupOrDataset(Writable *);
 
-    enum class IfFileNotOpen : bool
+    enum class IfFileNotOpen : char
     {
+        ReopenImplicitly,
         OpenImplicitly,
+        CreateImplicitly,
         ThrowError
     };
 

include/openPMD/IO/AbstractIOHandler.hpp

@@ -218,6 +218,10 @@ class AbstractIOHandler
                 break;
             }
         }
+        else
+        {
+            m_backendAccess = m_frontendAccess;
+        }
 
         m_encoding = encoding;
     }

include/openPMD/IO/Access.hpp

@@ -82,6 +82,8 @@ enum class Access
     APPEND //!< write new iterations to an existing series without reading
 }; // Access
 
+std::ostream &operator<<(std::ostream &o, Access const &a);
+
 namespace access
 {
     inline bool readOnly(Access access)

include/openPMD/IO/IOTask.hpp

@@ -188,6 +188,8 @@ struct OPENPMDAPI_EXPORT Parameter<Operation::OPEN_FILE>
     }
 
     std::string name = "";
+    // true <-> file was previously created and is now opened again
+    bool reopen = false;
     using ParsePreference = internal::ParsePreference;
     std::shared_ptr<ParsePreference> out_parsePreference =
         std::make_shared<ParsePreference>(ParsePreference::UpFront);

include/openPMD/Iteration.hpp

@@ -48,10 +48,8 @@ namespace internal
         Open, //!< Iteration has not been closed
         ClosedInFrontend, /*!< Iteration has been closed, but task has not yet
                                been propagated to the backend */
-        ClosedInBackend, /*!< Iteration has been closed and task has been
+        Closed, /*!< Iteration has been closed and task has been
                               propagated to the backend */
-        ClosedTemporarily /*!< Iteration has been closed internally and may
-                               be reopened later */
     };
 
     struct DeferredParseAccess
@@ -71,11 +69,6 @@ namespace internal
          * (Group- and variable-based parsing shares the same code logic.)
          */
         bool fileBased = false;
-        /**
-         * If fileBased == true, the file name (without file path) of the file
-         * containing this iteration.
-         */
-        std::string filename;
         bool beginStep = false;
     };
 
@@ -92,6 +85,14 @@ namespace internal
          * overwritten.
          */
         CloseStatus m_closed = CloseStatus::Open;
+        /*
+         * While parsing a file-based Series, each file is opened, read, then
+         * closed again. Explicitly `Iteration::open()`ing a file should only be
+         * necessary after having explicitly closed it (or in
+         * defer_iteration_parsing mode). So, the parsing procedures will set
+         * this flag as true when closing an Iteration.
+         */
+        bool allow_reopening_implicitly = false;
 
         /**
          * Whether a step is currently active for this iteration.
@@ -107,14 +108,6 @@ namespace internal
          * Otherwise empty.
          */
         std::optional<DeferredParseAccess> m_deferredParseAccess{};
-
-        /**
-         * Upon reading a file, set this field to the used file name.
-         * In inconsistent iteration paddings, we must remember the name of the
-         * file since it cannot be reconstructed from the filename pattern
-         * alone.
-         */
-        std::optional<std::string> m_overrideFilebasedFilename{};
     };
 } // namespace internal
 /** @brief  Logical compilation of data from one snapshot (e.g. a single
@@ -128,15 +121,17 @@ class Iteration : public Attributable
     template <typename T, typename T_key, typename T_container>
     friend class Container;
     friend class Series;
-    friend class WriteIterations;
-    friend class SeriesIterator;
     friend class internal::AttributableData;
     template <typename T>
     friend T &internal::makeOwning(T &self, Series);
+    friend class StatefulIterator;
+    friend class StatefulSnapshotsContainer;
 
 public:
     Iteration(Iteration const &) = default;
+    Iteration(Iteration &&) = default;
     Iteration &operator=(Iteration const &) = default;
+    Iteration &operator=(Iteration &&) = default;
 
     using IterationIndex_t = uint64_t;
 
@@ -219,12 +214,19 @@ class Iteration : public Attributable
 
     /**
      * @brief Has the iteration been closed?
-     *        A closed iteration may not (yet) be reopened.
      *
      * @return Whether the iteration has been closed.
      */
     bool closed() const;
 
+    /**
+     * @brief Has the iteration been parsed yet?
+              If not, it will contain no structure yet.
+     *
+     * @return Whether the iteration has been parsed.
+     */
+    bool parsed() const;
+
     /**
      * @brief Has the iteration been closed by the writer?
      *        Background: Upon calling Iteration::close(), the openPMD API
@@ -298,6 +300,7 @@ class Iteration : public Attributable
      */
     void reread(std::string const &path);
     void readFileBased(
+        IterationIndex_t,
         std::string const &filePath,
         std::string const &groupPath,
         bool beginStep);
@@ -313,7 +316,7 @@ class Iteration : public Attributable
      */
     struct BeginStepStatus
     {
-        using AvailableIterations_t = std::optional<std::deque<uint64_t> >;
+        using AvailableIterations_t = std::vector<uint64_t>;
 
         AdvanceStatus stepStatus{};
         /*
@@ -355,11 +358,8 @@ class Iteration : public Attributable
      * Useful in group-based iteration encoding where the Iteration will only
      * be known after opening the step.
      */
-    static BeginStepStatus beginStep(
-        std::optional<Iteration> thisObject,
-        Series &series,
-        bool reread,
-        std::set<IterationIndex_t> const &ignoreIterations = {});
+    static BeginStepStatus
+    beginStep(std::optional<Iteration> thisObject, Series &series, bool reread);
 
     /**
      * @brief End an IO step on the IO file (or file-like object)
@@ -433,13 +433,17 @@ inline T Iteration::dt() const
  */
 class IndexedIteration : public Iteration
 {
-    friend class SeriesIterator;
-    friend class WriteIterations;
+    friend class StatefulIterator;
+    friend class LegacyIteratorAdaptor;
 
 public:
     using index_t = Iteration::IterationIndex_t;
     index_t const iterationIndex;
 
+    inline IndexedIteration(std::pair<index_t const, Iteration> pair)
+        : Iteration(std::move(pair.second)), iterationIndex(pair.first)
+    {}
+
 private:
     template <typename Iteration_t>
     IndexedIteration(Iteration_t &&it, index_t index)