Also make this available as a default parameter

openPMD · Jul 31, 2024 · 21430bf · 21430bf
1 parent 83875c1
commit 21430bf
Show file tree

Hide file tree

Showing 4 changed files with 60 additions and 40 deletions.
diff --git a/include/openPMD/IO/HDF5/HDF5IOHandlerImpl.hpp b/include/openPMD/IO/HDF5/HDF5IOHandlerImpl.hpp
@@ -38,6 +38,9 @@ namespace openPMD
 #if openPMD_HAVE_HDF5
 class HDF5IOHandlerImpl : public AbstractIOHandlerImpl
 {
+    friend class HDF5IOHandler;
+    friend class ParallelHDF5IOHandler;
+
 public:
     HDF5IOHandlerImpl(
         AbstractIOHandler *,
@@ -122,7 +125,8 @@ class HDF5IOHandlerImpl : public AbstractIOHandlerImpl
 #endif
 
     json::TracingJSON m_config;
-    std::optional<nlohmann::json> m_buffered_dataset_config;
+    nlohmann::json m_global_dataset_config;
+    nlohmann::json m_global_flush_config;
 
 private:
     struct File

diff --git a/src/IO/HDF5/HDF5IOHandler.cpp b/src/IO/HDF5/HDF5IOHandler.cpp
@@ -150,11 +150,29 @@ HDF5IOHandlerImpl::HDF5IOHandlerImpl(
 
         {
             constexpr char const *const init_json_shadow_str = R"(
+            {
+              "dataset": {
+                "chunks": null
+              },
+              "independent_stores": null
+            })";
+            constexpr char const *const dataset_cfg_mask = R"(
             {
               "dataset": {
                 "chunks": null
               }
             })";
+            constexpr char const *const flush_cfg_mask = R"(
+            {
+              "independent_stores": null
+            })";
+            m_global_dataset_config = m_config.json();
+            json::filterByTemplate(
+                m_global_dataset_config,
+                nlohmann::json::parse(dataset_cfg_mask));
+            m_global_flush_config = m_config.json();
+            json::filterByTemplate(
+                m_global_flush_config, nlohmann::json::parse(flush_cfg_mask));
             auto init_json_shadow = nlohmann::json::parse(init_json_shadow_str);
             json::merge(m_config.getShadow(), init_json_shadow);
         }
@@ -484,34 +502,18 @@ void HDF5IOHandlerImpl::createDataset(
         }
 
         json::TracingJSON config = [&]() {
-            if (!m_buffered_dataset_config.has_value())
-            {
-                // we are only interested in these values from the global config
-                constexpr char const *const mask_for_global_conf = R"(
-                {
-                "dataset": {
-                    "chunks": null
-                }
-                })";
-                m_buffered_dataset_config = m_config.json();
-                json::filterByTemplate(
-                    *m_buffered_dataset_config,
-                    nlohmann::json::parse(mask_for_global_conf));
-            }
-            auto const &buffered_config = *m_buffered_dataset_config;
             auto parsed_config = json::parseOptions(
                 parameters.options, /* considerFiles = */ false);
             if (auto hdf5_config_it = parsed_config.config.find("hdf5");
                 hdf5_config_it != parsed_config.config.end())
             {
-                auto copy = buffered_config;
+                auto copy = m_global_dataset_config;
                 json::merge(copy, hdf5_config_it.value());
-                copy = nlohmann::json{{"hdf5", std::move(copy)}};
-                parsed_config.config = std::move(copy);
+                hdf5_config_it.value() = std::move(copy);
             }
             else
             {
-                parsed_config.config["hdf5"] = buffered_config;
+                parsed_config.config["hdf5"] = m_global_dataset_config;
             }
             return parsed_config;
         }();

diff --git a/src/IO/HDF5/ParallelHDF5IOHandler.cpp b/src/IO/HDF5/ParallelHDF5IOHandler.cpp
@@ -68,6 +68,17 @@ ParallelHDF5IOHandler::~ParallelHDF5IOHandler() = default;
 std::future<void>
 ParallelHDF5IOHandler::flush(internal::ParsedFlushParams &params)
 {
+    if (auto hdf5_config_it = params.backendConfig.json().find("hdf5");
+        hdf5_config_it != params.backendConfig.json().end())
+    {
+        auto copied_global_cfg = m_impl->m_global_flush_config;
+        json::merge(copied_global_cfg, hdf5_config_it.value());
+        hdf5_config_it.value() = std::move(copied_global_cfg);
+    }
+    else
+    {
+        params.backendConfig["hdf5"].json() = m_impl->m_global_flush_config;
+    }
     return m_impl->flush(params);
 }
 
@@ -321,26 +332,26 @@ ParallelHDF5IOHandlerImpl::ParallelHDF5IOHandlerImpl(
                 {"hdf5", "vfd", "type"},
                 "Unknown value: '" + user_specified_type + "'.");
         }
+    }
 
-        // unused params
-        auto shadow = m_config.invertShadow();
-        if (shadow.size() > 0)
+    // unused params
+    auto shadow = m_config.invertShadow();
+    if (shadow.size() > 0)
+    {
+        switch (m_config.originallySpecifiedAs)
         {
-            switch (m_config.originallySpecifiedAs)
-            {
-            case json::SupportedLanguages::JSON:
-                std::cerr << "Warning: parts of the backend configuration for "
-                             "HDF5 remain unused:\n"
-                          << shadow << std::endl;
-                break;
-            case json::SupportedLanguages::TOML: {
-                auto asToml = json::jsonToToml(shadow);
-                std::cerr << "Warning: parts of the backend configuration for "
-                             "HDF5 remain unused:\n"
-                          << json::format_toml(asToml) << std::endl;
-                break;
-            }
-            }
+        case json::SupportedLanguages::JSON:
+            std::cerr << "Warning: parts of the backend configuration for "
+                         "HDF5 remain unused:\n"
+                      << shadow << std::endl;
+            break;
+        case json::SupportedLanguages::TOML: {
+            auto asToml = json::jsonToToml(shadow);
+            std::cerr << "Warning: parts of the backend configuration for "
+                         "HDF5 remain unused:\n"
+                      << json::format_toml(asToml) << std::endl;
+            break;
+        }
         }
     }
 }

diff --git a/test/ParallelIOTest.cpp b/test/ParallelIOTest.cpp
@@ -302,8 +302,11 @@ TEST_CASE("hdf5_write_test", "[parallel][hdf5]")
     MPI_Comm_rank(MPI_COMM_WORLD, &mpi_r);
     auto mpi_size = static_cast<uint64_t>(mpi_s);
     auto mpi_rank = static_cast<uint64_t>(mpi_r);
-    Series o =
-        Series("../samples/parallel_write.h5", Access::CREATE, MPI_COMM_WORLD);
+    Series o = Series(
+        "../samples/parallel_write.h5",
+        Access::CREATE,
+        MPI_COMM_WORLD,
+        "hdf5.independent_stores = false");
 
     o.setAuthor("Parallel HDF5");
     ParticleSpecies &e = o.iterations[1].particles["e"];