From 21430bfbf0f75314f3573f9ff2a41700441f0b23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 31 Jul 2024 14:37:32 +0200 Subject: [PATCH] Also make this available as a default parameter --- include/openPMD/IO/HDF5/HDF5IOHandlerImpl.hpp | 6 ++- src/IO/HDF5/HDF5IOHandler.cpp | 40 ++++++++-------- src/IO/HDF5/ParallelHDF5IOHandler.cpp | 47 ++++++++++++------- test/ParallelIOTest.cpp | 7 ++- 4 files changed, 60 insertions(+), 40 deletions(-) diff --git a/include/openPMD/IO/HDF5/HDF5IOHandlerImpl.hpp b/include/openPMD/IO/HDF5/HDF5IOHandlerImpl.hpp index f8b649aaba..e4efc06ea6 100644 --- a/include/openPMD/IO/HDF5/HDF5IOHandlerImpl.hpp +++ b/include/openPMD/IO/HDF5/HDF5IOHandlerImpl.hpp @@ -38,6 +38,9 @@ namespace openPMD #if openPMD_HAVE_HDF5 class HDF5IOHandlerImpl : public AbstractIOHandlerImpl { + friend class HDF5IOHandler; + friend class ParallelHDF5IOHandler; + public: HDF5IOHandlerImpl( AbstractIOHandler *, @@ -122,7 +125,8 @@ class HDF5IOHandlerImpl : public AbstractIOHandlerImpl #endif json::TracingJSON m_config; - std::optional m_buffered_dataset_config; + nlohmann::json m_global_dataset_config; + nlohmann::json m_global_flush_config; private: struct File diff --git a/src/IO/HDF5/HDF5IOHandler.cpp b/src/IO/HDF5/HDF5IOHandler.cpp index 537994b542..bf8100c01d 100644 --- a/src/IO/HDF5/HDF5IOHandler.cpp +++ b/src/IO/HDF5/HDF5IOHandler.cpp @@ -150,11 +150,29 @@ HDF5IOHandlerImpl::HDF5IOHandlerImpl( { constexpr char const *const init_json_shadow_str = R"( + { + "dataset": { + "chunks": null + }, + "independent_stores": null + })"; + constexpr char const *const dataset_cfg_mask = R"( { "dataset": { "chunks": null } })"; + constexpr char const *const flush_cfg_mask = R"( + { + "independent_stores": null + })"; + m_global_dataset_config = m_config.json(); + json::filterByTemplate( + m_global_dataset_config, + nlohmann::json::parse(dataset_cfg_mask)); + m_global_flush_config = m_config.json(); + json::filterByTemplate( + m_global_flush_config, nlohmann::json::parse(flush_cfg_mask)); auto init_json_shadow = nlohmann::json::parse(init_json_shadow_str); json::merge(m_config.getShadow(), init_json_shadow); } @@ -484,34 +502,18 @@ void HDF5IOHandlerImpl::createDataset( } json::TracingJSON config = [&]() { - if (!m_buffered_dataset_config.has_value()) - { - // we are only interested in these values from the global config - constexpr char const *const mask_for_global_conf = R"( - { - "dataset": { - "chunks": null - } - })"; - m_buffered_dataset_config = m_config.json(); - json::filterByTemplate( - *m_buffered_dataset_config, - nlohmann::json::parse(mask_for_global_conf)); - } - auto const &buffered_config = *m_buffered_dataset_config; auto parsed_config = json::parseOptions( parameters.options, /* considerFiles = */ false); if (auto hdf5_config_it = parsed_config.config.find("hdf5"); hdf5_config_it != parsed_config.config.end()) { - auto copy = buffered_config; + auto copy = m_global_dataset_config; json::merge(copy, hdf5_config_it.value()); - copy = nlohmann::json{{"hdf5", std::move(copy)}}; - parsed_config.config = std::move(copy); + hdf5_config_it.value() = std::move(copy); } else { - parsed_config.config["hdf5"] = buffered_config; + parsed_config.config["hdf5"] = m_global_dataset_config; } return parsed_config; }(); diff --git a/src/IO/HDF5/ParallelHDF5IOHandler.cpp b/src/IO/HDF5/ParallelHDF5IOHandler.cpp index 0eb0a913c0..00d5741457 100644 --- a/src/IO/HDF5/ParallelHDF5IOHandler.cpp +++ b/src/IO/HDF5/ParallelHDF5IOHandler.cpp @@ -68,6 +68,17 @@ ParallelHDF5IOHandler::~ParallelHDF5IOHandler() = default; std::future ParallelHDF5IOHandler::flush(internal::ParsedFlushParams ¶ms) { + if (auto hdf5_config_it = params.backendConfig.json().find("hdf5"); + hdf5_config_it != params.backendConfig.json().end()) + { + auto copied_global_cfg = m_impl->m_global_flush_config; + json::merge(copied_global_cfg, hdf5_config_it.value()); + hdf5_config_it.value() = std::move(copied_global_cfg); + } + else + { + params.backendConfig["hdf5"].json() = m_impl->m_global_flush_config; + } return m_impl->flush(params); } @@ -321,26 +332,26 @@ ParallelHDF5IOHandlerImpl::ParallelHDF5IOHandlerImpl( {"hdf5", "vfd", "type"}, "Unknown value: '" + user_specified_type + "'."); } + } - // unused params - auto shadow = m_config.invertShadow(); - if (shadow.size() > 0) + // unused params + auto shadow = m_config.invertShadow(); + if (shadow.size() > 0) + { + switch (m_config.originallySpecifiedAs) { - switch (m_config.originallySpecifiedAs) - { - case json::SupportedLanguages::JSON: - std::cerr << "Warning: parts of the backend configuration for " - "HDF5 remain unused:\n" - << shadow << std::endl; - break; - case json::SupportedLanguages::TOML: { - auto asToml = json::jsonToToml(shadow); - std::cerr << "Warning: parts of the backend configuration for " - "HDF5 remain unused:\n" - << json::format_toml(asToml) << std::endl; - break; - } - } + case json::SupportedLanguages::JSON: + std::cerr << "Warning: parts of the backend configuration for " + "HDF5 remain unused:\n" + << shadow << std::endl; + break; + case json::SupportedLanguages::TOML: { + auto asToml = json::jsonToToml(shadow); + std::cerr << "Warning: parts of the backend configuration for " + "HDF5 remain unused:\n" + << json::format_toml(asToml) << std::endl; + break; + } } } } diff --git a/test/ParallelIOTest.cpp b/test/ParallelIOTest.cpp index 4f1bf25adf..5819ba899d 100644 --- a/test/ParallelIOTest.cpp +++ b/test/ParallelIOTest.cpp @@ -302,8 +302,11 @@ TEST_CASE("hdf5_write_test", "[parallel][hdf5]") MPI_Comm_rank(MPI_COMM_WORLD, &mpi_r); auto mpi_size = static_cast(mpi_s); auto mpi_rank = static_cast(mpi_r); - Series o = - Series("../samples/parallel_write.h5", Access::CREATE, MPI_COMM_WORLD); + Series o = Series( + "../samples/parallel_write.h5", + Access::CREATE, + MPI_COMM_WORLD, + "hdf5.independent_stores = false"); o.setAuthor("Parallel HDF5"); ParticleSpecies &e = o.iterations[1].particles["e"];