From fdcd9f177d4da8d442ff1bb6cb5963896bfb3598 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 4 Aug 2023 11:55:37 +0200 Subject: [PATCH 01/23] Introduce dataset template mode to JSON backend --- CMakeLists.txt | 1 + include/openPMD/IO/JSON/JSONIOHandler.hpp | 1 + include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp | 23 +- src/IO/JSON/JSONIOHandlerImpl.cpp | 361 ++++++++++++++---- test/SerialIOTest.cpp | 34 +- 5 files changed, 339 insertions(+), 81 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index fc5d6d5de5..7b221400dc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -691,6 +691,7 @@ set(openPMD_EXAMPLE_NAMES 10_streaming_read 12_span_write 13_write_dynamic_configuration + 14_toml_template ) set(openPMD_PYTHON_EXAMPLE_NAMES 2_read_serial diff --git a/include/openPMD/IO/JSON/JSONIOHandler.hpp b/include/openPMD/IO/JSON/JSONIOHandler.hpp index 7cb6870f5b..e22fdb93d1 100644 --- a/include/openPMD/IO/JSON/JSONIOHandler.hpp +++ b/include/openPMD/IO/JSON/JSONIOHandler.hpp @@ -23,6 +23,7 @@ #include "openPMD/IO/AbstractIOHandler.hpp" #include "openPMD/IO/JSON/JSONIOHandlerImpl.hpp" +#include "openPMD/auxiliary/JSON_internal.hpp" #if openPMD_HAVE_MPI #include diff --git a/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp index b67ac9138a..875961ea7a 100644 --- a/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp +++ b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp @@ -265,8 +265,27 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl */ FileFormat m_fileFormat{}; + std::string backendConfigKey() const; + + /* + * First return value: The location of the JSON value (either "json" or + * "toml") Second return value: The value that was maybe found at this place + */ + std::pair> + getBackendConfig(openPMD::json::TracingJSON &) const; + std::string m_originalExtension; + enum class IOMode + { + Dataset, + Template + }; + + IOMode m_mode = IOMode::Dataset; + + IOMode retrieveDatasetMode(openPMD::json::TracingJSON &config) const; + // HELPER FUNCTIONS // will use the IOHandler to retrieve the correct directory. @@ -313,7 +332,7 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl // essentially: m_i = \prod_{j=0}^{i-1} extent_j static Extent getMultiplicators(Extent const &extent); - static Extent getExtent(nlohmann::json &j); + static std::pair getExtent(nlohmann::json &j); // remove single '/' in the beginning and end of a string static std::string removeSlashes(std::string); @@ -371,7 +390,7 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl // check whether the json reference contains a valid dataset template - void verifyDataset(Param const ¶meters, nlohmann::json &); + IOMode verifyDataset(Param const ¶meters, nlohmann::json &); static nlohmann::json platformSpecifics(); diff --git a/src/IO/JSON/JSONIOHandlerImpl.cpp b/src/IO/JSON/JSONIOHandlerImpl.cpp index e06aa36ed8..cacbbac18a 100644 --- a/src/IO/JSON/JSONIOHandlerImpl.cpp +++ b/src/IO/JSON/JSONIOHandlerImpl.cpp @@ -125,18 +125,120 @@ namespace } return *accum_ptr; } + + void warnUnusedJson(openPMD::json::TracingJSON const &jsonConfig) + { + auto shadow = jsonConfig.invertShadow(); + if (shadow.size() > 0) + { + switch (jsonConfig.originallySpecifiedAs) + { + case openPMD::json::SupportedLanguages::JSON: + std::cerr << "Warning: parts of the backend configuration for " + "JSON/TOML backend remain unused:\n" + << shadow << std::endl; + break; + case openPMD::json::SupportedLanguages::TOML: { + auto asToml = openPMD::json::jsonToToml(shadow); + std::cerr << "Warning: parts of the backend configuration for " + "JSON/TOML backend remain unused:\n" + << asToml << std::endl; + break; + } + } + } + } } // namespace +auto JSONIOHandlerImpl::retrieveDatasetMode( + openPMD::json::TracingJSON &config) const -> IOMode +{ + IOMode res = m_mode; + if (auto [configLocation, maybeConfig] = getBackendConfig(config); + maybeConfig.has_value()) + { + auto jsonConfig = maybeConfig.value(); + if (jsonConfig.json().contains("dataset")) + { + auto datasetConfig = jsonConfig["dataset"]; + if (datasetConfig.json().contains("mode")) + { + auto modeOption = openPMD::json::asLowerCaseStringDynamic( + datasetConfig["mode"].json()); + if (!modeOption.has_value()) + { + throw error::BackendConfigSchema( + {configLocation, "mode"}, + "Invalid value of non-string type (accepted values are " + "'dataset' and 'template'."); + } + auto mode = modeOption.value(); + if (mode == "dataset") + { + res = IOMode::Dataset; + } + else if (mode == "template") + { + res = IOMode::Template; + } + else + { + throw error::BackendConfigSchema( + {configLocation, "dataset", "mode"}, + "Invalid value: '" + mode + + "' (accepted values are 'dataset' and 'template'."); + } + } + } + } + return res; +} + +std::string JSONIOHandlerImpl::backendConfigKey() const +{ + switch (m_fileFormat) + { + case FileFormat::Json: + return "json"; + case FileFormat::Toml: + return "toml"; + } + throw std::runtime_error("Unreachable!"); +} + +std::pair> +JSONIOHandlerImpl::getBackendConfig(openPMD::json::TracingJSON &config) const +{ + std::string configLocation = backendConfigKey(); + if (config.json().contains(configLocation)) + { + return std::make_pair( + std::move(configLocation), config[configLocation]); + } + else + { + return std::make_pair(std::move(configLocation), std::nullopt); + } +} + JSONIOHandlerImpl::JSONIOHandlerImpl( AbstractIOHandler *handler, - // NOLINTNEXTLINE(performance-unnecessary-value-param) - [[maybe_unused]] openPMD::json::TracingJSON config, + openPMD::json::TracingJSON config, FileFormat format, std::string originalExtension) : AbstractIOHandlerImpl(handler) , m_fileFormat{format} , m_originalExtension{std::move(originalExtension)} -{} +{ + m_mode = retrieveDatasetMode(config); + + if (auto [_, backendConfig] = getBackendConfig(config); + backendConfig.has_value()) + { + (void)_; + warnUnusedJson(backendConfig.value()); + } +} #if openPMD_HAVE_MPI JSONIOHandlerImpl::JSONIOHandlerImpl( @@ -286,6 +388,18 @@ void JSONIOHandlerImpl::createDataset( "ADIOS1", "Joined Arrays currently only supported in ADIOS2"); } + openPMD::json::TracingJSON config = openPMD::json::parseOptions( + parameter.options, /* considerFiles = */ false); + // Retrieves mode from dataset-specific configuration, falls back to global + // value if not defined + IOMode localMode = retrieveDatasetMode(config); + + parameter.warnUnusedParameters( + config, + backendConfigKey(), + "Warning: parts of the dataset-specific backend configuration for " + "JSON/TOML backend remain unused"); + if (!writable->written) { /* Sanitize name */ @@ -303,23 +417,41 @@ void JSONIOHandlerImpl::createDataset( setAndGetFilePosition(writable, name); auto &dset = jsonVal[name]; dset["datatype"] = datatypeToString(parameter.dtype); - auto extent = parameter.extent; - switch (parameter.dtype) + + switch (localMode) { - case Datatype::CFLOAT: - case Datatype::CDOUBLE: - case Datatype::CLONG_DOUBLE: { - extent.push_back(2); + case IOMode::Dataset: { + auto extent = parameter.extent; + switch (parameter.dtype) + { + case Datatype::CFLOAT: + case Datatype::CDOUBLE: + case Datatype::CLONG_DOUBLE: { + extent.push_back(2); + break; + } + default: + break; + } + // TOML does not support nulls, so initialize with zero + dset["data"] = initializeNDArray( + extent, + m_fileFormat == FileFormat::Json ? std::optional{} + : parameter.dtype); break; } - default: + case IOMode::Template: + if (parameter.extent != Extent{0}) + { + dset["extent"] = parameter.extent; + } + else + { + // no-op + // If extent is empty, don't bother writing it + } break; } - // TOML does not support nulls, so initialize with zero - dset["data"] = initializeNDArray( - extent, - m_fileFormat == FileFormat::Json ? std::optional() - : parameter.dtype); writable->written = true; m_dirty.emplace(file); } @@ -358,9 +490,11 @@ void JSONIOHandlerImpl::extendDataset( refreshFileFromParent(writable); auto &j = obtainJsonContents(writable); + IOMode localIOMode; try { - auto datasetExtent = getExtent(j); + Extent datasetExtent; + std::tie(datasetExtent, localIOMode) = getExtent(j); VERIFY_ALWAYS( datasetExtent.size() == parameters.extent.size(), "[JSON] Cannot change dimensionality of a dataset") @@ -377,28 +511,40 @@ void JSONIOHandlerImpl::extendDataset( throw std::runtime_error( "[JSON] The specified location contains no valid dataset"); } - auto extent = parameters.extent; - auto datatype = stringToDatatype(j["datatype"].get()); - switch (datatype) + + switch (localIOMode) { - case Datatype::CFLOAT: - case Datatype::CDOUBLE: - case Datatype::CLONG_DOUBLE: { - extent.push_back(2); - break; + case IOMode::Dataset: { + auto extent = parameters.extent; + auto datatype = stringToDatatype(j["datatype"].get()); + switch (datatype) + { + case Datatype::CFLOAT: + case Datatype::CDOUBLE: + case Datatype::CLONG_DOUBLE: { + extent.push_back(2); + break; + } + default: + // nothing to do + break; + } + // TOML does not support nulls, so initialize with zero + nlohmann::json newData = initializeNDArray( + extent, + m_fileFormat == FileFormat::Json ? std::optional{} + : datatype); + nlohmann::json &oldData = j["data"]; + mergeInto(newData, oldData); + j["data"] = newData; } - default: - // nothing to do - break; + break; + case IOMode::Template: { + j["extent"] = parameters.extent; + } + break; } - // TOML does not support nulls, so initialize with zero - nlohmann::json newData = initializeNDArray( - extent, - m_fileFormat == FileFormat::Json ? std::optional() - : datatype); - nlohmann::json &oldData = j["data"]; - mergeInto(newData, oldData); - j["data"] = newData; + writable->written = true; } @@ -694,7 +840,7 @@ void JSONIOHandlerImpl::openDataset( *parameters.dtype = Datatype(stringToDatatype(datasetJson["datatype"].get())); - *parameters.extent = getExtent(datasetJson); + *parameters.extent = getExtent(datasetJson).first; writable->written = true; } @@ -877,7 +1023,16 @@ void JSONIOHandlerImpl::writeDataset( auto file = refreshFileFromParent(writable); auto &j = obtainJsonContents(writable); - verifyDataset(parameters, j); + switch (verifyDataset(parameters, j)) + { + case IOMode::Dataset: + break; + case IOMode::Template: + std::cerr << "[JSON/TOML backend: Warning] Trying to write data to a " + "template dataset. Will skip." + << std::endl; + return; + } switchType(parameters.dtype, j, parameters); @@ -919,22 +1074,55 @@ void JSONIOHandlerImpl::writeAttribute( m_dirty.emplace(file); } +namespace +{ + struct FillWithZeroes + { + template + static void call(void *ptr, Extent const &extent) + { + T *casted = static_cast(ptr); + size_t flattenedExtent = std::accumulate( + extent.begin(), + extent.end(), + size_t(1), + [](size_t left, size_t right) { return left * right; }); + std::fill_n(casted, flattenedExtent, T{}); + } + + static constexpr char const *errorMsg = + "[JSON Backend] Fill with zeroes."; + }; +} // namespace + void JSONIOHandlerImpl::readDataset( Writable *writable, Parameter ¶meters) { refreshFileFromParent(writable); setAndGetFilePosition(writable); auto &j = obtainJsonContents(writable); - verifyDataset(parameters, j); + IOMode localMode = verifyDataset(parameters, j); - try + switch (localMode) { - switchType(parameters.dtype, j["data"], parameters); - } - catch (json::basic_json::type_error &) - { - throw std::runtime_error( - "[JSON] The given path does not contain a valid dataset."); + case IOMode::Template: + std::cerr << "[Warning] Cannot read chunks in Template mode of JSON " + "backend. Will fill with zeroes instead." + << std::endl; + switchNonVectorType( + parameters.dtype, parameters.data.get(), parameters.extent); + return; + case IOMode::Dataset: + try + { + switchType(parameters.dtype, j["data"], parameters); + } + catch (json::basic_json::type_error &) + { + throw std::runtime_error( + "[JSON] The given path does not contain a valid dataset."); + } + break; } } @@ -1182,28 +1370,44 @@ Extent JSONIOHandlerImpl::getMultiplicators(Extent const &extent) return res; } -Extent JSONIOHandlerImpl::getExtent(nlohmann::json &j) +auto JSONIOHandlerImpl::getExtent(nlohmann::json &j) + -> std::pair { Extent res; - nlohmann::json *ptr = &j["data"]; - while (ptr->is_array()) + IOMode ioMode; + if (j.contains("data")) + { + ioMode = IOMode::Dataset; + nlohmann::json *ptr = &j["data"]; + while (ptr->is_array()) + { + res.push_back(ptr->size()); + ptr = &(*ptr)[0]; + } + switch (stringToDatatype(j["datatype"].get())) + { + case Datatype::CFLOAT: + case Datatype::CDOUBLE: + case Datatype::CLONG_DOUBLE: + // the last "dimension" is only the two entries for the complex + // number, so remove that again + res.erase(res.end() - 1); + break; + default: + break; + } + } + else if (j.contains("extent")) { - res.push_back(ptr->size()); - ptr = &(*ptr)[0]; + ioMode = IOMode::Template; + res = j["extent"].get(); } - switch (stringToDatatype(j["datatype"].get())) + else { - case Datatype::CFLOAT: - case Datatype::CDOUBLE: - case Datatype::CLONG_DOUBLE: - // the last "dimension" is only the two entries for the complex - // number, so remove that again - res.erase(res.end() - 1); - break; - default: - break; + ioMode = IOMode::Template; + res = {0}; } - return res; + return std::make_pair(std::move(res), ioMode); } std::string JSONIOHandlerImpl::removeSlashes(std::string s) @@ -1365,7 +1569,14 @@ auto JSONIOHandlerImpl::putJsonContents( return it; } - (*it->second)["platform_byte_widths"] = platformSpecifics(); + switch (m_mode) + { + case IOMode::Dataset: + (*it->second)["platform_byte_widths"] = platformSpecifics(); + break; + case IOMode::Template: + break; + } auto writeSingleFile = [this, &it](std::string const &writeThisFile) { auto [fh, _, fh_with_precision] = @@ -1568,8 +1779,8 @@ bool JSONIOHandlerImpl::isDataset(nlohmann::json const &j) { return false; } - auto i = j.find("data"); - return i != j.end() && i.value().is_array(); + auto i = j.find("datatype"); + return i != j.end() && i.value().is_string(); } bool JSONIOHandlerImpl::isGroup(nlohmann::json::const_iterator const &it) @@ -1580,21 +1791,24 @@ bool JSONIOHandlerImpl::isGroup(nlohmann::json::const_iterator const &it) { return false; } - auto i = j.find("data"); - return i == j.end() || !i.value().is_array(); + + auto i = j.find("datatype"); + return i == j.end() || !i.value().is_string(); } template -void JSONIOHandlerImpl::verifyDataset( - Param const ¶meters, nlohmann::json &j) +auto JSONIOHandlerImpl::verifyDataset( + Param const ¶meters, nlohmann::json &j) -> IOMode { VERIFY_ALWAYS( isDataset(j), "[JSON] Specified dataset does not exist or is not a dataset."); + IOMode res; try { - auto datasetExtent = getExtent(j); + Extent datasetExtent; + std::tie(datasetExtent, res) = getExtent(j); VERIFY_ALWAYS( datasetExtent.size() == parameters.extent.size(), "[JSON] Read/Write request does not fit the dataset's dimension"); @@ -1616,6 +1830,7 @@ void JSONIOHandlerImpl::verifyDataset( throw std::runtime_error( "[JSON] The given path does not contain a valid dataset."); } + return res; } nlohmann::json JSONIOHandlerImpl::platformSpecifics() @@ -1697,7 +1912,7 @@ nlohmann::json JSONIOHandlerImpl::CppToJSON::operator()(const T &val) } template -nlohmann::json JSONIOHandlerImpl::CppToJSON >::operator()( +nlohmann::json JSONIOHandlerImpl::CppToJSON>::operator()( const std::vector &v) { nlohmann::json j; @@ -1710,7 +1925,7 @@ nlohmann::json JSONIOHandlerImpl::CppToJSON >::operator()( } template -nlohmann::json JSONIOHandlerImpl::CppToJSON >::operator()( +nlohmann::json JSONIOHandlerImpl::CppToJSON>::operator()( const std::array &v) { nlohmann::json j; @@ -1729,7 +1944,7 @@ T JSONIOHandlerImpl::JsonToCpp::operator()(nlohmann::json const &json) } template -std::vector JSONIOHandlerImpl::JsonToCpp >::operator()( +std::vector JSONIOHandlerImpl::JsonToCpp>::operator()( nlohmann::json const &json) { std::vector v; @@ -1742,7 +1957,7 @@ std::vector JSONIOHandlerImpl::JsonToCpp >::operator()( } template -std::array JSONIOHandlerImpl::JsonToCpp >::operator()( +std::array JSONIOHandlerImpl::JsonToCpp>::operator()( nlohmann::json const &json) { std::array a; diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index fe5eb92e3b..19d026285f 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -2,6 +2,7 @@ #include "openPMD/ChunkInfo_internal.hpp" #include "openPMD/Datatype.hpp" #include "openPMD/IO/Access.hpp" +#include "openPMD/auxiliary/JSON.hpp" #if openPMD_USE_INVASIVE_TESTS #define OPENPMD_private public: #define OPENPMD_protected public: @@ -1271,12 +1272,19 @@ TEST_CASE("particle_patches", "[serial]") } } -inline void dtype_test(const std::string &backend) +inline void dtype_test( + const std::string &backend, + std::optional activateTemplateMode = {}) { bool test_long_double = backend != "json" && backend != "toml"; bool test_long_long = (backend != "json") || sizeof(long long) <= 8; { - Series s = Series("../samples/dtype_test." + backend, Access::CREATE); + Series s = activateTemplateMode.has_value() + ? Series( + "../samples/dtype_test." + backend, + Access::CREATE, + activateTemplateMode.value()) + : Series("../samples/dtype_test." + backend, Access::CREATE); char c = 'c'; s.setAttribute("char", c); @@ -1398,8 +1406,12 @@ inline void dtype_test(const std::string &backend) } } - Series s = Series("../samples/dtype_test." + backend, Access::READ_ONLY); - + Series s = activateTemplateMode.has_value() + ? Series( + "../samples/dtype_test." + backend, + Access::READ_ONLY, + activateTemplateMode.value()) + : Series("../samples/dtype_test." + backend, Access::READ_ONLY); REQUIRE(s.getAttribute("char").get() == 'c'); REQUIRE(s.getAttribute("uchar").get() == 'u'); REQUIRE(s.getAttribute("schar").get() == 's'); @@ -1469,6 +1481,10 @@ inline void dtype_test(const std::string &backend) REQUIRE(s.getAttribute("bool").get() == true); REQUIRE(s.getAttribute("boolF").get() == false); + if (activateTemplateMode.has_value()) + { + return; + } // same implementation types (not necessary aliases) detection #if !defined(_MSC_VER) REQUIRE(s.getAttribute("short").dtype == Datatype::SHORT); @@ -1541,6 +1557,7 @@ TEST_CASE("dtype_test", "[serial]") { dtype_test(t); } + dtype_test("json", R"({"json":{"dataset":{"mode":"template"}}})"); if (auto extensions = getFileExtensions(); std::find(extensions.begin(), extensions.end(), "toml") != extensions.end()) @@ -1549,6 +1566,7 @@ TEST_CASE("dtype_test", "[serial]") * testing it here. */ dtype_test("toml"); + dtype_test("toml", R"({"toml":{"dataset":{"mode":"template"}}})"); } } @@ -1572,6 +1590,8 @@ inline void write_test(const std::string &backend) host_info::byMethod( host_info::methodFromStringDescription("posix_hostname", false))}}; #endif + jsonCfg = + json::merge(jsonCfg, R"({"json":{"dataset":{"mode":"template"}}})"); Series o = Series("../samples/serial_write." + backend, Access::CREATE, jsonCfg); @@ -1599,8 +1619,10 @@ inline void write_test(const std::string &backend) return posOff++; }); std::shared_ptr positionOffset_local_1(new uint64_t); - e_1["positionOffset"]["x"].resetDataset( - Dataset(determineDatatype(positionOffset_local_1), {4})); + e_1["positionOffset"]["x"].resetDataset(Dataset( + determineDatatype(positionOffset_local_1), + {4}, + R"({"json":{"dataset":{"mode":"dataset"}}})")); for (uint64_t i = 0; i < 4; ++i) { From bdfce2de9db8cd35f52a29e7f7df5fb9dc8cfccc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 4 Aug 2023 13:33:24 +0200 Subject: [PATCH 02/23] Write used mode to JSON file --- include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp | 10 ++- src/IO/JSON/JSONIOHandlerImpl.cpp | 63 +++++++++++++++++-- 2 files changed, 67 insertions(+), 6 deletions(-) diff --git a/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp index 875961ea7a..541758a4b1 100644 --- a/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp +++ b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp @@ -284,7 +284,15 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl IOMode m_mode = IOMode::Dataset; - IOMode retrieveDatasetMode(openPMD::json::TracingJSON &config) const; + enum class SpecificationVia + { + DefaultValue, + Manually + }; + SpecificationVia m_IOModeSpecificationVia = SpecificationVia::DefaultValue; + + std::pair + retrieveDatasetMode(openPMD::json::TracingJSON &config) const; // HELPER FUNCTIONS diff --git a/src/IO/JSON/JSONIOHandlerImpl.cpp b/src/IO/JSON/JSONIOHandlerImpl.cpp index cacbbac18a..649b34c587 100644 --- a/src/IO/JSON/JSONIOHandlerImpl.cpp +++ b/src/IO/JSON/JSONIOHandlerImpl.cpp @@ -63,6 +63,13 @@ namespace openPMD throw std::runtime_error((TEXT)); \ } +namespace JSONDefaults +{ + using const_str = char const *const; + constexpr const_str openpmd_internal = "__openPMD_internal"; + constexpr const_str IOMode = "IO_mode"; +} // namespace JSONDefaults + namespace { struct DefaultValue @@ -150,10 +157,11 @@ namespace } } // namespace -auto JSONIOHandlerImpl::retrieveDatasetMode( - openPMD::json::TracingJSON &config) const -> IOMode +auto JSONIOHandlerImpl::retrieveDatasetMode(openPMD::json::TracingJSON &config) + const -> std::pair { IOMode res = m_mode; + SpecificationVia res_2 = SpecificationVia::DefaultValue; if (auto [configLocation, maybeConfig] = getBackendConfig(config); maybeConfig.has_value()) { @@ -176,10 +184,12 @@ auto JSONIOHandlerImpl::retrieveDatasetMode( if (mode == "dataset") { res = IOMode::Dataset; + res_2 = SpecificationVia::Manually; } else if (mode == "template") { res = IOMode::Template; + res_2 = SpecificationVia::Manually; } else { @@ -191,7 +201,7 @@ auto JSONIOHandlerImpl::retrieveDatasetMode( } } } - return res; + return std::make_pair(res, res_2); } std::string JSONIOHandlerImpl::backendConfigKey() const @@ -230,7 +240,7 @@ JSONIOHandlerImpl::JSONIOHandlerImpl( , m_fileFormat{format} , m_originalExtension{std::move(originalExtension)} { - m_mode = retrieveDatasetMode(config); + std::tie(m_mode, m_IOModeSpecificationVia) = retrieveDatasetMode(config); if (auto [_, backendConfig] = getBackendConfig(config); backendConfig.has_value()) @@ -392,7 +402,7 @@ void JSONIOHandlerImpl::createDataset( parameter.options, /* considerFiles = */ false); // Retrieves mode from dataset-specific configuration, falls back to global // value if not defined - IOMode localMode = retrieveDatasetMode(config); + IOMode localMode = retrieveDatasetMode(config).first; parameter.warnUnusedParameters( config, @@ -1544,6 +1554,45 @@ JSONIOHandlerImpl::obtainJsonContents(File const &file) auto res = serialImplementation(); #endif + if (res->contains(JSONDefaults::openpmd_internal)) + { + auto const &openpmd_internal = res->at(JSONDefaults::openpmd_internal); + if (openpmd_internal.contains(JSONDefaults::IOMode)) + { + auto modeOption = openPMD::json::asLowerCaseStringDynamic( + openpmd_internal.at(JSONDefaults::IOMode)); + if (!modeOption.has_value()) + { + std::cerr + << "[JSON/TOML backend] Warning: Invalid value of " + "non-string type at internal meta table for entry '" + << JSONDefaults::IOMode << "'. Will ignore and continue." + << std::endl; + } + else if (modeOption.value() == "dataset") + { + if (m_IOModeSpecificationVia == SpecificationVia::DefaultValue) + { + m_mode = IOMode::Dataset; + } + } + else if (modeOption.value() == "template") + { + if (m_IOModeSpecificationVia == SpecificationVia::DefaultValue) + { + m_mode = IOMode::Template; + } + } + else + { + std::cerr << "[JSON/TOML backend] Warning: Invalid value '" + << modeOption.value() + << "' at internal meta table for entry '" + << JSONDefaults::IOMode + << "'. Will ignore and continue." << std::endl; + } + } + } m_jsonVals.emplace(file, res); return res; } @@ -1573,8 +1622,12 @@ auto JSONIOHandlerImpl::putJsonContents( { case IOMode::Dataset: (*it->second)["platform_byte_widths"] = platformSpecifics(); + (*it->second)[JSONDefaults::openpmd_internal][JSONDefaults::IOMode] = + "dataset"; break; case IOMode::Template: + (*it->second)[JSONDefaults::openpmd_internal][JSONDefaults::IOMode] = + "template"; break; } From 23a3ce00c041eccf5fe212bd0c7a88788f223c10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 23 Feb 2023 11:05:39 +0100 Subject: [PATCH 03/23] Use Attribute::getOptional for snapshot attribute --- src/Series.cpp | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/src/Series.cpp b/src/Series.cpp index a3749c5e73..9d83d35eb8 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -2976,19 +2976,13 @@ auto Series::currentSnapshot() const if (series.iterations.containsAttribute("snapshot")) { auto const &attribute = series.iterations.getAttribute("snapshot"); - switch (attribute.dtype) + auto res = attribute.getOptional(); + if (res.has_value()) { - case Datatype::ULONGLONG: - case Datatype::VEC_ULONGLONG: { - auto const &vec = attribute.get>(); - return vec_t{vec.begin(), vec.end()}; + return res.value(); } - case Datatype::ULONG: - case Datatype::VEC_ULONG: { - auto const &vec = attribute.get>(); - return vec_t{vec.begin(), vec.end()}; - } - default: { + else + { std::stringstream s; s << "Unexpected datatype for '/data/snapshot': " << attribute.dtype << " (expected a vector of integer, found " + @@ -3000,7 +2994,6 @@ auto Series::currentSnapshot() const {}, s.str()); } - } } else { From d09e654e77d24607911f6d123d73653d7b78d826 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 4 Aug 2023 14:49:16 +0200 Subject: [PATCH 04/23] Introduce attribute mode --- include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp | 33 +- src/IO/JSON/JSONIOHandlerImpl.cpp | 330 +++++++++++++++++- 2 files changed, 342 insertions(+), 21 deletions(-) diff --git a/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp index 541758a4b1..0a5a9779fd 100644 --- a/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp +++ b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp @@ -276,6 +276,16 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl std::string m_originalExtension; + enum class SpecificationVia + { + DefaultValue, + Manually + }; + + ///////////////////// + // Dataset IO mode // + ///////////////////// + enum class IOMode { Dataset, @@ -283,17 +293,28 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl }; IOMode m_mode = IOMode::Dataset; - - enum class SpecificationVia - { - DefaultValue, - Manually - }; SpecificationVia m_IOModeSpecificationVia = SpecificationVia::DefaultValue; std::pair retrieveDatasetMode(openPMD::json::TracingJSON &config) const; + /////////////////////// + // Attribute IO mode // + /////////////////////// + + enum class AttributeMode + { + Short, + Long + }; + + AttributeMode m_attributeMode = AttributeMode::Long; + SpecificationVia m_attributeModeSpecificationVia = + SpecificationVia::DefaultValue; + + std::pair + retrieveAttributeMode(openPMD::json::TracingJSON &config) const; + // HELPER FUNCTIONS // will use the IOHandler to retrieve the correct directory. diff --git a/src/IO/JSON/JSONIOHandlerImpl.cpp b/src/IO/JSON/JSONIOHandlerImpl.cpp index 649b34c587..93be87136b 100644 --- a/src/IO/JSON/JSONIOHandlerImpl.cpp +++ b/src/IO/JSON/JSONIOHandlerImpl.cpp @@ -30,6 +30,7 @@ #include "openPMD/auxiliary/Memory.hpp" #include "openPMD/auxiliary/StringManip.hpp" #include "openPMD/auxiliary/TypeTraits.hpp" +#include "openPMD/backend/Attribute.hpp" #include "openPMD/backend/Writable.hpp" #include @@ -67,7 +68,8 @@ namespace JSONDefaults { using const_str = char const *const; constexpr const_str openpmd_internal = "__openPMD_internal"; - constexpr const_str IOMode = "IO_mode"; + constexpr const_str IOMode = "dataset_mode"; + constexpr const_str AttributeMode = "attribute_mode"; } // namespace JSONDefaults namespace @@ -204,6 +206,54 @@ auto JSONIOHandlerImpl::retrieveDatasetMode(openPMD::json::TracingJSON &config) return std::make_pair(res, res_2); } +auto JSONIOHandlerImpl::retrieveAttributeMode( + openPMD::json::TracingJSON &config) const + -> std::pair +{ + AttributeMode res = m_attributeMode; + SpecificationVia res_2 = SpecificationVia::DefaultValue; + if (auto [configLocation, maybeConfig] = getBackendConfig(config); + maybeConfig.has_value()) + { + auto jsonConfig = maybeConfig.value(); + if (jsonConfig.json().contains("attribute")) + { + auto attributeConfig = jsonConfig["attribute"]; + if (attributeConfig.json().contains("mode")) + { + auto modeOption = openPMD::json::asLowerCaseStringDynamic( + attributeConfig["mode"].json()); + if (!modeOption.has_value()) + { + throw error::BackendConfigSchema( + {configLocation, "mode"}, + "Invalid value of non-string type (accepted values are " + "'dataset' and 'template'."); + } + auto mode = modeOption.value(); + if (mode == "short") + { + res = AttributeMode::Short; + res_2 = SpecificationVia::Manually; + } + else if (mode == "long") + { + res = AttributeMode::Long; + res_2 = SpecificationVia::Manually; + } + else + { + throw error::BackendConfigSchema( + {configLocation, "attribute", "mode"}, + "Invalid value: '" + mode + + "' (accepted values are 'short' and 'long'."); + } + } + } + } + return std::make_pair(res, res_2); +} + std::string JSONIOHandlerImpl::backendConfigKey() const { switch (m_fileFormat) @@ -241,6 +291,8 @@ JSONIOHandlerImpl::JSONIOHandlerImpl( , m_originalExtension{std::move(originalExtension)} { std::tie(m_mode, m_IOModeSpecificationVia) = retrieveDatasetMode(config); + std::tie(m_attributeMode, m_attributeModeSpecificationVia) = + retrieveAttributeMode(config); if (auto [_, backendConfig] = getBackendConfig(config); backendConfig.has_value()) @@ -1078,8 +1130,17 @@ void JSONIOHandlerImpl::writeAttribute( } nlohmann::json value; switchType(parameter.dtype, value, parameter.resource); - (*jsonVal)[filePosition->id]["attributes"][parameter.name] = { - {"datatype", datatypeToString(parameter.dtype)}, {"value", value}}; + switch (m_attributeMode) + { + case AttributeMode::Long: + (*jsonVal)[filePosition->id]["attributes"][parameter.name] = { + {"datatype", datatypeToString(parameter.dtype)}, {"value", value}}; + break; + case AttributeMode::Short: + // short form + (*jsonVal)[filePosition->id]["attributes"][parameter.name] = value; + break; + } writable->written = true; m_dirty.emplace(file); } @@ -1136,6 +1197,195 @@ void JSONIOHandlerImpl::readDataset( } } +namespace +{ + template + Attribute recoverVectorAttributeFromJson(nlohmann::json const &j) + { + if (!j.is_array()) + { + throw std::runtime_error( + "[JSON backend: recoverVectorAttributeFromJson] Internal " + "control flow error."); + } + + if (j.size() == 7 && + (std::is_same_v || + std::is_same_v || + std::is_same_v)) + { + /* + * The frontend must deal with wrong type reports here. + */ + std::array res; + for (size_t i = 0; i < 7; ++i) + { + res[i] = j[i].get(); + } + return res; + } + else + { + std::vector res; + res.reserve(j.size()); + for (auto const &i : j) + { + res.push_back(i.get()); + } + return res; + } + } + + nlohmann::json::value_t unifyNumericType(nlohmann::json const &j) + { + if (!j.is_array() || j.empty()) + { + throw std::runtime_error( + "[JSON backend: recoverVectorAttributeFromJson] Internal " + "control flow error."); + } + auto dtypeRanking = [](nlohmann::json::value_t dtype) -> unsigned { + switch (dtype) + { + case nlohmann::json::value_t::number_unsigned: + return 0; + case nlohmann::json::value_t::number_integer: + return 1; + case nlohmann::json::value_t::number_float: + return 2; + default: + throw std::runtime_error( + "[JSON backend] Encountered vector with mixed number and " + "non-number datatypes."); + } + }; + auto higherDtype = + [&dtypeRanking]( + nlohmann::json::value_t dt1, + nlohmann::json::value_t dt2) -> nlohmann::json::value_t { + if (dtypeRanking(dt1) > dtypeRanking(dt2)) + { + return dt1; + } + else + { + return dt2; + } + }; + + nlohmann::json::value_t res = j[0].type(); + for (size_t i = 1; i < j.size(); ++i) + { + res = higherDtype(res, j[i].type()); + } + return res; + } + + Attribute recoverAttributeFromJson( + nlohmann::json const &j, std::string const &nameForErrorMessages) + { + // @todo use ReadError once it's mainlined + switch (j.type()) + { + case nlohmann::json::value_t::null: + throw std::runtime_error( + "[JSON backend] Attribute must not be null: '" + + nameForErrorMessages + "'."); + case nlohmann::json::value_t::object: + throw std::runtime_error( + "[JSON backend] Shorthand-style attribute must not be an " + "object: '" + + nameForErrorMessages + "'."); + case nlohmann::json::value_t::array: + if (j.empty()) + { + std::cerr << "Cannot recover datatype of empty vector without " + "explicit type annotation for attribute '" + << nameForErrorMessages + << "'. Will continue with VEC_INT datatype." + << std::endl; + return std::vector{}; + } + else + { + auto valueType = j[0].type(); + /* + * If the vector is of numeric type, it might happen that the + * first entry is an integer, but a later entry is a float. + * We need to pick the most generic datatype in that case. + */ + if (valueType == nlohmann::json::value_t::number_float || + valueType == nlohmann::json::value_t::number_unsigned || + valueType == nlohmann::json::value_t::number_integer) + { + valueType = unifyNumericType(j); + } + switch (valueType) + { + case nlohmann::json::value_t::null: + throw std::runtime_error( + "[JSON backend] Attribute must not be null: '" + + nameForErrorMessages + "'."); + case nlohmann::json::value_t::object: + throw std::runtime_error( + "[JSON backend] Invalid contained datatype (object) " + "inside vector-type attribute: '" + + nameForErrorMessages + "'."); + case nlohmann::json::value_t::array: + throw std::runtime_error( + "[JSON backend] Invalid contained datatype (array) " + "inside vector-type attribute: '" + + nameForErrorMessages + "'."); + case nlohmann::json::value_t::string: + return recoverVectorAttributeFromJson(j); + case nlohmann::json::value_t::boolean: + throw std::runtime_error( + "[JSON backend] Attribute must not be vector of bool: " + "'" + + nameForErrorMessages + "'."); + case nlohmann::json::value_t::number_integer: + return recoverVectorAttributeFromJson< + nlohmann::json::number_integer_t>(j); + case nlohmann::json::value_t::number_unsigned: + return recoverVectorAttributeFromJson< + nlohmann::json::number_unsigned_t>(j); + case nlohmann::json::value_t::number_float: + return recoverVectorAttributeFromJson< + nlohmann::json::number_float_t>(j); + case nlohmann::json::value_t::binary: + throw std::runtime_error( + "[JSON backend] Attribute must not have binary type: " + "'" + + nameForErrorMessages + "'."); + case nlohmann::json::value_t::discarded: + throw std::runtime_error( + "Internal JSON parser datatype leaked into JSON " + "value."); + } + throw std::runtime_error("Unreachable!"); + } + case nlohmann::json::value_t::string: + return j.get(); + case nlohmann::json::value_t::boolean: + return j.get(); + case nlohmann::json::value_t::number_integer: + return j.get(); + case nlohmann::json::value_t::number_unsigned: + return j.get(); + case nlohmann::json::value_t::number_float: + return j.get(); + case nlohmann::json::value_t::binary: + throw std::runtime_error( + "[JSON backend] Attribute must not have binary type: '" + + nameForErrorMessages + "'."); + case nlohmann::json::value_t::discarded: + throw std::runtime_error( + "Internal JSON parser datatype leaked into JSON value."); + } + throw std::runtime_error("Unreachable!"); + } +} // namespace + void JSONIOHandlerImpl::readAttribute( Writable *writable, Parameter ¶meters) { @@ -1160,9 +1410,19 @@ void JSONIOHandlerImpl::readAttribute( auto &j = jsonLoc[name]; try { - *parameters.dtype = - Datatype(stringToDatatype(j["datatype"].get())); - switchType(*parameters.dtype, j["value"], parameters); + if (j.is_object()) + { + *parameters.dtype = + Datatype(stringToDatatype(j["datatype"].get())); + switchType( + *parameters.dtype, j["value"], parameters); + } + else + { + Attribute attr = recoverAttributeFromJson(j, name); + *parameters.dtype = attr.dtype; + *parameters.resource = attr.getResource(); + } } catch (json::type_error &) { @@ -1557,7 +1817,10 @@ JSONIOHandlerImpl::obtainJsonContents(File const &file) if (res->contains(JSONDefaults::openpmd_internal)) { auto const &openpmd_internal = res->at(JSONDefaults::openpmd_internal); - if (openpmd_internal.contains(JSONDefaults::IOMode)) + + // Init dataset mode according to file's default + if (m_IOModeSpecificationVia == SpecificationVia::DefaultValue && + openpmd_internal.contains(JSONDefaults::IOMode)) { auto modeOption = openPMD::json::asLowerCaseStringDynamic( openpmd_internal.at(JSONDefaults::IOMode)); @@ -1571,17 +1834,42 @@ JSONIOHandlerImpl::obtainJsonContents(File const &file) } else if (modeOption.value() == "dataset") { - if (m_IOModeSpecificationVia == SpecificationVia::DefaultValue) - { - m_mode = IOMode::Dataset; - } + m_mode = IOMode::Dataset; } else if (modeOption.value() == "template") { - if (m_IOModeSpecificationVia == SpecificationVia::DefaultValue) - { - m_mode = IOMode::Template; - } + m_mode = IOMode::Template; + } + else + { + std::cerr << "[JSON/TOML backend] Warning: Invalid value '" + << modeOption.value() + << "' at internal meta table for entry '" + << JSONDefaults::IOMode + << "'. Will ignore and continue." << std::endl; + } + } + + if (m_IOModeSpecificationVia == SpecificationVia::DefaultValue && + openpmd_internal.contains(JSONDefaults::AttributeMode)) + { + auto modeOption = openPMD::json::asLowerCaseStringDynamic( + openpmd_internal.at(JSONDefaults::AttributeMode)); + if (!modeOption.has_value()) + { + std::cerr + << "[JSON/TOML backend] Warning: Invalid value of " + "non-string type at internal meta table for entry '" + << JSONDefaults::AttributeMode + << "'. Will ignore and continue." << std::endl; + } + else if (modeOption.value() == "long") + { + m_attributeMode = AttributeMode::Long; + } + else if (modeOption.value() == "short") + { + m_attributeMode = AttributeMode::Short; } else { @@ -1631,6 +1919,18 @@ auto JSONIOHandlerImpl::putJsonContents( break; } + switch (m_attributeMode) + { + case AttributeMode::Short: + (*it->second)[JSONDefaults::openpmd_internal] + [JSONDefaults::AttributeMode] = "short"; + break; + case AttributeMode::Long: + (*it->second)[JSONDefaults::openpmd_internal] + [JSONDefaults::AttributeMode] = "long"; + break; + } + auto writeSingleFile = [this, &it](std::string const &writeThisFile) { auto [fh, _, fh_with_precision] = getFilehandle(File(writeThisFile), Access::CREATE); From b4299b2be2fecbbfcfb52fcdd9d0c4542101ebfb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 4 Aug 2023 14:50:34 +0200 Subject: [PATCH 05/23] Add example 14_toml_template.cpp --- examples/14_toml_template.cpp | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 examples/14_toml_template.cpp diff --git a/examples/14_toml_template.cpp b/examples/14_toml_template.cpp new file mode 100644 index 0000000000..33b9115129 --- /dev/null +++ b/examples/14_toml_template.cpp @@ -0,0 +1,22 @@ +#include + +int main() +{ + std::string config = R"( +{ + "iteration_encoding": "variable_based", + "toml": { + "dataset": {"mode": "template"}, + "attribute": {"mode": "short"} + } +} +)"; + + openPMD::Series writeTemplate( + "../samples/tomlTemplate.toml", openPMD::Access::CREATE, config); + auto iteration = writeTemplate.writeIterations()[0]; + + auto temperature = + iteration.meshes["temperature"][openPMD::RecordComponent::SCALAR]; + temperature.resetDataset({openPMD::Datatype::FLOAT, {5, 5}}); +} From 5a0355dc0a12aa3fc4f8057e60f40c69fbab0a4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 10 Mar 2023 15:42:59 +0100 Subject: [PATCH 06/23] Use Datatype::UNDEFINED to indicate no dataset definition in template --- include/openPMD/Dataset.hpp | 2 +- include/openPMD/RecordComponent.hpp | 2 +- .../openPMD/backend/PatchRecordComponent.hpp | 2 -- src/RecordComponent.cpp | 19 ++++++++++++------- src/backend/PatchRecordComponent.cpp | 17 ----------------- 5 files changed, 14 insertions(+), 28 deletions(-) diff --git a/include/openPMD/Dataset.hpp b/include/openPMD/Dataset.hpp index 0032888541..a610ce6a67 100644 --- a/include/openPMD/Dataset.hpp +++ b/include/openPMD/Dataset.hpp @@ -44,7 +44,7 @@ class Dataset JOINED_DIMENSION = std::numeric_limits::max() }; - Dataset(Datatype, Extent, std::string options = "{}"); + Dataset(Datatype, Extent = {1}, std::string options = "{}"); /** * @brief Constructor that sets the datatype to undefined. diff --git a/include/openPMD/RecordComponent.hpp b/include/openPMD/RecordComponent.hpp index 9072b93a32..ee29a6d7fa 100644 --- a/include/openPMD/RecordComponent.hpp +++ b/include/openPMD/RecordComponent.hpp @@ -173,7 +173,7 @@ class RecordComponent : public BaseRecordComponent * * @return RecordComponent& */ - virtual RecordComponent &resetDataset(Dataset); + RecordComponent &resetDataset(Dataset); uint8_t getDimensionality() const; Extent getExtent() const; diff --git a/include/openPMD/backend/PatchRecordComponent.hpp b/include/openPMD/backend/PatchRecordComponent.hpp index 63875b11e2..5c0cf6bfe7 100644 --- a/include/openPMD/backend/PatchRecordComponent.hpp +++ b/include/openPMD/backend/PatchRecordComponent.hpp @@ -66,8 +66,6 @@ class PatchRecordComponent : public RecordComponent PatchRecordComponent &setUnitSI(double); - PatchRecordComponent &resetDataset(Dataset) override; - uint8_t getDimensionality() const; Extent getExtent() const; diff --git a/src/RecordComponent.cpp b/src/RecordComponent.cpp index 0387268514..fc17909fc6 100644 --- a/src/RecordComponent.cpp +++ b/src/RecordComponent.cpp @@ -104,15 +104,20 @@ RecordComponent &RecordComponent::resetDataset(Dataset d) rc.m_hasBeenExtended = true; } - if (d.dtype == Datatype::UNDEFINED) + if (d.extent.empty()) + throw std::runtime_error("Dataset extent must be at least 1D."); + if (d.empty()) { - throw error::WrongAPIUsage( - "[RecordComponent] Must set specific datatype."); + if (d.dtype != Datatype::UNDEFINED) + { + return makeEmpty(std::move(d)); + } + else + { + rc.m_dataset = std::move(d); + return *this; + } } - // if( d.extent.empty() ) - // throw std::runtime_error("Dataset extent must be at least 1D."); - if (d.empty()) - return makeEmpty(std::move(d)); rc.m_isEmpty = false; if (written()) diff --git a/src/backend/PatchRecordComponent.cpp b/src/backend/PatchRecordComponent.cpp index af19923fad..2ac202e44a 100644 --- a/src/backend/PatchRecordComponent.cpp +++ b/src/backend/PatchRecordComponent.cpp @@ -34,23 +34,6 @@ PatchRecordComponent &PatchRecordComponent::setUnitSI(double usi) return *this; } -PatchRecordComponent &PatchRecordComponent::resetDataset(Dataset d) -{ - if (written()) - throw std::runtime_error( - "A Records Dataset can not (yet) be changed after it has been " - "written."); - if (d.extent.empty()) - throw std::runtime_error("Dataset extent must be at least 1D."); - if (d.empty()) - throw std::runtime_error( - "Dataset extent must not be zero in any dimension."); - - get().m_dataset = std::move(d); - setDirty(true); - return *this; -} - uint8_t PatchRecordComponent::getDimensionality() const { return 1; From e0c3b5306dc180f6df51c0fad332f047a8c04c0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 19 May 2022 17:29:24 +0200 Subject: [PATCH 07/23] Extend example --- examples/14_toml_template.cpp | 91 +++++++++++++++++++++++++++++++++-- 1 file changed, 88 insertions(+), 3 deletions(-) diff --git a/examples/14_toml_template.cpp b/examples/14_toml_template.cpp index 33b9115129..284db9ee84 100644 --- a/examples/14_toml_template.cpp +++ b/examples/14_toml_template.cpp @@ -1,6 +1,21 @@ #include -int main() +std::string backendEnding() +{ + auto extensions = openPMD::getFileExtensions(); + if (auto it = std::find(extensions.begin(), extensions.end(), "toml"); + it != extensions.end()) + { + return *it; + } + else + { + // Fallback for buggy old NVidia compiler + return "json"; + } +} + +void write() { std::string config = R"( { @@ -13,10 +28,80 @@ int main() )"; openPMD::Series writeTemplate( - "../samples/tomlTemplate.toml", openPMD::Access::CREATE, config); + "../samples/tomlTemplate." + backendEnding(), + openPMD::Access::CREATE, + config); auto iteration = writeTemplate.writeIterations()[0]; + openPMD::Dataset ds{openPMD::Datatype::FLOAT, {5, 5}}; + auto temperature = iteration.meshes["temperature"][openPMD::RecordComponent::SCALAR]; - temperature.resetDataset({openPMD::Datatype::FLOAT, {5, 5}}); + temperature.resetDataset(ds); + + auto E = iteration.meshes["E"]; + E["x"].resetDataset(ds); + E["y"].resetDataset(ds); + /* + * Don't specify datatype and extent for this one to indicate that this + * information is not yet known. + */ + E["z"].resetDataset({openPMD::Datatype::UNDEFINED}); + + ds.extent = {10}; + + auto electrons = iteration.particles["e"]; + electrons["position"]["x"].resetDataset(ds); + electrons["position"]["y"].resetDataset(ds); + electrons["position"]["z"].resetDataset(ds); + + electrons["positionOffset"]["x"].resetDataset(ds); + electrons["positionOffset"]["y"].resetDataset(ds); + electrons["positionOffset"]["z"].resetDataset(ds); + electrons["positionOffset"]["x"].makeConstant(3.14); + electrons["positionOffset"]["y"].makeConstant(3.14); + electrons["positionOffset"]["z"].makeConstant(3.14); + + ds.dtype = openPMD::determineDatatype(); + electrons.particlePatches["numParticles"][openPMD::RecordComponent::SCALAR] + .resetDataset(ds); + electrons + .particlePatches["numParticlesOffset"][openPMD::RecordComponent::SCALAR] + .resetDataset(ds); + electrons.particlePatches["offset"]["x"].resetDataset(ds); + electrons.particlePatches["offset"]["y"].resetDataset(ds); + electrons.particlePatches["offset"]["z"].resetDataset(ds); + electrons.particlePatches["extent"]["x"].resetDataset(ds); + electrons.particlePatches["extent"]["y"].resetDataset(ds); + electrons.particlePatches["extent"]["z"].resetDataset(ds); +} + +void read() +{ + /* + * The config is entirely optional, these things are also detected + * automatically when reading + */ + + // std::string config = R"( + // { + // "iteration_encoding": "variable_based", + // "toml": { + // "dataset": {"mode": "template"}, + // "attribute": {"mode": "short"} + // } + // } + // )"; + + openPMD::Series read( + "../samples/tomlTemplate." + backendEnding(), + openPMD::Access::READ_LINEAR); + read.readIterations(); // @todo change to read.parseBase() + openPMD::helper::listSeries(read); +} + +int main() +{ + write(); + read(); } From 8d3e88ccd4e272c88e650184566393e1b8b12ab0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 7 Aug 2023 11:04:07 +0200 Subject: [PATCH 08/23] Test short attribute mode --- test/SerialIOTest.cpp | 37 +++++++++++++++++++++++++++++++++---- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index 19d026285f..134417b4a6 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -1557,7 +1557,17 @@ TEST_CASE("dtype_test", "[serial]") { dtype_test(t); } - dtype_test("json", R"({"json":{"dataset":{"mode":"template"}}})"); + dtype_test("json", R"( +{ + "json": { + "dataset": { + "mode": "template" + }, + "attribute": { + "mode": "short" + } + } +})"); if (auto extensions = getFileExtensions(); std::find(extensions.begin(), extensions.end(), "toml") != extensions.end()) @@ -1566,7 +1576,17 @@ TEST_CASE("dtype_test", "[serial]") * testing it here. */ dtype_test("toml"); - dtype_test("toml", R"({"toml":{"dataset":{"mode":"template"}}})"); + dtype_test("toml", R"( +{ + "toml": { + "dataset": { + "mode": "template" + }, + "attribute": { + "mode": "short" + } + } +})"); } } @@ -1590,8 +1610,17 @@ inline void write_test(const std::string &backend) host_info::byMethod( host_info::methodFromStringDescription("posix_hostname", false))}}; #endif - jsonCfg = - json::merge(jsonCfg, R"({"json":{"dataset":{"mode":"template"}}})"); + jsonCfg = json::merge(jsonCfg, R"( +{ + "json": { + "dataset": { + "mode": "template" + }, + "attribute": { + "mode": "short" + } + } +})"); Series o = Series("../samples/serial_write." + backend, Access::CREATE, jsonCfg); From 8905ea882b2c76c75030213b4cd21d2bb0042236 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 7 Aug 2023 11:26:07 +0200 Subject: [PATCH 09/23] Copy datatypeToString to JSON implementation --- src/IO/JSON/JSONIOHandlerImpl.cpp | 96 ++++++++++++++++++++++++++++++- 1 file changed, 93 insertions(+), 3 deletions(-) diff --git a/src/IO/JSON/JSONIOHandlerImpl.cpp b/src/IO/JSON/JSONIOHandlerImpl.cpp index 93be87136b..1c2cd20771 100644 --- a/src/IO/JSON/JSONIOHandlerImpl.cpp +++ b/src/IO/JSON/JSONIOHandlerImpl.cpp @@ -157,6 +157,95 @@ namespace } } } + + // Does the same as datatypeToString(), but this makes sure that we don't + // accidentally change the JSON schema by modifying datatypeToString() + std::string jsonDatatypeToString(Datatype dt) + { + switch (dt) + { + using DT = Datatype; + case DT::CHAR: + return "CHAR"; + case DT::UCHAR: + return "UCHAR"; + case DT::SCHAR: + return "SCHAR"; + case DT::SHORT: + return "SHORT"; + case DT::INT: + return "INT"; + case DT::LONG: + return "LONG"; + case DT::LONGLONG: + return "LONGLONG"; + case DT::USHORT: + return "USHORT"; + case DT::UINT: + return "UINT"; + case DT::ULONG: + return "ULONG"; + case DT::ULONGLONG: + return "ULONGLONG"; + case DT::FLOAT: + return "FLOAT"; + case DT::DOUBLE: + return "DOUBLE"; + case DT::LONG_DOUBLE: + return "LONG_DOUBLE"; + case DT::CFLOAT: + return "CFLOAT"; + case DT::CDOUBLE: + return "CDOUBLE"; + case DT::CLONG_DOUBLE: + return "CLONG_DOUBLE"; + case DT::STRING: + return "STRING"; + case DT::VEC_CHAR: + return "VEC_CHAR"; + case DT::VEC_SHORT: + return "VEC_SHORT"; + case DT::VEC_INT: + return "VEC_INT"; + case DT::VEC_LONG: + return "VEC_LONG"; + case DT::VEC_LONGLONG: + return "VEC_LONGLONG"; + case DT::VEC_UCHAR: + return "VEC_UCHAR"; + case DT::VEC_USHORT: + return "VEC_USHORT"; + case DT::VEC_UINT: + return "VEC_UINT"; + case DT::VEC_ULONG: + return "VEC_ULONG"; + case DT::VEC_ULONGLONG: + return "VEC_ULONGLONG"; + case DT::VEC_FLOAT: + return "VEC_FLOAT"; + case DT::VEC_DOUBLE: + return "VEC_DOUBLE"; + case DT::VEC_LONG_DOUBLE: + return "VEC_LONG_DOUBLE"; + case DT::VEC_CFLOAT: + return "VEC_CFLOAT"; + case DT::VEC_CDOUBLE: + return "VEC_CDOUBLE"; + case DT::VEC_CLONG_DOUBLE: + return "VEC_CLONG_DOUBLE"; + case DT::VEC_SCHAR: + return "VEC_SCHAR"; + case DT::VEC_STRING: + return "VEC_STRING"; + case DT::ARR_DBL_7: + return "ARR_DBL_7"; + case DT::BOOL: + return "BOOL"; + case DT::UNDEFINED: + return "UNDEFINED"; + } + return "Unreachable!"; + } } // namespace auto JSONIOHandlerImpl::retrieveDatasetMode(openPMD::json::TracingJSON &config) @@ -478,7 +567,7 @@ void JSONIOHandlerImpl::createDataset( } setAndGetFilePosition(writable, name); auto &dset = jsonVal[name]; - dset["datatype"] = datatypeToString(parameter.dtype); + dset["datatype"] = jsonDatatypeToString(parameter.dtype); switch (localMode) { @@ -1134,7 +1223,8 @@ void JSONIOHandlerImpl::writeAttribute( { case AttributeMode::Long: (*jsonVal)[filePosition->id]["attributes"][parameter.name] = { - {"datatype", datatypeToString(parameter.dtype)}, {"value", value}}; + {"datatype", jsonDatatypeToString(parameter.dtype)}, + {"value", value}}; break; case AttributeMode::Short: // short form @@ -2209,7 +2299,7 @@ nlohmann::json JSONIOHandlerImpl::platformSpecifics() Datatype::BOOL}; for (auto it = std::begin(datatypes); it != std::end(datatypes); it++) { - res[datatypeToString(*it)] = toBytes(*it); + res[jsonDatatypeToString(*it)] = toBytes(*it); } return res; } From 2e3d790ebcea3bbeab6d01fa6c347c309c069876 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 22 Sep 2023 17:31:34 +0200 Subject: [PATCH 10/23] Fix after rebase: Init JSON config in parallel mode --- src/IO/JSON/JSONIOHandlerImpl.cpp | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/IO/JSON/JSONIOHandlerImpl.cpp b/src/IO/JSON/JSONIOHandlerImpl.cpp index 1c2cd20771..918d3b9d80 100644 --- a/src/IO/JSON/JSONIOHandlerImpl.cpp +++ b/src/IO/JSON/JSONIOHandlerImpl.cpp @@ -394,16 +394,25 @@ JSONIOHandlerImpl::JSONIOHandlerImpl( #if openPMD_HAVE_MPI JSONIOHandlerImpl::JSONIOHandlerImpl( AbstractIOHandler *handler, - MPI_Comm comm, - // NOLINTNEXTLINE(performance-unnecessary-value-param) - [[maybe_unused]] openPMD::json::TracingJSON config, + MPI_Comm comm,openPMD::json::TracingJSON config, FileFormat format, std::string originalExtension) : AbstractIOHandlerImpl(handler) , m_communicator{comm} , m_fileFormat{format} , m_originalExtension{std::move(originalExtension)} -{} +{ + std::tie(m_mode, m_IOModeSpecificationVia) = retrieveDatasetMode(config); + std::tie(m_attributeMode, m_attributeModeSpecificationVia) = + retrieveAttributeMode(config); + + if (auto [_, backendConfig] = getBackendConfig(config); + backendConfig.has_value()) + { + (void)_; + warnUnusedJson(backendConfig.value()); + } +} #endif JSONIOHandlerImpl::~JSONIOHandlerImpl() = default; From 1fe955d5e9b68dacf1a9fd873367b98c46d268c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 22 Sep 2023 17:32:07 +0200 Subject: [PATCH 11/23] Fix after rebase: Don't erase JSON datasets when writing --- src/IO/JSON/JSONIOHandlerImpl.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/IO/JSON/JSONIOHandlerImpl.cpp b/src/IO/JSON/JSONIOHandlerImpl.cpp index 918d3b9d80..17a35b7770 100644 --- a/src/IO/JSON/JSONIOHandlerImpl.cpp +++ b/src/IO/JSON/JSONIOHandlerImpl.cpp @@ -2145,6 +2145,7 @@ merge the .json files somehow (no tooling provided for this (yet)). #else serialImplementation(); #endif + if (unsetDirty) { m_dirty.erase(filename); From 55b85dfa369026155e2162fca60371c924d8be30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 12 Oct 2023 09:48:23 +0200 Subject: [PATCH 12/23] openpmd-pipe: use short modes for test --- CMakeLists.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7b221400dc..d494e48207 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1304,6 +1304,9 @@ if(openPMD_BUILD_TESTING) ${openPMD_RUNTIME_OUTPUT_DIRECTORY}/openpmd-pipe \ --infile ../samples/git-sample/thetaMode/data_%T.bp \ --outfile ../samples/git-sample/thetaMode/data%T.json \ + --outconfig ' \ + json.attribute.mode = \"short\" \n\ + json.dataset.mode = \"template\"' \ " WORKING_DIRECTORY ${openPMD_RUNTIME_OUTPUT_DIRECTORY} ) From fb2c661b893dc24f08c8f853c6ea61105b3d3e2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 12 Oct 2023 10:50:28 +0200 Subject: [PATCH 13/23] Less intrusive warnings, allow disabling them --- CMakeLists.txt | 2 +- include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp | 17 +++++- src/IO/JSON/JSONIOHandlerImpl.cpp | 52 +++++++++++++------ 3 files changed, 53 insertions(+), 18 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d494e48207..07d89bd694 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1306,7 +1306,7 @@ if(openPMD_BUILD_TESTING) --outfile ../samples/git-sample/thetaMode/data%T.json \ --outconfig ' \ json.attribute.mode = \"short\" \n\ - json.dataset.mode = \"template\"' \ + json.dataset.mode = \"template_no_warn\"' \ " WORKING_DIRECTORY ${openPMD_RUNTIME_OUTPUT_DIRECTORY} ) diff --git a/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp index 0a5a9779fd..68d0635fd1 100644 --- a/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp +++ b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp @@ -294,9 +294,22 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl IOMode m_mode = IOMode::Dataset; SpecificationVia m_IOModeSpecificationVia = SpecificationVia::DefaultValue; + bool m_printedSkippedWriteWarningAlready = false; - std::pair - retrieveDatasetMode(openPMD::json::TracingJSON &config) const; + struct DatasetMode + { + IOMode m_IOMode; + SpecificationVia m_specificationVia; + bool m_skipWarnings; + + template + operator std::tuple() + { + return std::tuple{ + m_IOMode, m_specificationVia, m_skipWarnings}; + } + }; + DatasetMode retrieveDatasetMode(openPMD::json::TracingJSON &config) const; /////////////////////// // Attribute IO mode // diff --git a/src/IO/JSON/JSONIOHandlerImpl.cpp b/src/IO/JSON/JSONIOHandlerImpl.cpp index 17a35b7770..9e7eb1651f 100644 --- a/src/IO/JSON/JSONIOHandlerImpl.cpp +++ b/src/IO/JSON/JSONIOHandlerImpl.cpp @@ -248,11 +248,12 @@ namespace } } // namespace -auto JSONIOHandlerImpl::retrieveDatasetMode(openPMD::json::TracingJSON &config) - const -> std::pair +auto JSONIOHandlerImpl::retrieveDatasetMode( + openPMD::json::TracingJSON &config) const -> DatasetMode { - IOMode res = m_mode; - SpecificationVia res_2 = SpecificationVia::DefaultValue; + IOMode ioMode = m_mode; + SpecificationVia specificationVia = SpecificationVia::DefaultValue; + bool skipWarnings = false; if (auto [configLocation, maybeConfig] = getBackendConfig(config); maybeConfig.has_value()) { @@ -274,13 +275,19 @@ auto JSONIOHandlerImpl::retrieveDatasetMode(openPMD::json::TracingJSON &config) auto mode = modeOption.value(); if (mode == "dataset") { - res = IOMode::Dataset; - res_2 = SpecificationVia::Manually; + ioMode = IOMode::Dataset; + specificationVia = SpecificationVia::Manually; } else if (mode == "template") { - res = IOMode::Template; - res_2 = SpecificationVia::Manually; + ioMode = IOMode::Template; + specificationVia = SpecificationVia::Manually; + } + else if (mode == "template_no_warn") + { + ioMode = IOMode::Template; + specificationVia = SpecificationVia::Manually; + skipWarnings = true; } else { @@ -292,7 +299,7 @@ auto JSONIOHandlerImpl::retrieveDatasetMode(openPMD::json::TracingJSON &config) } } } - return std::make_pair(res, res_2); + return DatasetMode{ioMode, specificationVia, skipWarnings}; } auto JSONIOHandlerImpl::retrieveAttributeMode( @@ -379,7 +386,9 @@ JSONIOHandlerImpl::JSONIOHandlerImpl( , m_fileFormat{format} , m_originalExtension{std::move(originalExtension)} { - std::tie(m_mode, m_IOModeSpecificationVia) = retrieveDatasetMode(config); + std::tie( + m_mode, m_IOModeSpecificationVia, m_printedSkippedWriteWarningAlready) = + retrieveDatasetMode(config); std::tie(m_attributeMode, m_attributeModeSpecificationVia) = retrieveAttributeMode(config); @@ -402,7 +411,9 @@ JSONIOHandlerImpl::JSONIOHandlerImpl( , m_fileFormat{format} , m_originalExtension{std::move(originalExtension)} { - std::tie(m_mode, m_IOModeSpecificationVia) = retrieveDatasetMode(config); + std::tie( + m_mode, m_IOModeSpecificationVia, m_printedSkippedWriteWarningAlready) = + retrieveDatasetMode(config); std::tie(m_attributeMode, m_attributeModeSpecificationVia) = retrieveAttributeMode(config); @@ -552,7 +563,13 @@ void JSONIOHandlerImpl::createDataset( parameter.options, /* considerFiles = */ false); // Retrieves mode from dataset-specific configuration, falls back to global // value if not defined - IOMode localMode = retrieveDatasetMode(config).first; + auto [localMode, _, skipWarnings] = retrieveDatasetMode(config); + (void)_; + // No use in introducing logic to skip warnings only for one particular + // dataset. If warnings are skipped, then they are skipped consistently. + // Use |= since `false` is the default value and we don't wish to reset + // the flag. + m_printedSkippedWriteWarningAlready |= skipWarnings; parameter.warnUnusedParameters( config, @@ -1188,9 +1205,14 @@ void JSONIOHandlerImpl::writeDataset( case IOMode::Dataset: break; case IOMode::Template: - std::cerr << "[JSON/TOML backend: Warning] Trying to write data to a " - "template dataset. Will skip." - << std::endl; + if (!m_printedSkippedWriteWarningAlready) + { + std::cerr + << "[JSON/TOML backend: Warning] Trying to write data to a " + "template dataset. Will skip." + << std::endl; + m_printedSkippedWriteWarningAlready = true; + } return; } From 61d37171eed49aea51db297bbb68fb274686cf33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 12 Oct 2023 11:29:09 +0200 Subject: [PATCH 14/23] TOML: Use short modes by default --- include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp | 2 ++ src/IO/JSON/JSONIOHandlerImpl.cpp | 34 ++++++++++++------- test/SerialIOTest.cpp | 25 ++++++++++---- test/python/unittest/API/APITest.py | 3 +- 4 files changed, 44 insertions(+), 20 deletions(-) diff --git a/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp index 68d0635fd1..fc369047ec 100644 --- a/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp +++ b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp @@ -180,6 +180,8 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl std::string originalExtension); #endif + void init(openPMD::json::TracingJSON config); + ~JSONIOHandlerImpl() override; void diff --git a/src/IO/JSON/JSONIOHandlerImpl.cpp b/src/IO/JSON/JSONIOHandlerImpl.cpp index 9e7eb1651f..5af5bdd920 100644 --- a/src/IO/JSON/JSONIOHandlerImpl.cpp +++ b/src/IO/JSON/JSONIOHandlerImpl.cpp @@ -386,18 +386,7 @@ JSONIOHandlerImpl::JSONIOHandlerImpl( , m_fileFormat{format} , m_originalExtension{std::move(originalExtension)} { - std::tie( - m_mode, m_IOModeSpecificationVia, m_printedSkippedWriteWarningAlready) = - retrieveDatasetMode(config); - std::tie(m_attributeMode, m_attributeModeSpecificationVia) = - retrieveAttributeMode(config); - - if (auto [_, backendConfig] = getBackendConfig(config); - backendConfig.has_value()) - { - (void)_; - warnUnusedJson(backendConfig.value()); - } + init(std::move(config)); } #if openPMD_HAVE_MPI @@ -411,6 +400,26 @@ JSONIOHandlerImpl::JSONIOHandlerImpl( , m_fileFormat{format} , m_originalExtension{std::move(originalExtension)} { + init(std::move(config)); +} +#endif + +void JSONIOHandlerImpl::init(openPMD::json::TracingJSON config) +{ + // set the defaults + switch (m_fileFormat) + { + case FileFormat::Json: + // @todo take the switch to openPMD 2.0 as a chance to switch to + // short attribute mode as a default here + m_attributeMode = AttributeMode::Long; + m_mode = IOMode::Dataset; + break; + case FileFormat::Toml: + m_attributeMode = AttributeMode::Short; + m_mode = IOMode::Template; + break; + } std::tie( m_mode, m_IOModeSpecificationVia, m_printedSkippedWriteWarningAlready) = retrieveDatasetMode(config); @@ -424,7 +433,6 @@ JSONIOHandlerImpl::JSONIOHandlerImpl( warnUnusedJson(backendConfig.value()); } } -#endif JSONIOHandlerImpl::~JSONIOHandlerImpl() = default; diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index 134417b4a6..6b7405be90 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -1284,8 +1284,12 @@ inline void dtype_test( "../samples/dtype_test." + backend, Access::CREATE, activateTemplateMode.value()) - : Series("../samples/dtype_test." + backend, Access::CREATE); - + : + // test TOML long attribute mode by default + Series( + "../samples/dtype_test." + backend, + Access::CREATE, + R"({"toml":{"attribute":{"mode":"long"}}})"); char c = 'c'; s.setAttribute("char", c); unsigned char uc = 'u'; @@ -1901,7 +1905,7 @@ inline void fileBased_write_test(const std::string &backend) Series o = Series( "../samples/subdir/serial_fileBased_write%03T." + backend, Access::CREATE, - jsonCfg); + json::merge(jsonCfg, R"({"toml":{"dataset":{"mode":"dataset"}}})")); ParticleSpecies &e_1 = o.iterations[1].particles["e"]; @@ -7519,7 +7523,10 @@ void groupbased_read_write(std::string const &ext) std::string filename = "../samples/groupbased_read_write." + ext; { - Series write(filename, Access::CREATE); + Series write( + filename, + Access::CREATE, + R"({"toml":{"dataset":{"mode":"dataset"}}})"); auto E_x = write.iterations[0].meshes["E"]["x"]; auto E_y = write.iterations[0].meshes["E"]["y"]; E_x.resetDataset(ds); @@ -7532,7 +7539,10 @@ void groupbased_read_write(std::string const &ext) } { - Series write(filename, Access::READ_WRITE); + Series write( + filename, + Access::READ_WRITE, + R"({"toml":{"dataset":{"mode":"dataset"}}})"); // create a new iteration auto E_x = write.iterations[1].meshes["E"]["x"]; E_x.resetDataset(ds); @@ -7572,7 +7582,10 @@ void groupbased_read_write(std::string const &ext) // check that truncation works correctly { - Series write(filename, Access::CREATE); + Series write( + filename, + Access::CREATE, + R"({"toml":{"dataset":{"mode":"dataset"}}})"); // create a new iteration auto E_x = write.iterations[2].meshes["E"]["x"]; E_x.resetDataset(ds); diff --git a/test/python/unittest/API/APITest.py b/test/python/unittest/API/APITest.py index 59e6b5c97e..1c79d63a0c 100644 --- a/test/python/unittest/API/APITest.py +++ b/test/python/unittest/API/APITest.py @@ -25,7 +25,8 @@ from TestUtilities.TestUtilities import generateTestFilePath tested_file_extensions = [ - ext for ext in io.file_extensions if ext != 'sst' and ext != 'ssc' + ext for ext in io.file_extensions + if ext != 'sst' and ext != 'ssc' and ext != 'toml' ] From 5f9788bfbaef7d748144c6657d1b303ba72f2271 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 26 Oct 2023 15:41:46 +0200 Subject: [PATCH 15/23] Python formatting --- test/python/unittest/API/APITest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/python/unittest/API/APITest.py b/test/python/unittest/API/APITest.py index 1c79d63a0c..1abe26340a 100644 --- a/test/python/unittest/API/APITest.py +++ b/test/python/unittest/API/APITest.py @@ -26,7 +26,7 @@ tested_file_extensions = [ ext for ext in io.file_extensions - if ext != 'sst' and ext != 'ssc' and ext != 'toml' + if ext != 'sst' and ext != 'ssc' and ext != 'toml' ] From 9d9d1cee3f67fbf909aee624adcb69af2df75506 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 24 Nov 2023 12:48:52 +0000 Subject: [PATCH 16/23] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/IO/JSON/JSONIOHandlerImpl.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/IO/JSON/JSONIOHandlerImpl.cpp b/src/IO/JSON/JSONIOHandlerImpl.cpp index 5af5bdd920..782a8b7a25 100644 --- a/src/IO/JSON/JSONIOHandlerImpl.cpp +++ b/src/IO/JSON/JSONIOHandlerImpl.cpp @@ -392,7 +392,8 @@ JSONIOHandlerImpl::JSONIOHandlerImpl( #if openPMD_HAVE_MPI JSONIOHandlerImpl::JSONIOHandlerImpl( AbstractIOHandler *handler, - MPI_Comm comm,openPMD::json::TracingJSON config, + MPI_Comm comm, + openPMD::json::TracingJSON config, FileFormat format, std::string originalExtension) : AbstractIOHandlerImpl(handler) From 16825d4ecb9731793fd61bec9dd178cda4524238 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 24 Nov 2023 16:16:31 +0100 Subject: [PATCH 17/23] Documentation --- docs/source/backends/json.rst | 36 +++++++++++++++++++++++---- docs/source/details/backendconfig.rst | 23 ++++++++++++++--- docs/source/details/json.json | 10 ++++++++ 3 files changed, 60 insertions(+), 9 deletions(-) create mode 100644 docs/source/details/json.json diff --git a/docs/source/backends/json.rst b/docs/source/backends/json.rst index bbae92aaf6..0f0d9510c9 100644 --- a/docs/source/backends/json.rst +++ b/docs/source/backends/json.rst @@ -38,20 +38,46 @@ when working with the JSON backend. Datasets and groups have the same namespace, meaning that there may not be a subgroup and a dataset with the same name contained in one group. -Any **openPMD dataset** is a JSON object with three keys: +Datasets +........ - * ``attributes``: Attributes associated with the dataset. May be ``null`` or not present if no attributes are associated with the dataset. - * ``datatype``: A string describing the type of the stored data. - * ``data`` A nested array storing the actual data in row-major manner. +Datasets can be stored in two modes, either as actual datasets or as dataset templates. +The mode is selected by the :ref:`JSON/TOML parameter` ``json.dataset.mode`` (resp. ``toml.dataset.mode``) with possible values ``["dataset", "template"]`` (default: ``"dataset"``). + +Stored as an actual dataset, an **openPMD dataset** is a JSON object with three JSON keys: + + * ``datatype`` (required): A string describing the type of the stored data. + * ``data`` (required): A nested array storing the actual data in row-major manner. The data needs to be consistent with the fields ``datatype`` and ``extent``. Checking whether this key points to an array can be (and is internally) used to distinguish groups from datasets. + * ``attributes``: Attributes associated with the dataset. May be ``null`` or not present if no attributes are associated with the dataset. + +Stored as a **dataset template**, an openPMD dataset is represented by three JSON keys: + +* ``datatype`` (required): As above. +* ``extent`` (required): A list of integers, describing the extent of the dataset. +* ``attributes``: As above. -**Attributes** are stored as a JSON object with a key for each attribute. +This mode stores only the dataset metadata. +Chunk load/store operations are ignored. + +Attributes +.......... + +In order to avoid name clashes, attributes are generally stored within a separate subgroup ``attributes``. + +Attributes can be stored in two formats. +The format is selected by the :ref:`JSON/TOML parameter` ``json.attribute.mode`` (resp. ``toml.attribute.mode``) with possible values ``["long", "short"]`` (default: ``"long"`` in openPMD 1.*, ``"short"`` in openPMD >= 2.0). + +Attributes in **long format** store the datatype explicitly, by representing attributes as JSON objects. Every such attribute is itself a JSON object with two keys: * ``datatype``: A string describing the type of the value. * ``value``: The actual value of type ``datatype``. +Attributes in **short format** are stored as just the simple value corresponding with the attribute. +Since JSON/TOML values are pretty-printed into a human-readable format, byte-level type details can be lost when reading those values again later on (e.g. the distinction between different integer types). + TOML File Format ---------------- diff --git a/docs/source/details/backendconfig.rst b/docs/source/details/backendconfig.rst index f6d15a7ac8..cf78d9cdea 100644 --- a/docs/source/details/backendconfig.rst +++ b/docs/source/details/backendconfig.rst @@ -104,6 +104,8 @@ The key ``rank_table`` allows specifying the creation of a **rank table**, used Configuration Structure per Backend ----------------------------------- +Please refer to the respective backends' documentations for further information on their configuration. + .. _backendconfig-adios2: ADIOS2 @@ -231,8 +233,21 @@ The parameters eligible for being passed to flush calls may be configured global .. _backendconfig-other: -Other backends -^^^^^^^^^^^^^^ +JSON/TOML +^^^^^^^^^ -Do currently not read the configuration string. -Please refer to the respective backends' documentations for further information on their configuration. +A full configuration of the JSON backend: + +.. literalinclude:: json.json + :language: json + +The TOML backend is configured analogously, replacing the ``"json"`` key with ``"toml"``. + +All keys found under ``hdf5.dataset`` are applicable globally as well as per dataset. +Explanation of the single keys: + +* ``json.dataset.mode`` / ``toml.dataset.mode``: One of ``"dataset"`` (default) or ``"template"``. + In "dataset" mode, the dataset will be written as an n-dimensional (recursive) array, padded with nulls (JSON) or zeroes (TOML) for missing values. + In "template" mode, only the dataset metadata (type, extent and attributes) are stored and no chunks can be written or read. +* ``json.attribute.mode`` / ``toml.attribute.mode``: One of ``"long"`` (default in openPMD 1.*) or ``"short"`` (default in openPMD 2.*). + The long format explicitly encodes the attribute type in the dataset on disk, the short format only writes the actual attribute as a JSON/TOML value, requiring readers to recover the type. diff --git a/docs/source/details/json.json b/docs/source/details/json.json new file mode 100644 index 0000000000..c1491f7245 --- /dev/null +++ b/docs/source/details/json.json @@ -0,0 +1,10 @@ +{ + "json": { + "dataset": { + "mode": "template" + }, + "attribute": { + "mode": "short" + } + } +} From 28526463243b1017f994dff77a8748115add6ac5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 24 Nov 2023 17:47:57 +0100 Subject: [PATCH 18/23] Short mode in default in openPMD >= 2. --- include/openPMD/IO/IOTask.hpp | 1 + include/openPMD/RecordComponent.tpp | 51 ++++++++++++++++++++++++++--- src/IO/JSON/JSONIOHandlerImpl.cpp | 7 ++++ src/Iteration.cpp | 1 + src/Series.cpp | 1 + 5 files changed, 57 insertions(+), 4 deletions(-) diff --git a/include/openPMD/IO/IOTask.hpp b/include/openPMD/IO/IOTask.hpp index 731372f9e1..2758a26aca 100644 --- a/include/openPMD/IO/IOTask.hpp +++ b/include/openPMD/IO/IOTask.hpp @@ -142,6 +142,7 @@ struct OPENPMDAPI_EXPORT Parameter } std::string name = ""; + std::string openPMDversion; // @todo: Maybe move this to AbstractIOHandler }; template <> diff --git a/include/openPMD/RecordComponent.tpp b/include/openPMD/RecordComponent.tpp index 542503e806..42c29994a2 100644 --- a/include/openPMD/RecordComponent.tpp +++ b/include/openPMD/RecordComponent.tpp @@ -21,6 +21,8 @@ #pragma once +#include "openPMD/Datatype.hpp" +#include "openPMD/Error.hpp" #include "openPMD/RecordComponent.hpp" #include "openPMD/Span.hpp" #include "openPMD/auxiliary/Memory.hpp" @@ -93,12 +95,38 @@ inline std::shared_ptr RecordComponent::loadChunk(Offset o, Extent e) #endif } +namespace detail +{ + template + struct do_convert + { + template + static std::optional call(Attribute &attr) + { + if constexpr (std::is_convertible_v) + { + return std::make_optional(attr.get()); + } + else + { + return std::nullopt; + } + } + + static constexpr char const *errorMsg = "is_conversible"; + }; +} // namespace detail + template inline void RecordComponent::loadChunk(std::shared_ptr data, Offset o, Extent e) { Datatype dtype = determineDatatype(data); - if (dtype != getDatatype()) + /* + * For constant components, we implement type conversion, so there is + * a separate check further below. + */ + if (dtype != getDatatype() && !constant()) if (!isSameInteger(getDatatype()) && !isSameFloatingPoint(getDatatype()) && !isSameComplexFloatingPoint(getDatatype()) && @@ -160,10 +188,25 @@ RecordComponent::loadChunk(std::shared_ptr data, Offset o, Extent e) for (auto const &dimensionSize : extent) numPoints *= dimensionSize; - T value = rc.m_constantValue.get(); + std::optional val = + switchNonVectorType>( + /* from = */ getDatatype(), rc.m_constantValue); - T *raw_ptr = data.get(); - std::fill(raw_ptr, raw_ptr + numPoints, value); + if (val.has_value()) + { + T *raw_ptr = data.get(); + std::fill(raw_ptr, raw_ptr + numPoints, *val); + } + else + { + std::string const data_type_str = datatypeToString(getDatatype()); + std::string const requ_type_str = + datatypeToString(determineDatatype()); + std::string err_msg = + "Type conversion during chunk loading not possible! "; + err_msg += "Data: " + data_type_str + "; Load as: " + requ_type_str; + throw error::WrongAPIUsage(err_msg); + } } else { diff --git a/src/IO/JSON/JSONIOHandlerImpl.cpp b/src/IO/JSON/JSONIOHandlerImpl.cpp index 782a8b7a25..f01c3d29aa 100644 --- a/src/IO/JSON/JSONIOHandlerImpl.cpp +++ b/src/IO/JSON/JSONIOHandlerImpl.cpp @@ -455,6 +455,13 @@ void JSONIOHandlerImpl::createFile( access::write(m_handler->m_backendAccess), "[JSON] Creating a file in read-only mode is not possible."); + if (m_attributeModeSpecificationVia == SpecificationVia::DefaultValue) + { + m_attributeMode = parameters.openPMDversion >= "2." + ? AttributeMode::Short + : AttributeMode::Long; + } + if (!writable->written) { std::string name = parameters.name + m_originalExtension; diff --git a/src/Iteration.cpp b/src/Iteration.cpp index 366fea0de1..c7388594f8 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -208,6 +208,7 @@ void Iteration::flushFileBased( /* create file */ Parameter fCreate; fCreate.name = filename; + fCreate.openPMDversion = s.openPMD(); IOHandler()->enqueue(IOTask(&s.writable(), fCreate)); /* diff --git a/src/Series.cpp b/src/Series.cpp index 9d83d35eb8..ccd90b31a8 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -1478,6 +1478,7 @@ void Series::flushGorVBased( } Parameter fCreate; fCreate.name = series.m_name; + fCreate.openPMDversion = openPMD(); IOHandler()->enqueue(IOTask(this, fCreate)); flushRankTable(); From 49bd7ba90789b4a0090f414423ad3b1a558f9db2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 19 Mar 2024 11:38:59 +0100 Subject: [PATCH 19/23] Short value by default in TOML --- docs/source/backends/json.rst | 2 +- src/IO/JSON/JSONIOHandlerImpl.cpp | 15 ++++++++++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/docs/source/backends/json.rst b/docs/source/backends/json.rst index 0f0d9510c9..bba6ca5df0 100644 --- a/docs/source/backends/json.rst +++ b/docs/source/backends/json.rst @@ -67,7 +67,7 @@ Attributes In order to avoid name clashes, attributes are generally stored within a separate subgroup ``attributes``. Attributes can be stored in two formats. -The format is selected by the :ref:`JSON/TOML parameter` ``json.attribute.mode`` (resp. ``toml.attribute.mode``) with possible values ``["long", "short"]`` (default: ``"long"`` in openPMD 1.*, ``"short"`` in openPMD >= 2.0). +The format is selected by the :ref:`JSON/TOML parameter` ``json.attribute.mode`` (resp. ``toml.attribute.mode``) with possible values ``["long", "short"]`` (default: ``"long"`` for JSON in openPMD 1.*, ``"short"`` otherwise, i.e. generally in openPMD 2.*, but always in TOML). Attributes in **long format** store the datatype explicitly, by representing attributes as JSON objects. Every such attribute is itself a JSON object with two keys: diff --git a/src/IO/JSON/JSONIOHandlerImpl.cpp b/src/IO/JSON/JSONIOHandlerImpl.cpp index f01c3d29aa..1700b1fd99 100644 --- a/src/IO/JSON/JSONIOHandlerImpl.cpp +++ b/src/IO/JSON/JSONIOHandlerImpl.cpp @@ -457,9 +457,18 @@ void JSONIOHandlerImpl::createFile( if (m_attributeModeSpecificationVia == SpecificationVia::DefaultValue) { - m_attributeMode = parameters.openPMDversion >= "2." - ? AttributeMode::Short - : AttributeMode::Long; + switch (m_fileFormat) + { + + case FileFormat::Json: + m_attributeMode = parameters.openPMDversion >= "2." + ? AttributeMode::Short + : AttributeMode::Long; + break; + case FileFormat::Toml: + m_attributeMode = AttributeMode::Short; + break; + } } if (!writable->written) From 59a88dab6498be3de00fedbe979488348b799683 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 19 Mar 2024 13:21:16 +0100 Subject: [PATCH 20/23] Store the openPMD version information in the IOHandler --- include/openPMD/IO/AbstractIOHandler.hpp | 2 ++ include/openPMD/IO/IOTask.hpp | 1 - src/IO/JSON/JSONIOHandlerImpl.cpp | 2 +- src/Iteration.cpp | 1 - src/Series.cpp | 2 +- 5 files changed, 4 insertions(+), 4 deletions(-) diff --git a/include/openPMD/IO/AbstractIOHandler.hpp b/include/openPMD/IO/AbstractIOHandler.hpp index 1288a87b21..a9f05ce871 100644 --- a/include/openPMD/IO/AbstractIOHandler.hpp +++ b/include/openPMD/IO/AbstractIOHandler.hpp @@ -186,9 +186,11 @@ class AbstractIOHandler { friend class Series; friend class ADIOS2IOHandlerImpl; + friend class JSONIOHandlerImpl; friend class detail::ADIOS2File; private: + std::string m_openPMDVersion; IterationEncoding m_encoding = IterationEncoding::groupBased; void setIterationEncoding(IterationEncoding encoding) diff --git a/include/openPMD/IO/IOTask.hpp b/include/openPMD/IO/IOTask.hpp index 2758a26aca..731372f9e1 100644 --- a/include/openPMD/IO/IOTask.hpp +++ b/include/openPMD/IO/IOTask.hpp @@ -142,7 +142,6 @@ struct OPENPMDAPI_EXPORT Parameter } std::string name = ""; - std::string openPMDversion; // @todo: Maybe move this to AbstractIOHandler }; template <> diff --git a/src/IO/JSON/JSONIOHandlerImpl.cpp b/src/IO/JSON/JSONIOHandlerImpl.cpp index 1700b1fd99..ffed613498 100644 --- a/src/IO/JSON/JSONIOHandlerImpl.cpp +++ b/src/IO/JSON/JSONIOHandlerImpl.cpp @@ -461,7 +461,7 @@ void JSONIOHandlerImpl::createFile( { case FileFormat::Json: - m_attributeMode = parameters.openPMDversion >= "2." + m_attributeMode = m_handler->m_openPMDVersion >= "2." ? AttributeMode::Short : AttributeMode::Long; break; diff --git a/src/Iteration.cpp b/src/Iteration.cpp index c7388594f8..366fea0de1 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -208,7 +208,6 @@ void Iteration::flushFileBased( /* create file */ Parameter fCreate; fCreate.name = filename; - fCreate.openPMDversion = s.openPMD(); IOHandler()->enqueue(IOTask(&s.writable(), fCreate)); /* diff --git a/src/Series.cpp b/src/Series.cpp index ccd90b31a8..9aa610f5ce 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -145,6 +145,7 @@ Series &Series::setOpenPMD(std::string const &o) << std::endl; } setAttribute("openPMD", o); + IOHandler()->m_openPMDVersion = o; return *this; } @@ -1478,7 +1479,6 @@ void Series::flushGorVBased( } Parameter fCreate; fCreate.name = series.m_name; - fCreate.openPMDversion = openPMD(); IOHandler()->enqueue(IOTask(this, fCreate)); flushRankTable(); From e686019dbeff34504b330ebd40497a9258ddc713 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 26 Mar 2024 12:39:26 +0100 Subject: [PATCH 21/23] Fixes --- examples/14_toml_template.cpp | 4 ++++ src/IO/JSON/JSONIOHandlerImpl.cpp | 6 ++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/examples/14_toml_template.cpp b/examples/14_toml_template.cpp index 284db9ee84..f595e9b10f 100644 --- a/examples/14_toml_template.cpp +++ b/examples/14_toml_template.cpp @@ -20,6 +20,10 @@ void write() std::string config = R"( { "iteration_encoding": "variable_based", + "json": { + "dataset": {"mode": "template"}, + "attribute": {"mode": "short"} + }, "toml": { "dataset": {"mode": "template"}, "attribute": {"mode": "short"} diff --git a/src/IO/JSON/JSONIOHandlerImpl.cpp b/src/IO/JSON/JSONIOHandlerImpl.cpp index ffed613498..8a7b2086cc 100644 --- a/src/IO/JSON/JSONIOHandlerImpl.cpp +++ b/src/IO/JSON/JSONIOHandlerImpl.cpp @@ -643,14 +643,16 @@ void JSONIOHandlerImpl::createDataset( break; } case IOMode::Template: - if (parameter.extent != Extent{0}) + if (parameter.extent != Extent{0} && + parameter.dtype != Datatype::UNDEFINED) { dset["extent"] = parameter.extent; } else { // no-op - // If extent is empty, don't bother writing it + // If extent is empty or no datatype is defined, don't bother + // writing it } break; } From 305b340acd26f69114728770039faa1c7c65d2f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 7 Jun 2024 14:33:33 +0200 Subject: [PATCH 22/23] Adapt test to recent rebase Reading the chunk table requires NOT using template mode, otherwise the string just consists of '\0' bytes. --- test/SerialIOTest.cpp | 63 ++++++++++++++++++++++++++++++------------- 1 file changed, 45 insertions(+), 18 deletions(-) diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index 6b7405be90..50108992a8 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -1603,28 +1603,18 @@ struct ReadFromAnyType } }; -inline void write_test(const std::string &backend) +inline void write_test( + const std::string &backend, + std::string jsonCfg = "{}", + bool test_rank_table = true) { -#ifdef _WIN32 - std::string jsonCfg = "{}"; -#else - std::string jsonCfg = R"({"rank_table": "posix_hostname"})"; +#ifndef _WIN32 + jsonCfg = json::merge(jsonCfg, R"({"rank_table": "posix_hostname"})"); chunk_assignment::RankMeta compare{ {0, host_info::byMethod( host_info::methodFromStringDescription("posix_hostname", false))}}; #endif - jsonCfg = json::merge(jsonCfg, R"( -{ - "json": { - "dataset": { - "mode": "template" - }, - "attribute": { - "mode": "short" - } - } -})"); Series o = Series("../samples/serial_write." + backend, Access::CREATE, jsonCfg); @@ -1741,7 +1731,10 @@ inline void write_test(const std::string &backend) variantTypeDataset); #ifndef _WIN32 - REQUIRE(read.rankTable(/* collective = */ false) == compare); + if (test_rank_table) + { + REQUIRE(read.rankTable(/* collective = */ false) == compare); + } #endif } @@ -1749,7 +1742,41 @@ TEST_CASE("write_test", "[serial]") { for (auto const &t : testedFileExtensions()) { - write_test(t); + if (t == "json") + { + write_test( + "template." + t, + R"( +{ + "json": { + "dataset": { + "mode": "template" + }, + "attribute": { + "mode": "short" + } + } +})", + false); + write_test( + t, + R"( +{ + "json": { + "dataset": { + "mode": "dataset" + }, + "attribute": { + "mode": "short" + } + } +})", + true); + } + else + { + write_test(t); + } Series list{"../samples/serial_write." + t, Access::READ_ONLY}; helper::listSeries(list); } From 17288941e20ca603439f5047be0a1f3451c37849 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 5 Aug 2024 11:43:54 +0200 Subject: [PATCH 23/23] toml11 4.0 compatibility --- src/IO/JSON/JSONIOHandlerImpl.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/IO/JSON/JSONIOHandlerImpl.cpp b/src/IO/JSON/JSONIOHandlerImpl.cpp index 8a7b2086cc..f4b17569cb 100644 --- a/src/IO/JSON/JSONIOHandlerImpl.cpp +++ b/src/IO/JSON/JSONIOHandlerImpl.cpp @@ -151,7 +151,7 @@ namespace auto asToml = openPMD::json::jsonToToml(shadow); std::cerr << "Warning: parts of the backend configuration for " "JSON/TOML backend remain unused:\n" - << asToml << std::endl; + << json::format_toml(asToml) << std::endl; break; } }