From 9ec90b68b91203513046147e1f4336edb8b9547b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 17 Aug 2023 19:34:07 +0200 Subject: [PATCH] TOML Backend (#1436) * TOML backend * Add documentation for TOML * Fixes for long double and long integer types * Only run TOML tests if TOML is available TOML is not shown as available on NVIDIA compilers * Deactivate long double entirely for JSON/TOML * CI fix: unused variable * Hide/deactivate/warn Toml backend on nvcc compilers https://github.com/ToruNiina/toml11/issues/205 * Usage notes for JSON / TOML * Update comment in test/python/unittest/API/APITest.py Co-authored-by: Axel Huebl * Update documentation text Co-authored-by: Axel Huebl --------- Co-authored-by: Axel Huebl --- docs/source/backends/json.rst | 43 +++- include/openPMD/IO/Format.hpp | 1 + include/openPMD/IO/JSON/JSONIOHandler.hpp | 7 +- include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp | 45 +++- include/openPMD/auxiliary/TypeTraits.hpp | 16 ++ src/Format.cpp | 4 + src/IO/AbstractIOHandlerHelper.cpp | 15 +- src/IO/JSON/JSONIOHandler.cpp | 10 +- src/IO/JSON/JSONIOHandlerImpl.cpp | 225 +++++++++++++----- src/Series.cpp | 3 +- src/auxiliary/JSON.cpp | 8 +- src/config.cpp | 11 + test/CoreTest.cpp | 2 +- test/SerialIOTest.cpp | 57 ++++- test/python/unittest/API/APITest.py | 5 +- 15 files changed, 349 insertions(+), 103 deletions(-) diff --git a/docs/source/backends/json.rst b/docs/source/backends/json.rst index 752497aa09..48ec6b1f44 100644 --- a/docs/source/backends/json.rst +++ b/docs/source/backends/json.rst @@ -1,10 +1,19 @@ .. _backends-json: -JSON -==== +JSON/TOML +========= -openPMD supports writing to and reading from JSON files. -The JSON backend is always available. +openPMD supports writing to and reading from JSON and TOML files. +The JSON and TOML backends are always available. + +.. note:: + + Both the JSON and the TOML backends are not intended for large-scale data I/O. + + The JSON backend is mainly intended for prototyping and learning, or similar workflows where setting up a large IO backend such as HDF5 or ADIOS2 is perceived as obstructive. It can also be used for small datasets that need to be stored in text format rather than binary. + + The TOML backend is intended for exchanging the *structure* of a data series without its "heavy" data fields. + For instance, one can easily create and exchange human-readable, machine-actionable data configurations for experiments and simulations. JSON File Format @@ -43,9 +52,17 @@ Every such attribute is itself a JSON object with two keys: * ``datatype``: A string describing the type of the value. * ``value``: The actual value of type ``datatype``. +TOML File Format +---------------- + +A TOML file uses the file ending ``.toml``. The TOML backend is chosen by creating a ``Series`` object with a filename that has this file ending. + +The TOML backend internally works with JSON datasets and converts to/from TOML during I/O. +As a result, data layout and usage are equivalent to the JSON backend. + -Restrictions ------------- +JSON Restrictions +----------------- For creation of JSON serializations (i.e. writing), the restrictions of the JSON backend are equivalent to those of the `JSON library by Niels Lohmann `_ @@ -77,6 +94,20 @@ The (keys) names ``"attributes"``, ``"data"`` and ``"datatype"`` are reserved an A parallel (i.e. MPI) implementation is *not* available. +TOML Restrictions +----------------- + +Note that the JSON datatype-specific restrictions do not automatically hold for TOML, as those affect only the representation on disk, not the internal representation. + +TOML supports most numeric types, with the support for long double and long integer types being platform-defined. +Special floating point values such as NaN are also support. + +TOML does not support null values. + +The (keys) names ``"attributes"``, ``"data"`` and ``"datatype"`` are reserved and must not be used for base/mesh/particles path, records and their components. + +A parallel (i.e. MPI) implementation is *not* available. + Example ------- diff --git a/include/openPMD/IO/Format.hpp b/include/openPMD/IO/Format.hpp index 43ec4d04a1..858da29a40 100644 --- a/include/openPMD/IO/Format.hpp +++ b/include/openPMD/IO/Format.hpp @@ -35,6 +35,7 @@ enum class Format ADIOS2_SST, ADIOS2_SSC, JSON, + TOML, DUMMY }; diff --git a/include/openPMD/IO/JSON/JSONIOHandler.hpp b/include/openPMD/IO/JSON/JSONIOHandler.hpp index 0cdc6f3c36..452098137e 100644 --- a/include/openPMD/IO/JSON/JSONIOHandler.hpp +++ b/include/openPMD/IO/JSON/JSONIOHandler.hpp @@ -29,7 +29,12 @@ namespace openPMD class JSONIOHandler : public AbstractIOHandler { public: - JSONIOHandler(std::string path, Access at); + JSONIOHandler( + std::string path, + Access at, + openPMD::json::TracingJSON config, + JSONIOHandlerImpl::FileFormat, + std::string originalExtension); ~JSONIOHandler() override; diff --git a/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp index 7f10f62cd9..c935647665 100644 --- a/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp +++ b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp @@ -26,8 +26,10 @@ #include "openPMD/IO/Access.hpp" #include "openPMD/IO/JSON/JSONFilePosition.hpp" #include "openPMD/auxiliary/Filesystem.hpp" +#include "openPMD/auxiliary/JSON_internal.hpp" #include "openPMD/config.hpp" +#include #include #include @@ -153,7 +155,17 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl using json = nlohmann::json; public: - explicit JSONIOHandlerImpl(AbstractIOHandler *); + enum class FileFormat + { + Json, + Toml + }; + + explicit JSONIOHandlerImpl( + AbstractIOHandler *, + openPMD::json::TracingJSON config, + FileFormat, + std::string originalExtension); ~JSONIOHandlerImpl() override; @@ -229,15 +241,25 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl // files that have logically, but not physically been written to std::unordered_set m_dirty; + /* + * Is set by constructor. + */ + FileFormat m_fileFormat{}; + + std::string m_originalExtension; + // HELPER FUNCTIONS - // will use the IOHandler to retrieve the correct directory - // shared pointer to circumvent the fact that c++ pre 17 does - // not enforce (only allow) copy elision in return statements - std::shared_ptr getFilehandle( - File, - Access access); //, Access - // m_frontendAccess=this->m_handler->m_frontendAccess); + // will use the IOHandler to retrieve the correct directory. + // first tuple element will be the underlying opened file handle. + // if Access is read mode, then the second tuple element will be the istream + // casted to precision std::numeric_limits::digits10 + 1, else null. + // if Access is write mode, then the second tuple element will be the + // ostream casted to precision std::numeric_limits::digits10 + 1, + // else null. first tuple element needs to be a pointer, since the casted + // streams are references only. + std::tuple, std::istream *, std::ostream *> + getFilehandle(File, Access access); // full operating system path of the given file std::string fullPath(File); @@ -272,15 +294,13 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl // essentially: m_i = \prod_{j=0}^{i-1} extent_j static Extent getMultiplicators(Extent const &extent); - static nlohmann::json initializeNDArray(Extent const &extent); - static Extent getExtent(nlohmann::json &j); // remove single '/' in the beginning and end of a string static std::string removeSlashes(std::string); template - static bool hasKey(nlohmann::json &, KeyT &&key); + static bool hasKey(nlohmann::json const &, KeyT &&key); // make sure that the given path exists in proper form in // the passed json value @@ -366,7 +386,8 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl struct AttributeReader { template - static void call(nlohmann::json &, Parameter &); + static void + call(nlohmann::json const &, Parameter &); static constexpr char const *errorMsg = "JSON: writeAttribute"; }; diff --git a/include/openPMD/auxiliary/TypeTraits.hpp b/include/openPMD/auxiliary/TypeTraits.hpp index 923cfb5be2..3e5a36774e 100644 --- a/include/openPMD/auxiliary/TypeTraits.hpp +++ b/include/openPMD/auxiliary/TypeTraits.hpp @@ -24,6 +24,7 @@ #include "openPMD/auxiliary/UniquePtr.hpp" #include +#include #include // size_t #include #include @@ -56,6 +57,18 @@ namespace detail static constexpr bool value = true; }; + template + struct IsComplex + { + static constexpr bool value = false; + }; + + template + struct IsComplex> + { + static constexpr bool value = true; + }; + template struct IsPointer { @@ -114,6 +127,9 @@ using IsPointer_t = typename detail::IsPointer::type; template inline constexpr bool IsContiguousContainer_v = IsVector_v || IsArray_v; +template +inline constexpr bool IsComplex_v = detail::IsComplex::value; + namespace { // see https://en.cppreference.com/w/cpp/language/if diff --git a/src/Format.cpp b/src/Format.cpp index d5a8acf5f3..8a6ead832a 100644 --- a/src/Format.cpp +++ b/src/Format.cpp @@ -43,6 +43,8 @@ Format determineFormat(std::string const &filename) return Format::ADIOS2_SSC; if (auxiliary::ends_with(filename, ".json")) return Format::JSON; + if (auxiliary::ends_with(filename, ".toml")) + return Format::TOML; // Format might still be specified via JSON return Format::DUMMY; @@ -66,6 +68,8 @@ std::string suffix(Format f) return ".ssc"; case Format::JSON: return ".json"; + case Format::TOML: + return ".toml"; default: return ""; } diff --git a/src/IO/AbstractIOHandlerHelper.cpp b/src/IO/AbstractIOHandlerHelper.cpp index 4cd74a4de2..c6cd69f8a5 100644 --- a/src/IO/AbstractIOHandlerHelper.cpp +++ b/src/IO/AbstractIOHandlerHelper.cpp @@ -194,7 +194,20 @@ std::unique_ptr createIOHandler( std::move(originalExtension)); case Format::JSON: return constructIOHandler( - "JSON", path, access); + "JSON", + path, + access, + std::move(options), + JSONIOHandlerImpl::FileFormat::Json, + std::move(originalExtension)); + case Format::TOML: + return constructIOHandler( + "JSON", + path, + access, + std::move(options), + JSONIOHandlerImpl::FileFormat::Toml, + std::move(originalExtension)); default: throw std::runtime_error( "Unknown file format! Did you specify a file ending? Specified " diff --git a/src/IO/JSON/JSONIOHandler.cpp b/src/IO/JSON/JSONIOHandler.cpp index 10ffce9927..7eb8a57278 100644 --- a/src/IO/JSON/JSONIOHandler.cpp +++ b/src/IO/JSON/JSONIOHandler.cpp @@ -25,8 +25,14 @@ namespace openPMD { JSONIOHandler::~JSONIOHandler() = default; -JSONIOHandler::JSONIOHandler(std::string path, Access at) - : AbstractIOHandler{path, at}, m_impl{JSONIOHandlerImpl{this}} +JSONIOHandler::JSONIOHandler( + std::string path, + Access at, + openPMD::json::TracingJSON jsonCfg, + JSONIOHandlerImpl::FileFormat format, + std::string originalExtension) + : AbstractIOHandler{path, at} + , m_impl{this, std::move(jsonCfg), format, std::move(originalExtension)} {} std::future JSONIOHandler::flush(internal::ParsedFlushParams &) diff --git a/src/IO/JSON/JSONIOHandlerImpl.cpp b/src/IO/JSON/JSONIOHandlerImpl.cpp index bec6cebb71..73d50366f7 100644 --- a/src/IO/JSON/JSONIOHandlerImpl.cpp +++ b/src/IO/JSON/JSONIOHandlerImpl.cpp @@ -26,8 +26,12 @@ #include "openPMD/auxiliary/Filesystem.hpp" #include "openPMD/auxiliary/Memory.hpp" #include "openPMD/auxiliary/StringManip.hpp" +#include "openPMD/auxiliary/TypeTraits.hpp" #include "openPMD/backend/Writable.hpp" +#include + +#include #include #include #include @@ -54,9 +58,82 @@ namespace openPMD throw std::runtime_error((TEXT)); \ } -JSONIOHandlerImpl::JSONIOHandlerImpl(AbstractIOHandler *handler) +namespace +{ + struct DefaultValue + { + template + static nlohmann::json call() + { + if constexpr (auxiliary::IsComplex_v) + { + return typename T::value_type{}; + } + else + { + return T{}; + } +#if defined(__INTEL_COMPILER) +/* + * ICPC has trouble with if constexpr, thinking that return statements are + * missing afterwards. Deactivate the warning. + * Note that putting a statement here will not help to fix this since it will + * then complain about unreachable code. + * https://community.intel.com/t5/Intel-C-Compiler/quot-if-constexpr-quot-and-quot-missing-return-statement-quot-in/td-p/1154551 + */ +#pragma warning(disable : 1011) + } +#pragma warning(default : 1011) +#else + } +#endif + + static constexpr char const *errorMsg = "JSON default value"; + }; + + /* + * If initializeWithDefaultValue contains a datatype, then the dataset ought + * to be initialized with the zero value of that dataset. + * Otherwise with null. + */ + nlohmann::json initializeNDArray( + Extent const &extent, + std::optional initializeWithDefaultValue) + { + // idea: begin from the innermost shale and copy the result into the + // outer shales + nlohmann::json accum = initializeWithDefaultValue.has_value() + ? switchNonVectorType( + initializeWithDefaultValue.value()) + : nlohmann::json(); + nlohmann::json old; + auto *accum_ptr = &accum; + auto *old_ptr = &old; + for (auto it = extent.rbegin(); it != extent.rend(); it++) + { + std::swap(old_ptr, accum_ptr); + *accum_ptr = nlohmann::json::array(); + for (Extent::value_type i = 0; i < *it; i++) + { + (*accum_ptr)[i] = *old_ptr; // copy boi + } + } + return *accum_ptr; + } +} // namespace + +JSONIOHandlerImpl::JSONIOHandlerImpl( + AbstractIOHandler *handler, + openPMD::json::TracingJSON config, + FileFormat format, + std::string originalExtension) : AbstractIOHandlerImpl(handler) -{} + , m_fileFormat{format} + , m_originalExtension{std::move(originalExtension)} +{ + // Currently unused + (void)config; +} JSONIOHandlerImpl::~JSONIOHandlerImpl() = default; @@ -80,11 +157,7 @@ void JSONIOHandlerImpl::createFile( if (!writable->written) { - std::string name = parameters.name; - if (!auxiliary::ends_with(name, ".json")) - { - name += ".json"; - } + std::string name = parameters.name + m_originalExtension; auto res_pair = getPossiblyExisting(name); auto fullPathToFile = fullPath(std::get<0>(res_pair)); @@ -205,20 +278,23 @@ void JSONIOHandlerImpl::createDataset( setAndGetFilePosition(writable, name); auto &dset = jsonVal[name]; dset["datatype"] = datatypeToString(parameter.dtype); + auto extent = parameter.extent; switch (parameter.dtype) { case Datatype::CFLOAT: case Datatype::CDOUBLE: case Datatype::CLONG_DOUBLE: { - auto complexExtent = parameter.extent; - complexExtent.push_back(2); - dset["data"] = initializeNDArray(complexExtent); + extent.push_back(2); break; } default: - dset["data"] = initializeNDArray(parameter.extent); break; } + // TOML does not support nulls, so initialize with zero + dset["data"] = initializeNDArray( + extent, + m_fileFormat == FileFormat::Json ? std::optional() + : parameter.dtype); writable->written = true; m_dirty.emplace(file); } @@ -276,27 +352,28 @@ void JSONIOHandlerImpl::extendDataset( throw std::runtime_error( "[JSON] The specified location contains no valid dataset"); } - switch (stringToDatatype(j["datatype"].get())) + auto extent = parameters.extent; + auto datatype = stringToDatatype(j["datatype"].get()); + switch (datatype) { case Datatype::CFLOAT: case Datatype::CDOUBLE: case Datatype::CLONG_DOUBLE: { - // @todo test complex resizing - auto complexExtent = parameters.extent; - complexExtent.push_back(2); - nlohmann::json newData = initializeNDArray(complexExtent); - nlohmann::json &oldData = j["data"]; - mergeInto(newData, oldData); - j["data"] = newData; + extent.push_back(2); break; } default: - nlohmann::json newData = initializeNDArray(parameters.extent); - nlohmann::json &oldData = j["data"]; - mergeInto(newData, oldData); - j["data"] = newData; + // nothing to do break; } + // TOML does not support nulls, so initialize with zero + nlohmann::json newData = initializeNDArray( + extent, + m_fileFormat == FileFormat::Json ? std::optional() + : datatype); + nlohmann::json &oldData = j["data"]; + mergeInto(newData, oldData); + j["data"] = newData; writable->written = true; } @@ -521,11 +598,7 @@ void JSONIOHandlerImpl::openFile( "Supplied directory is not valid: " + m_handler->directory); } - std::string name = parameter.name; - if (!auxiliary::ends_with(name, ".json")) - { - name += ".json"; - } + std::string name = parameter.name + m_originalExtension; auto file = std::get<0>(getPossiblyExisting(name)); @@ -844,7 +917,8 @@ void JSONIOHandlerImpl::readAttribute( "[JSON] Attributes have to be written before reading.") refreshFileFromParent(writable); auto name = removeSlashes(parameters.name); - auto &jsonLoc = obtainJsonContents(writable)["attributes"]; + auto const &jsonContents = obtainJsonContents(writable); + auto const &jsonLoc = jsonContents["attributes"]; setAndGetFilePosition(writable); std::string error_msg("[JSON] No such attribute '"); if (!hasKey(jsonLoc, name)) @@ -919,7 +993,12 @@ void JSONIOHandlerImpl::listAttributes( "[JSON] Attributes have to be written before reading.") refreshFileFromParent(writable); auto filePosition = setAndGetFilePosition(writable); - auto &j = obtainJsonContents(writable)["attributes"]; + auto const &jsonContents = obtainJsonContents(writable); + if (!jsonContents.contains("attributes")) + { + return; + } + auto const &j = jsonContents["attributes"]; for (auto it = j.begin(); it != j.end(); it++) { parameters.attributes->push_back(it.key()); @@ -932,14 +1011,16 @@ void JSONIOHandlerImpl::deregister( m_files.erase(writable); } -std::shared_ptr -JSONIOHandlerImpl::getFilehandle(File fileName, Access access) +auto JSONIOHandlerImpl::getFilehandle(File fileName, Access access) + -> std::tuple, std::istream *, std::ostream *> { VERIFY_ALWAYS( fileName.valid(), "[JSON] Tried opening a file that has been overwritten or deleted.") auto path = fullPath(std::move(fileName)); - auto fs = std::make_shared(); + auto fs = std::make_unique(); + std::istream *istream = nullptr; + std::ostream *ostream = nullptr; if (access::write(access)) { /* @@ -949,14 +1030,31 @@ JSONIOHandlerImpl::getFilehandle(File fileName, Access access) * equivalent, but the openPMD frontend exposes no reading * functionality in APPEND mode. */ - fs->open(path, std::ios_base::out | std::ios_base::trunc); + std::ios_base::openmode openmode = + std::ios_base::out | std::ios_base::trunc; + if (m_fileFormat == FileFormat::Toml) + { + openmode |= std::ios_base::binary; + } + fs->open(path, openmode); + ostream = + &(*fs << std::setprecision( + std::numeric_limits::digits10 + 1)); } else { - fs->open(path, std::ios_base::in); + std::ios_base::openmode openmode = std::ios_base::in; + if (m_fileFormat == FileFormat::Toml) + { + openmode |= std::ios_base::binary; + } + fs->open(path, openmode); + istream = + &(*fs >> + std::setprecision(std::numeric_limits::digits10 + 1)); } VERIFY(fs->good(), "[JSON] Failed opening a file '" + path + "'"); - return fs; + return std::make_tuple(std::move(fs), istream, ostream); } std::string JSONIOHandlerImpl::fullPath(File fileName) @@ -1048,26 +1146,6 @@ Extent JSONIOHandlerImpl::getMultiplicators(Extent const &extent) return res; } -nlohmann::json JSONIOHandlerImpl::initializeNDArray(Extent const &extent) -{ - // idea: begin from the innermost shale and copy the result into the - // outer shales - nlohmann::json accum; - nlohmann::json old; - auto *accum_ptr = &accum; - auto *old_ptr = &old; - for (auto it = extent.rbegin(); it != extent.rend(); it++) - { - std::swap(old_ptr, accum_ptr); - *accum_ptr = nlohmann::json{}; - for (Extent::value_type i = 0; i < *it; i++) - { - (*accum_ptr)[i] = *old_ptr; // copy boi - } - } - return *accum_ptr; -} - Extent JSONIOHandlerImpl::getExtent(nlohmann::json &j) { Extent res; @@ -1106,7 +1184,7 @@ std::string JSONIOHandlerImpl::removeSlashes(std::string s) } template -bool JSONIOHandlerImpl::hasKey(nlohmann::json &j, KeyT &&key) +bool JSONIOHandlerImpl::hasKey(nlohmann::json const &j, KeyT &&key) { return j.find(std::forward(key)) != j.end(); } @@ -1166,9 +1244,19 @@ std::shared_ptr JSONIOHandlerImpl::obtainJsonContents(File file) return it->second; } // read from file - auto fh = getFilehandle(file, Access::READ_ONLY); + auto [fh, fh_with_precision, _] = getFilehandle(file, Access::READ_ONLY); + (void)_; std::shared_ptr res = std::make_shared(); - *fh >> *res; + switch (m_fileFormat) + { + case FileFormat::Json: + *fh_with_precision >> *res; + break; + case FileFormat::Toml: + *res = + openPMD::json::tomlToJson(toml::parse(*fh_with_precision, *file)); + break; + } VERIFY(fh->good(), "[JSON] Failed reading from a file."); m_jsonVals.emplace(file, res); return res; @@ -1192,9 +1280,22 @@ void JSONIOHandlerImpl::putJsonContents( auto it = m_jsonVals.find(filename); if (it != m_jsonVals.end()) { - auto fh = getFilehandle(filename, Access::CREATE); + auto [fh, _, fh_with_precision] = + getFilehandle(filename, Access::CREATE); + (void)_; (*it->second)["platform_byte_widths"] = platformSpecifics(); - *fh << *it->second << std::endl; + + switch (m_fileFormat) + { + case FileFormat::Json: + *fh_with_precision << *it->second << std::endl; + break; + case FileFormat::Toml: + *fh_with_precision << openPMD::json::jsonToToml(*it->second) + << std::endl; + break; + } + VERIFY(fh->good(), "[JSON] Failed writing data to disk.") m_jsonVals.erase(it); if (unsetDirty) @@ -1399,7 +1500,7 @@ void JSONIOHandlerImpl::AttributeWriter::call( template void JSONIOHandlerImpl::AttributeReader::call( - nlohmann::json &json, Parameter ¶meters) + nlohmann::json const &json, Parameter ¶meters) { JsonToCpp jtc; *parameters.resource = jtc(json); diff --git a/src/Series.cpp b/src/Series.cpp index 4a22330179..59a96efdb7 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -2180,7 +2180,8 @@ void Series::parseJsonOptions(TracingJSON &options, ParsedInput &input) std::map const backendDescriptors{ {"hdf5", Format::HDF5}, {"adios2", Format::ADIOS2_BP}, - {"json", Format::JSON}}; + {"json", Format::JSON}, + {"toml", Format::TOML}}; std::string backend; getJsonOptionLowerCase(options, "backend", backend); if (!backend.empty()) diff --git a/src/auxiliary/JSON.cpp b/src/auxiliary/JSON.cpp index c04e672ae6..168cab7bf6 100644 --- a/src/auxiliary/JSON.cpp +++ b/src/auxiliary/JSON.cpp @@ -200,6 +200,7 @@ namespace return nlohmann::json(); // null } + // @todo maybe generalize error type throw error::BackendConfigSchema( currentPath, "Unexpected datatype in TOML configuration. This is probably a " @@ -215,7 +216,8 @@ namespace switch (val.type()) { case nlohmann::json::value_t::null: - return toml::value(); + throw error::BackendConfigSchema( + currentPath, "TOML does not support null values."); case nlohmann::json::value_t::object: { toml::value::table_type res; for (auto pair = val.begin(); pair != val.end(); ++pair) @@ -247,7 +249,7 @@ namespace case nlohmann::json::value_t::number_unsigned: return val.get(); case nlohmann::json::value_t::number_float: - return val.get(); + return (long double)val.get(); case nlohmann::json::value_t::binary: return val.get(); case nlohmann::json::value_t::discarded: @@ -501,7 +503,7 @@ std::optional asLowerCaseStringDynamic(nlohmann::json const &value) std::vector backendKeys() { - return {"adios2", "json", "hdf5"}; + return {"adios2", "json", "toml", "hdf5"}; } void warnGlobalUnusedOptions(TracingJSON const &config) diff --git a/src/config.cpp b/src/config.cpp index a44925287a..89a824500c 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -28,20 +28,31 @@ #include #include +// @todo add TOML here std::map openPMD::getVariants() { + // clang-format off return std::map{ {"mpi", bool(openPMD_HAVE_MPI)}, {"json", true}, +// https://github.com/ToruNiina/toml11/issues/205 +#if !defined(__NVCOMPILER_MAJOR__) || __NVCOMPILER_MAJOR__ >= 23 + {"toml", true}, +#endif {"hdf5", bool(openPMD_HAVE_HDF5)}, {"adios1", false}, {"adios2", bool(openPMD_HAVE_ADIOS2)}}; + // clang-format on } std::vector openPMD::getFileExtensions() { std::vector fext; fext.emplace_back("json"); +// https://github.com/ToruNiina/toml11/issues/205 +#if !defined(__NVCOMPILER_MAJOR__) || __NVCOMPILER_MAJOR__ >= 23 + fext.emplace_back("toml"); +#endif #if openPMD_HAVE_ADIOS2 fext.emplace_back("bp"); #endif diff --git a/test/CoreTest.cpp b/test/CoreTest.cpp index 224596864e..d660e29ec4 100644 --- a/test/CoreTest.cpp +++ b/test/CoreTest.cpp @@ -1051,7 +1051,7 @@ TEST_CASE("no_file_ending", "[core]") Access::CREATE, R"({"backend": "json"})"); } - REQUIRE(auxiliary::file_exists("../samples/no_extension_specified.json")); + REQUIRE(auxiliary::file_exists("../samples/no_extension_specified")); } TEST_CASE("backend_via_json", "[core]") diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index 11297c7595..962ca636aa 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -81,20 +81,20 @@ std::vector testedFileExtensions() allExtensions.begin(), allExtensions.end(), []([[maybe_unused]] std::string const &ext) { -#if openPMD_HAVE_ADIOS2 -#define HAS_ADIOS_2_9 (ADIOS2_VERSION_MAJOR * 100 + ADIOS2_VERSION_MINOR >= 209) -#if HAS_ADIOS_2_9 +#if openPMD_HAS_ADIOS_2_9 // sst and ssc need a receiver for testing // bp5 is already tested via bp - return ext == "sst" || ext == "ssc" || ext == "bp5"; + // toml parsing is very slow and its implementation is equivalent to + // the json backend, so it is only activated for selected tests + return ext == "sst" || ext == "ssc" || ext == "bp5" || + ext == "toml"; #else + // toml parsing is very slow and its implementation is equivalent to + // the json backend, so it is only activated for selected tests // sst and ssc need a receiver for testing // bp4 is already tested via bp - return ext == "sst" || ext == "ssc" || ext == "bp4"; -#endif -#undef HAS_ADIOS_2_9 -#else - return false; + return ext == "sst" || ext == "ssc" || ext == "bp4" || + ext == "toml"; #endif }); return {allExtensions.begin(), newEnd}; @@ -1231,7 +1231,7 @@ TEST_CASE("particle_patches", "[serial]") inline void dtype_test(const std::string &backend) { - bool test_long_double = (backend != "json") || sizeof(long double) <= 8; + bool test_long_double = backend != "json" && backend != "toml"; bool test_long_long = (backend != "json") || sizeof(long long) <= 8; { Series s = Series("../samples/dtype_test." + backend, Access::CREATE); @@ -1447,7 +1447,10 @@ inline void dtype_test(const std::string &backend) REQUIRE(s.getAttribute("short").dtype == Datatype::SHORT); REQUIRE(s.getAttribute("int").dtype == Datatype::INT); REQUIRE(s.getAttribute("long").dtype == Datatype::LONG); - REQUIRE(s.getAttribute("longlong").dtype == Datatype::LONGLONG); + if (test_long_long) + { + REQUIRE(s.getAttribute("longlong").dtype == Datatype::LONGLONG); + } REQUIRE(s.getAttribute("ushort").dtype == Datatype::USHORT); REQUIRE(s.getAttribute("uint").dtype == Datatype::UINT); REQUIRE(s.getAttribute("ulong").dtype == Datatype::ULONG); @@ -1459,7 +1462,10 @@ inline void dtype_test(const std::string &backend) REQUIRE(s.getAttribute("vecShort").dtype == Datatype::VEC_SHORT); REQUIRE(s.getAttribute("vecInt").dtype == Datatype::VEC_INT); REQUIRE(s.getAttribute("vecLong").dtype == Datatype::VEC_LONG); - REQUIRE(s.getAttribute("vecLongLong").dtype == Datatype::VEC_LONGLONG); + if (test_long_long) + { + REQUIRE(s.getAttribute("vecLongLong").dtype == Datatype::VEC_LONGLONG); + } REQUIRE(s.getAttribute("vecUShort").dtype == Datatype::VEC_USHORT); REQUIRE(s.getAttribute("vecUInt").dtype == Datatype::VEC_UINT); REQUIRE(s.getAttribute("vecULong").dtype == Datatype::VEC_ULONG); @@ -1508,6 +1514,15 @@ TEST_CASE("dtype_test", "[serial]") { dtype_test(t); } + if (auto extensions = getFileExtensions(); + std::find(extensions.begin(), extensions.end(), "toml") != + extensions.end()) + { /* + * TOML backend is not generally tested for performance reasons, opt in to + * testing it here. + */ + dtype_test("toml"); + } } inline void write_test(const std::string &backend) @@ -2106,6 +2121,15 @@ TEST_CASE("fileBased_write_test", "[serial]") { fileBased_write_test(t); } + if (auto extensions = getFileExtensions(); + std::find(extensions.begin(), extensions.end(), "toml") != + extensions.end()) + { /* + * TOML backend is not generally tested for performance reasons, opt in to + * testing it here. + */ + fileBased_write_test("toml"); + } } inline void sample_write_thetaMode(std::string file_ending) @@ -7405,4 +7429,13 @@ TEST_CASE("groupbased_read_write", "[serial]") } } } + if (auto extensions = getFileExtensions(); + std::find(extensions.begin(), extensions.end(), "toml") != + extensions.end()) + { /* + * TOML backend is not generally tested for performance reasons, opt in to + * testing it here. + */ + groupbased_read_write("toml"); + } } diff --git a/test/python/unittest/API/APITest.py b/test/python/unittest/API/APITest.py index 0aa71cf7a8..a510098c8d 100644 --- a/test/python/unittest/API/APITest.py +++ b/test/python/unittest/API/APITest.py @@ -359,7 +359,8 @@ def attributeRoundTrip(self, file_ending): # c_types self.assertEqual(series.get_attribute("byte_c"), 30) self.assertEqual(series.get_attribute("ubyte_c"), 50) - if file_ending != "json": # TODO: returns [100] instead of 100 in json + # TODO: returns [100] instead of 100 in json/toml + if file_ending != "json" and file_ending != "toml": self.assertEqual(chr(series.get_attribute("char_c")), 'd') self.assertEqual(series.get_attribute("int16_c"), 2) self.assertEqual(series.get_attribute("int32_c"), 3) @@ -1824,7 +1825,7 @@ def testIterator(self): self.makeIteratorRoundTrip(b, backend_filesupport[b]) def makeAvailableChunksRoundTrip(self, ext): - if ext == "h5": + if ext == "h5" or ext == "toml": return name = "../samples/available_chunks_python." + ext write = io.Series(