TOML Backend (#1436)

* TOML backend * Add documentation for TOML * Fixes for long double and long integer types * Only run TOML tests if TOML is available TOML is not shown as available on NVIDIA compilers * Deactivate long double entirely for JSON/TOML * CI fix: unused variable * Hide/deactivate/warn Toml backend on nvcc compilers ToruNiina/toml11#205 * Usage notes for JSON / TOML * Update comment in test/python/unittest/API/APITest.py Co-authored-by: Axel Huebl <[email protected]> * Update documentation text Co-authored-by: Axel Huebl <[email protected]> --------- Co-authored-by: Axel Huebl <[email protected]>
openPMD · Aug 17, 2023 · 9ec90b6 · 9ec90b6
1 parent 99daca7
commit 9ec90b6
Show file tree

Hide file tree

Showing 15 changed files with 349 additions and 103 deletions.
diff --git a/docs/source/backends/json.rst b/docs/source/backends/json.rst
@@ -1,10 +1,19 @@
 .. _backends-json:
 
-JSON
-====
+JSON/TOML
+=========
 
-openPMD supports writing to and reading from JSON files.
-The JSON backend is always available.
+openPMD supports writing to and reading from JSON and TOML files.
+The JSON and TOML backends are always available.
+
+.. note::
+
+   Both the JSON and the TOML backends are not intended for large-scale data I/O.
+
+   The JSON backend is mainly intended for prototyping and learning, or similar workflows where setting up a large IO backend such as HDF5 or ADIOS2 is perceived as obstructive. It can also be used for small datasets that need to be stored in text format rather than binary.
+
+   The TOML backend is intended for exchanging the *structure* of a data series without its "heavy" data fields.
+   For instance, one can easily create and exchange human-readable, machine-actionable data configurations for experiments and simulations.
 
 
 JSON File Format
@@ -43,9 +52,17 @@ Every such attribute is itself a JSON object with two keys:
  * ``datatype``: A string describing the type of the value.
  * ``value``: The actual value of type ``datatype``.
 
+TOML File Format
+----------------
+
+A TOML file uses the file ending ``.toml``. The TOML backend is chosen by creating a ``Series`` object with a filename that has this file ending.
+
+The TOML backend internally works with JSON datasets and converts to/from TOML during I/O.
+As a result, data layout and usage are equivalent to the JSON backend.
+
 
-Restrictions
-------------
+JSON Restrictions
+-----------------
 
 For creation of JSON serializations (i.e. writing), the restrictions of the JSON backend are
 equivalent to those of the `JSON library by Niels Lohmann <https://github.com/nlohmann/json>`_
@@ -77,6 +94,20 @@ The (keys) names ``"attributes"``, ``"data"`` and ``"datatype"`` are reserved an
 
 A parallel (i.e. MPI) implementation is *not* available.
 
+TOML Restrictions
+-----------------
+
+Note that the JSON datatype-specific restrictions do not automatically hold for TOML, as those affect only the representation on disk, not the internal representation.
+
+TOML supports most numeric types, with the support for long double and long integer types being platform-defined.
+Special floating point values such as NaN are also support.
+
+TOML does not support null values.
+
+The (keys) names ``"attributes"``, ``"data"`` and ``"datatype"`` are reserved and must not be used for base/mesh/particles path, records and their components.
+
+A parallel (i.e. MPI) implementation is *not* available.
+
 
 Example
 -------

diff --git a/include/openPMD/IO/Format.hpp b/include/openPMD/IO/Format.hpp
@@ -35,6 +35,7 @@ enum class Format
     ADIOS2_SST,
     ADIOS2_SSC,
     JSON,
+    TOML,
     DUMMY
 };
 

diff --git a/include/openPMD/IO/JSON/JSONIOHandler.hpp b/include/openPMD/IO/JSON/JSONIOHandler.hpp
@@ -29,7 +29,12 @@ namespace openPMD
 class JSONIOHandler : public AbstractIOHandler
 {
 public:
-    JSONIOHandler(std::string path, Access at);
+    JSONIOHandler(
+        std::string path,
+        Access at,
+        openPMD::json::TracingJSON config,
+        JSONIOHandlerImpl::FileFormat,
+        std::string originalExtension);
 
     ~JSONIOHandler() override;
 

diff --git a/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp
@@ -26,8 +26,10 @@
 #include "openPMD/IO/Access.hpp"
 #include "openPMD/IO/JSON/JSONFilePosition.hpp"
 #include "openPMD/auxiliary/Filesystem.hpp"
+#include "openPMD/auxiliary/JSON_internal.hpp"
 #include "openPMD/config.hpp"
 
+#include <istream>
 #include <nlohmann/json.hpp>
 
 #include <complex>
@@ -153,7 +155,17 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl
     using json = nlohmann::json;
 
 public:
-    explicit JSONIOHandlerImpl(AbstractIOHandler *);
+    enum class FileFormat
+    {
+        Json,
+        Toml
+    };
+
+    explicit JSONIOHandlerImpl(
+        AbstractIOHandler *,
+        openPMD::json::TracingJSON config,
+        FileFormat,
+        std::string originalExtension);
 
     ~JSONIOHandlerImpl() override;
 
@@ -229,15 +241,25 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl
     // files that have logically, but not physically been written to
     std::unordered_set<File> m_dirty;
 
+    /*
+     * Is set by constructor.
+     */
+    FileFormat m_fileFormat{};
+
+    std::string m_originalExtension;
+
     // HELPER FUNCTIONS
 
-    // will use the IOHandler to retrieve the correct directory
-    // shared pointer to circumvent the fact that c++ pre 17 does
-    // not enforce (only allow) copy elision in return statements
-    std::shared_ptr<FILEHANDLE> getFilehandle(
-        File,
-        Access access); //, Access
-                        // m_frontendAccess=this->m_handler->m_frontendAccess);
+    // will use the IOHandler to retrieve the correct directory.
+    // first tuple element will be the underlying opened file handle.
+    // if Access is read mode, then the second tuple element will be the istream
+    // casted to precision std::numeric_limits<double>::digits10 + 1, else null.
+    // if Access is write mode, then the second tuple element will be the
+    // ostream casted to precision std::numeric_limits<double>::digits10 + 1,
+    // else null. first tuple element needs to be a pointer, since the casted
+    // streams are references only.
+    std::tuple<std::unique_ptr<FILEHANDLE>, std::istream *, std::ostream *>
+    getFilehandle(File, Access access);
 
     // full operating system path of the given file
     std::string fullPath(File);
@@ -272,15 +294,13 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl
     // essentially: m_i = \prod_{j=0}^{i-1} extent_j
     static Extent getMultiplicators(Extent const &extent);
 
-    static nlohmann::json initializeNDArray(Extent const &extent);
-
     static Extent getExtent(nlohmann::json &j);
 
     // remove single '/' in the beginning and end of a string
     static std::string removeSlashes(std::string);
 
     template <typename KeyT>
-    static bool hasKey(nlohmann::json &, KeyT &&key);
+    static bool hasKey(nlohmann::json const &, KeyT &&key);
 
     // make sure that the given path exists in proper form in
     // the passed json value
@@ -366,7 +386,8 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl
     struct AttributeReader
     {
         template <typename T>
-        static void call(nlohmann::json &, Parameter<Operation::READ_ATT> &);
+        static void
+        call(nlohmann::json const &, Parameter<Operation::READ_ATT> &);
 
         static constexpr char const *errorMsg = "JSON: writeAttribute";
     };

diff --git a/include/openPMD/auxiliary/TypeTraits.hpp b/include/openPMD/auxiliary/TypeTraits.hpp
@@ -24,6 +24,7 @@
 #include "openPMD/auxiliary/UniquePtr.hpp"
 
 #include <array>
+#include <complex>
 #include <cstddef> // size_t
 #include <memory>
 #include <vector>
@@ -56,6 +57,18 @@ namespace detail
         static constexpr bool value = true;
     };
 
+    template <typename>
+    struct IsComplex
+    {
+        static constexpr bool value = false;
+    };
+
+    template <typename T>
+    struct IsComplex<std::complex<T>>
+    {
+        static constexpr bool value = true;
+    };
+
     template <typename T>
     struct IsPointer
     {
@@ -114,6 +127,9 @@ using IsPointer_t = typename detail::IsPointer<T>::type;
 template <typename T>
 inline constexpr bool IsContiguousContainer_v = IsVector_v<T> || IsArray_v<T>;
 
+template <typename T>
+inline constexpr bool IsComplex_v = detail::IsComplex<T>::value;
+
 namespace
 {
     // see https://en.cppreference.com/w/cpp/language/if

diff --git a/src/Format.cpp b/src/Format.cpp
@@ -43,6 +43,8 @@ Format determineFormat(std::string const &filename)
         return Format::ADIOS2_SSC;
     if (auxiliary::ends_with(filename, ".json"))
         return Format::JSON;
+    if (auxiliary::ends_with(filename, ".toml"))
+        return Format::TOML;
 
     // Format might still be specified via JSON
     return Format::DUMMY;
@@ -66,6 +68,8 @@ std::string suffix(Format f)
         return ".ssc";
     case Format::JSON:
         return ".json";
+    case Format::TOML:
+        return ".toml";
     default:
         return "";
     }

diff --git a/src/IO/AbstractIOHandlerHelper.cpp b/src/IO/AbstractIOHandlerHelper.cpp
@@ -194,7 +194,20 @@ std::unique_ptr<AbstractIOHandler> createIOHandler<json::TracingJSON>(
             std::move(originalExtension));
     case Format::JSON:
         return constructIOHandler<JSONIOHandler, openPMD_HAVE_JSON>(
-            "JSON", path, access);
+            "JSON",
+            path,
+            access,
+            std::move(options),
+            JSONIOHandlerImpl::FileFormat::Json,
+            std::move(originalExtension));
+    case Format::TOML:
+        return constructIOHandler<JSONIOHandler, openPMD_HAVE_JSON>(
+            "JSON",
+            path,
+            access,
+            std::move(options),
+            JSONIOHandlerImpl::FileFormat::Toml,
+            std::move(originalExtension));
     default:
         throw std::runtime_error(
             "Unknown file format! Did you specify a file ending? Specified "

diff --git a/src/IO/JSON/JSONIOHandler.cpp b/src/IO/JSON/JSONIOHandler.cpp
@@ -25,8 +25,14 @@ namespace openPMD
 {
 JSONIOHandler::~JSONIOHandler() = default;
 
-JSONIOHandler::JSONIOHandler(std::string path, Access at)
-    : AbstractIOHandler{path, at}, m_impl{JSONIOHandlerImpl{this}}
+JSONIOHandler::JSONIOHandler(
+    std::string path,
+    Access at,
+    openPMD::json::TracingJSON jsonCfg,
+    JSONIOHandlerImpl::FileFormat format,
+    std::string originalExtension)
+    : AbstractIOHandler{path, at}
+    , m_impl{this, std::move(jsonCfg), format, std::move(originalExtension)}
 {}
 
 std::future<void> JSONIOHandler::flush(internal::ParsedFlushParams &)