Compatibility with toml11 4.0 (#1645)

* Support toml11 4.0 in CMake * Fixes for toml11 4.Fixes for toml11 4.00 Still missing: proper setting for precision, backwards compatibility with toml11 3.0. Ideally, move toml::format to an internal helper that takes care of all of this. * Manually set precisions... * Compatibility with toml11 3.* * Add a little comment in CMake * Add a toml11 4.0.3 CI run * tmp * Use two different find_package() calls * More concise build script for toml11 Co-authored-by: Axel Huebl <[email protected]> * Some cleanup, fixes and documentation in the find_package() logic --------- Co-authored-by: Axel Huebl <[email protected]>
openPMD · Jul 19, 2024 · 0358d09 · 0358d09
1 parent 0fa1803
commit 0358d09
Show file tree

Hide file tree

Showing 11 changed files with 86 additions and 17 deletions.
diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml
@@ -133,6 +133,12 @@ jobs:
         python3.11 -m venv /opt/python_venv
         . /opt/python_venv/bin/activate
         python -m pip install numpy pandas
+        git clone -b v4.0.3 https://github.com/ToruNiina/toml11
+        cmake -S toml11 -B build_toml11               \
+          -DCMAKE_INSTALL_PREFIX=toml11_install \
+          -DCMAKE_CXX_STANDARD_REQUIRED=OFF           \
+          -DCMAKE_CXX_STANDARD=11
+        cmake --build build_toml11 -j 2 --target install
     - name: Build
       env: {CC: clang-14, CXX: clang++-14, CXXFLAGS: -Werror}
       run: |
@@ -144,13 +150,15 @@ jobs:
         . /opt/python_venv/bin/activate
 
         share/openPMD/download_samples.sh build
+        export CMAKE_PREFIX_PATH="$(realpath toml11_install):$CMAKE_PREFIX_PATH"
         cmake -S . -B build \
           -DopenPMD_USE_PYTHON=ON \
           -DopenPMD_USE_MPI=OFF   \
           -DopenPMD_USE_HDF5=ON   \
           -DopenPMD_USE_ADIOS2=ON \
-          -DopenPMD_USE_INVASIVE_TESTS=ON \
-          -DCMAKE_VERBOSE_MAKEFILE=ON     \
+          -DopenPMD_USE_INVASIVE_TESTS=ON   \
+          -DopenPMD_USE_INTERNAL_TOML11=OFF \
+          -DCMAKE_VERBOSE_MAKEFILE=ON       \
           -DPython_EXECUTABLE="$(which python)"
         cmake --build build --parallel 2
         ctest --test-dir build --output-on-failure

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -278,7 +278,15 @@ if(openPMD_USE_INTERNAL_TOML11)
     add_subdirectory("${openPMD_SOURCE_DIR}/share/openPMD/thirdParty/toml11")
     message(STATUS "toml11: Using INTERNAL version '3.7.1'")
 else()
-    find_package(toml11 3.7.1 CONFIG REQUIRED)
+    # toml11 4.0 was a breaking change. This is reflected in the library's CMake
+    # logic: version 4.0 is not accepted by a call to find_package(toml11 3.7).
+    # Since we support both incompatible versions, we use two find_package()
+    # calls. Search for version 4 first in order to prefer that
+    # in (the unlikely) case that both versions are installed.
+    find_package(toml11 4.0 CONFIG QUIET)
+    if(NOT toml11_FOUND)
+        find_package(toml11 3.7.1 CONFIG REQUIRED)
+    endif()
     message(STATUS "toml11: Found version '${toml11_VERSION}'")
 endif()
 add_library(openPMD::thirdparty::toml11 INTERFACE IMPORTED)

diff --git a/include/openPMD/auxiliary/JSON_internal.hpp b/include/openPMD/auxiliary/JSON_internal.hpp
@@ -251,5 +251,8 @@ namespace json
 
     nlohmann::json &filterByTemplate(
         nlohmann::json &defaultVal, nlohmann::json const &positiveMask);
+
+    template <typename toml_t>
+    std::string format_toml(toml_t &&);
 } // namespace json
 } // namespace openPMD
diff --git a/src/IO/ADIOS/ADIOS2File.cpp b/src/IO/ADIOS/ADIOS2File.cpp
@@ -565,7 +565,7 @@ void ADIOS2File::configure_IO()
             auto asToml = json::jsonToToml(shadow);
             std::cerr << "Warning: parts of the backend configuration for "
                          "ADIOS2 remain unused:\n"
-                      << asToml << std::endl;
+                      << json::format_toml(asToml) << std::endl;
             break;
         }
         }

diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp
@@ -527,7 +527,7 @@ ADIOS2IOHandlerImpl::flush(internal::ParsedFlushParams &flushParams)
                 auto asToml = json::jsonToToml(shadow);
                 std::cerr << "Warning: parts of the backend configuration for "
                              "ADIOS2 remain unused:\n"
-                          << asToml << std::endl;
+                          << json::format_toml(asToml) << std::endl;
                 break;
             }
             }

diff --git a/src/IO/HDF5/HDF5IOHandler.cpp b/src/IO/HDF5/HDF5IOHandler.cpp
@@ -174,7 +174,7 @@ HDF5IOHandlerImpl::HDF5IOHandlerImpl(
                     std::cerr
                         << "Warning: parts of the backend configuration for "
                            "HDF5 remain unused:\n"
-                        << asToml << std::endl;
+                        << json::format_toml(asToml) << std::endl;
                     break;
                 }
                 }

diff --git a/src/IO/HDF5/ParallelHDF5IOHandler.cpp b/src/IO/HDF5/ParallelHDF5IOHandler.cpp
@@ -334,7 +334,7 @@ ParallelHDF5IOHandlerImpl::ParallelHDF5IOHandlerImpl(
                 auto asToml = json::jsonToToml(shadow);
                 std::cerr << "Warning: parts of the backend configuration for "
                              "HDF5 remain unused:\n"
-                          << asToml << std::endl;
+                          << json::format_toml(asToml) << std::endl;
                 break;
             }
             }

diff --git a/src/IO/IOTask.cpp b/src/IO/IOTask.cpp
@@ -66,7 +66,8 @@ void Parameter<Operation::CREATE_DATASET>::warnUnusedParameters<
             break;
         case json::SupportedLanguages::TOML: {
             auto asToml = json::jsonToToml(shadow);
-            std::cerr << warningMessage << asToml << std::endl;
+            std::cerr << warningMessage << json::format_toml(asToml)
+                      << std::endl;
             break;
         }
         }

diff --git a/src/IO/JSON/JSONIOHandlerImpl.cpp b/src/IO/JSON/JSONIOHandlerImpl.cpp
@@ -1378,7 +1378,8 @@ auto JSONIOHandlerImpl::putJsonContents(
             *fh_with_precision << *it->second << std::endl;
             break;
         case FileFormat::Toml:
-            *fh_with_precision << openPMD::json::jsonToToml(*it->second)
+            *fh_with_precision << openPMD::json::format_toml(
+                                      openPMD::json::jsonToToml(*it->second))
                                << std::endl;
             break;
         }

diff --git a/src/auxiliary/JSON.cpp b/src/auxiliary/JSON.cpp
@@ -25,8 +25,9 @@
 #include "openPMD/Error.hpp"
 #include "openPMD/auxiliary/Filesystem.hpp"
 #include "openPMD/auxiliary/StringManip.hpp"
+#include "openPMD/auxiliary/Variant.hpp"
 
-#include <nlohmann/json.hpp>
+#include <limits>
 #include <queue>
 #include <toml.hpp>
 
@@ -208,10 +209,6 @@ namespace
             }
             return result;
         }
-        else if (val.is_uninitialized())
-        {
-            return nlohmann::json(); // null
-        }
 
         // @todo maybe generalize error type
         throw error::BackendConfigSchema(
@@ -556,7 +553,7 @@ void warnGlobalUnusedOptions(TracingJSON const &config)
             std::cerr
                 << "[Series] The following parts of the global TOML config "
                    "remains unused:\n"
-                << asToml << std::endl;
+                << json::format_toml(asToml) << std::endl;
         }
         }
     }
@@ -612,7 +609,7 @@ std::string merge(std::string const &defaultValue, std::string const &overwrite)
     case SupportedLanguages::TOML: {
         auto asToml = json::jsonToToml(res);
         std::stringstream sstream;
-        sstream << asToml;
+        sstream << json::format_toml(asToml);
         return sstream.str();
     }
     }
@@ -646,4 +643,55 @@ filterByTemplate(nlohmann::json &defaultVal, nlohmann::json const &positiveMask)
     } // else noop
     return defaultVal;
 }
+
+constexpr int toml_precision = std::numeric_limits<double>::digits10 + 1;
+
+#if TOML11_VERSION_MAJOR < 4
+template <typename toml_t>
+std ::string format_toml(toml_t &&val)
+{
+    std::stringstream res;
+    res << std::setprecision(toml_precision) << std::forward<toml_t>(val);
+    return res.str();
+}
+
+#else
+
+namespace
+{
+    auto set_precision(toml::value &) -> void;
+    auto set_precision(toml::value &val) -> void
+    {
+        if (val.is_table())
+        {
+            for (auto &pair : val.as_table())
+            {
+                set_precision(pair.second);
+            }
+        }
+        else if (val.is_array())
+        {
+            for (auto &entry : val.as_array())
+            {
+                set_precision(entry);
+            }
+        }
+        else if (val.is_floating())
+        {
+            val.as_floating_fmt().prec = toml_precision;
+        }
+    }
+} // namespace
+
+template <typename toml_t>
+std::string format_toml(toml_t &&val)
+{
+    set_precision(val);
+    return toml::format(std::forward<toml_t>(val));
+}
+
+#endif
+
+template std::string format_toml(toml::value &&);
+template std::string format_toml(toml::value &);
 } // namespace openPMD::json
diff --git a/test/JSONTest.cpp b/test/JSONTest.cpp
@@ -210,7 +210,7 @@ right = "val"
                 raw, std::ios_base::binary | std::ios_base::in);
             toml::value tomlVal = toml::parse(istream);
             std::stringstream sstream;
-            sstream << tomlVal;
+            sstream << toml::format(tomlVal);
             return sort_lines(sstream.str());
         }();