Add JSONValues container for holding Python values as JSON objects if…

… possible, and as pybind11::object otherwise (#455)

* Add an optional `unserializable_handler_fn` callback to `mrc::pymrc::cast_from_pyobject` which will be invoked for any unsupported Python object. Allowing for serializing unsupported object.
ex:
```cpp
    pymrc::unserializable_handler_fn_t handler_fn = [](const py::object& source,
                                                       const std::string& path) {
        return nlohmann::json(py::cast<float>(source));
    };

    // decimal.Decimal is not serializable
    py::object Decimal = py::module_::import("decimal").attr("Decimal");
    py::object o       = Decimal("1.0");
    EXPECT_EQ(pymrc::cast_from_pyobject(o, handler_fn), nlohmann::json(1.0));
```

* Add `JSONValues` container class which is an immutable container for holding Python values as `nlohmann::json` objects if possible, and as `pybind11::object` otherwise. The container can be copied and moved, but the underlying `nlohmann::json` object is immutable.
* Updates `nlohmann_json` from 3.9 to 3.11 for `patch_inplace` method.
* Incorporates ideas from @drobison00's PR #417 with three key differences:

  1. Changes to the `cast_from_pyobject` are opt-in when `unserializable_handler_fn` is provided, otherwise there is no behavior change to the method.
  2. Unserializable objects are stored in `JSONValues` rather than a global cache.
  3. Does not rely on parsing the place-holder values.

This PR is related to nv-morpheus/Morpheus#1560

Authors:
  - David Gardner (https://github.com/dagardner-nv)
  - Devin Robison (https://github.com/drobison00)

Approvers:
  - Michael Demoret (https://github.com/mdemoret-nv)

URL: #455

ci/conda/recipes/libmrc/meta.yaml

@@ -58,7 +58,7 @@ requirements:
     - gtest =1.14
     - libhwloc =2.9.2
     - librmm {{ rapids_version }}
-    - nlohmann_json =3.9
+    - nlohmann_json =3.11
     - pybind11-abi # See: https://conda-forge.org/docs/maintainer/knowledge_base.html#pybind11-abi-constraints
     - pybind11-stubgen =0.10
     - python {{ python }}
@@ -90,12 +90,12 @@ outputs:
         - libgrpc =1.59
         - libhwloc =2.9.2
         - librmm {{ rapids_version }}
-        - nlohmann_json =3.9
+        - nlohmann_json =3.11
         - ucx =1.15
       run:
         # Manually add any packages necessary for run that do not have run_exports. Keep sorted!
         - cuda-version {{ cuda_version }}.*
-        - nlohmann_json =3.9
+        - nlohmann_json =3.11
         - ucx =1.15
         - cuda-cudart
         - boost-cpp =1.84

ci/iwyu/mappings.imp

@@ -11,6 +11,7 @@
 
 # boost
 { "include": ["@<boost/fiber/future/detail/.*>", "private", "<boost/fiber/future/future.hpp>", "public"] },
+{ "include": ["@<boost/algorithm/string/detail/.*>", "private", "<boost/algorithm/string.hpp>",  "public"] },
 
 # cuda
 { "include": ["<cuda_runtime_api.h>", "private", "<cuda_runtime.h>", "public"] },
@@ -33,6 +34,7 @@
 { "symbol": ["@grpc::.*", "private", "<grpcpp/grpcpp.h>", "public"] },
 
 # nlohmann json
+{ "include": ["<nlohmann/json_fwd.hpp>", "public", "<nlohmann/json.hpp>", "public"] },
 { "include": ["<nlohmann/detail/iterators/iter_impl.hpp>", "private", "<nlohmann/json.hpp>", "public"] },
 { "include": ["<nlohmann/detail/iterators/iteration_proxy.hpp>", "private", "<nlohmann/json.hpp>", "public"] },
 { "include": ["<nlohmann/detail/json_ref.hpp>", "private", "<nlohmann/json.hpp>", "public"] },

conda/environments/all_cuda-121_arch-x86_64.yaml

@@ -39,7 +39,7 @@ dependencies:
 - libxml2=2.11.6
 - llvmdev=16
 - ninja=1.11
-- nlohmann_json=3.9
+- nlohmann_json=3.11
 - numactl-libs-cos7-x86_64
 - numpy=1.24
 - pkg-config=0.29

conda/environments/ci_cuda-121_arch-x86_64.yaml

@@ -29,7 +29,7 @@ dependencies:
 - librmm=24.02
 - libxml2=2.11.6
 - ninja=1.11
-- nlohmann_json=3.9
+- nlohmann_json=3.11
 - numactl-libs-cos7-x86_64
 - pkg-config=0.29
 - pre-commit

dependencies.yaml

@@ -63,7 +63,7 @@ dependencies:
           - librmm=24.02
           - libxml2=2.11.6 # 2.12 has a bug preventing round-trip serialization in hwloc
           - ninja=1.11
-          - nlohmann_json=3.9
+          - nlohmann_json=3.11
           - numactl-libs-cos7-x86_64
           - pkg-config=0.29
           - pybind11-stubgen=0.10

python/mrc/_pymrc/CMakeLists.txt

@@ -37,6 +37,7 @@ add_library(pymrc
   src/utilities/acquire_gil.cpp
   src/utilities/deserializers.cpp
   src/utilities/function_wrappers.cpp
+  src/utilities/json_values.cpp
   src/utilities/object_cache.cpp
   src/utilities/object_wrappers.cpp
   src/utilities/serializers.cpp

python/mrc/_pymrc/include/pymrc/types.hpp

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: Copyright (c) 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-FileCopyrightText: Copyright (c) 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
  * SPDX-License-Identifier: Apache-2.0
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
@@ -21,9 +21,12 @@
 
 #include "mrc/segment/object.hpp"
 
+#include <nlohmann/json_fwd.hpp>
 #include <rxcpp/rx.hpp>
 
-#include <functional>
+#include <functional>  // for function
+#include <map>
+#include <string>
 
 namespace mrc::pymrc {
 
@@ -37,4 +40,16 @@ using PyNode             = mrc::segment::ObjectProperties;
 using PyObjectOperateFn  = std::function<PyObjectObservable(PyObjectObservable source)>;
 // NOLINTEND(readability-identifier-naming)
 
+using python_map_t = std::map<std::string, pybind11::object>;
+
+/**
+ * @brief Unserializable handler function type, invoked by `cast_from_pyobject` when an object cannot be serialized to
+ * JSON. Implementations should return a valid json object, or throw an exception if the object cannot be serialized.
+ * @param source : pybind11 object
+ * @param path : string json path to object
+ * @return nlohmann::json.
+ */
+using unserializable_handler_fn_t =
+    std::function<nlohmann::json(const pybind11::object& /* source*/, const std::string& /* path */)>;
+
 }  // namespace mrc::pymrc

python/mrc/_pymrc/include/pymrc/utilities/json_values.hpp

@@ -0,0 +1,157 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "pymrc/types.hpp"  // for python_map_t & unserializable_handler_fn_t
+
+#include <nlohmann/json.hpp>
+#include <pybind11/pytypes.h>  // for PYBIND11_EXPORT & pybind11::object
+
+#include <cstddef>  // for size_t
+#include <string>
+// IWYU wants us to use the pybind11.h for the PYBIND11_EXPORT macro, but we already have it in pytypes.h
+// IWYU pragma: no_include <pybind11/pybind11.h>
+
+namespace mrc::pymrc {
+
+#pragma GCC visibility push(default)
+
+/**
+ * @brief Immutable container for holding Python values as JSON objects if possible, and as pybind11::object otherwise.
+ * The container can be copied and moved, but the underlying JSON object is immutable.
+ **/
+class PYBIND11_EXPORT JSONValues
+{
+  public:
+    JSONValues();
+    JSONValues(pybind11::object values);
+    JSONValues(nlohmann::json values);
+
+    JSONValues(const JSONValues& other) = default;
+    JSONValues(JSONValues&& other)      = default;
+    ~JSONValues()                       = default;
+
+    JSONValues& operator=(const JSONValues& other) = default;
+    JSONValues& operator=(JSONValues&& other)      = default;
+
+    /**
+     * @brief Sets a value in the JSON object at the specified path with the provided Python object. If `value` is
+     * serializable as JSON it will be stored as JSON, otherwise it will be stored as-is.
+     * @param path The path in the JSON object where the value should be set.
+     * @param value The Python object to set.
+     * @throws std::runtime_error If the path is invalid.
+     * @return A new JSONValues object with the updated value.
+     */
+    JSONValues set_value(const std::string& path, const pybind11::object& value) const;
+
+    /**
+     * @brief Sets a value in the JSON object at the specified path with the provided JSON object.
+     * @param path The path in the JSON object where the value should be set.
+     * @param value The JSON object to set.
+     * @throws std::runtime_error If the path is invalid.
+     * @return A new JSONValues object with the updated value.
+     */
+    JSONValues set_value(const std::string& path, nlohmann::json value) const;
+
+    /**
+     * @brief Sets a value in the JSON object at the specified path with the provided JSONValues object.
+     * @param path The path in the JSON object where the value should be set.
+     * @param value The JSONValues object to set.
+     * @throws std::runtime_error If the path is invalid.
+     * @return A new JSONValues object with the updated value.
+     */
+    JSONValues set_value(const std::string& path, const JSONValues& value) const;
+
+    /**
+     * @brief Returns the number of unserializable Python objects.
+     * @return The number of unserializable Python objects.
+     */
+    std::size_t num_unserializable() const;
+
+    /**
+     * @brief Checks if there are any unserializable Python objects.
+     * @return True if there are unserializable Python objects, false otherwise.
+     */
+    bool has_unserializable() const;
+
+    /**
+     * @brief Convert to a Python object.
+     * @return The Python object representation of the values.
+     */
+    pybind11::object to_python() const;
+
+    /**
+     * @brief Returns a constant reference to the underlying JSON object. Any unserializable Python objects, will be
+     * represented in the JSON object with a string place-holder with the value `"**pymrc_placeholder"`.
+     * @return A constant reference to the JSON object.
+     */
+    nlohmann::json::const_reference view_json() const;
+
+    /**
+     * @brief Converts the JSON object to a JSON object. If any unserializable Python objects are present, the
+     * `unserializable_handler_fn` will be invoked to handle the object.
+     * @param unserializable_handler_fn Optional function to handle unserializable objects.
+     * @return The JSON string representation of the JSON object.
+     */
+    nlohmann::json to_json(unserializable_handler_fn_t unserializable_handler_fn) const;
+
+    /**
+     * @brief Converts a Python object to a JSON string. Convienence function that matches the
+     * `unserializable_handler_fn_t` signature. Convienent for use with `to_json` and `get_json`.
+     * @param obj The Python object to convert.
+     * @param path The path in the JSON object where the value should be set.
+     * @return The JSON string representation of the Python object.
+     */
+    static nlohmann::json stringify(const pybind11::object& obj, const std::string& path);
+
+    /**
+     * @brief Returns the object at the specified path as a Python object.
+     * @param path Path to the specified object.
+     * @throws std::runtime_error If the path does not exist or is not a valid path.
+     * @return Python representation of the object at the specified path.
+     */
+    pybind11::object get_python(const std::string& path) const;
+
+    /**
+     * @brief Returns the object at the specified path. If the object is an unserializable Python object the
+     * `unserializable_handler_fn` will be invoked.
+     * @param path Path to the specified object.
+     * @param unserializable_handler_fn Function to handle unserializable objects.
+     * @throws std::runtime_error If the path does not exist or is not a valid path.
+     * @return The JSON object at the specified path.
+     */
+    nlohmann::json get_json(const std::string& path, unserializable_handler_fn_t unserializable_handler_fn) const;
+
+    /**
+     * @brief Return a new JSONValues object with the value at the specified path.
+     * @param path Path to the specified object.
+     * @throws std::runtime_error If the path does not exist or is not a valid path.
+     * @return The value at the specified path.
+     */
+    JSONValues operator[](const std::string& path) const;
+
+  private:
+    JSONValues(nlohmann::json&& values, python_map_t&& py_objects);
+    nlohmann::json unserializable_handler(const pybind11::object& obj, const std::string& path);
+
+    nlohmann::json m_serialized_values;
+    python_map_t m_py_objects;
+};
+
+#pragma GCC visibility pop
+}  // namespace mrc::pymrc

python/mrc/_pymrc/include/pymrc/utilities/object_cache.hpp

@@ -17,6 +17,8 @@
 
 #pragma once
 
+#include "pymrc/types.hpp"
+
 #include <pybind11/pytypes.h>
 
 #include <cstddef>
@@ -95,7 +97,7 @@ class __attribute__((visibility("default"))) PythonObjectCache
      */
     void atexit_callback();
 
-    std::map<std::string, pybind11::object> m_object_cache;
+    python_map_t m_object_cache;
 };
 
 #pragma GCC visibility pop

python/mrc/_pymrc/include/pymrc/utils.hpp

@@ -17,6 +17,8 @@
 
 #pragma once
 
+#include "pymrc/types.hpp"
+
 #include <nlohmann/json_fwd.hpp>
 #include <pybind11/pybind11.h>
 #include <pybind11/pytypes.h>
@@ -31,8 +33,25 @@ namespace mrc::pymrc {
 #pragma GCC visibility push(default)
 
 pybind11::object cast_from_json(const nlohmann::json& source);
+
+/**
+ * @brief Convert a pybind11 object to a JSON object. If the object cannot be serialized, a pybind11::type_error
+ * exception be thrown.
+ * @param source : pybind11 object
+ * @return nlohmann::json.
+ */
 nlohmann::json cast_from_pyobject(const pybind11::object& source);
 
+/**
+ * @brief Convert a pybind11 object to a JSON object. If the object cannot be serialized, the unserializable_handler_fn
+ * will be invoked to handle the object.
+ * @param source : pybind11 object
+ * @param unserializable_handler_fn : unserializable_handler_fn_t
+ * @return nlohmann::json.
+ */
+nlohmann::json cast_from_pyobject(const pybind11::object& source,
+                                  unserializable_handler_fn_t unserializable_handler_fn);
+
 void import_module_object(pybind11::module_&, const std::string&, const std::string&);
 void import_module_object(pybind11::module_& dest, const pybind11::module_& mod);