Python support using converter nodes

phlex/app/phlex.cpp

@@ -92,5 +92,13 @@
   if (not vm["parallel"].defaulted()) {
     max_concurrency = vm["parallel"].as<int>();
   }
-  phlex::experimental::run(configurations, max_concurrency);
+  try {
+    phlex::experimental::run(configurations, max_concurrency);
+  } catch (std::exception const& e) {
+    std::cerr << e.what() << '\n';
+    return 1;
+  } catch (...) {
+    std::cerr << "Unknown exception caught.\n";
+    return 1;
+  }
 }

phlex/configuration.hpp

@@ -6,6 +6,7 @@
 
 #include <optional>
 #include <string>
+#include <utility>
 #include <vector>
 
 namespace phlex::experimental {
@@ -48,6 +49,25 @@ namespace phlex::experimental {
 
     std::vector<std::string> keys() const;
 
+    // Internal function for prototype purposes; do not use as this will change.
+    std::pair<boost::json::kind, bool> prototype_internal_kind(std::string const& key) const
+    {
+      auto const& value = config_.at(key); // may throw
+
+      auto k = value.kind();
+      bool is_array = k == boost::json::kind::array;
+
+      if (is_array) {
+        // The current configuration interface only supports homogenous containers,
+        // thus checking only the first element suffices. (This assumes arrays are
+        // not nested, which is fine for now.)
+        boost::json::array const& arr = value.as_array();
+        k = arr.empty() ? boost::json::kind::null : arr[0].kind();
+      }
+
+      return std::make_pair(k, is_array);
+    }
+
   private:
     boost::json::object config_;
   };

plugins/CMakeLists.txt

@@ -1,3 +1,8 @@
+# Phlex provided core plugins
+
+# plugin for running Python algorithms in phlex
+add_subdirectory(python)
+
 add_library(layer_generator layer_generator.cpp)
 target_link_libraries(layer_generator PRIVATE phlex::core)
 

plugins/python/CMakeLists.txt

@@ -0,0 +1,94 @@
+find_package(
+  Python 3.12
+  COMPONENTS Interpreter Development
+  QUIET
+  )
+
+if(Python_FOUND)
+
+  # Verify installation of necessary python modules for specific tests
+
+  function(check_python_module_version MODULE_NAME MIN_VERSION OUT_VAR)
+    execute_process(
+      COMMAND
+        ${Python_EXECUTABLE} -c "import sys
+try:
+    import ${MODULE_NAME}
+    from packaging.version import parse as parse_version
+    installed_version = getattr(${MODULE_NAME}, '__version__', None)
+    if parse_version(installed_version) >= parse_version('${MIN_VERSION}'):
+        sys.exit(0)
+    else:
+        sys.exit(2)  # Version too low
+except ImportError:
+    sys.exit(1)"
+      RESULT_VARIABLE _module_check_result
+      )
+
+    if(_module_check_result EQUAL 0)
+      set(${OUT_VAR}
+          TRUE
+          PARENT_SCOPE
+          )
+    elseif(_module_check_result EQUAL 1)
+      set(${OUT_VAR}
+          FALSE
+          PARENT_SCOPE
+          ) # silent b/c common
+    elseif(_module_check_result EQUAL 2)
+      message(
+        WARNING
+          "Python module '${MODULE_NAME}' found but version too low (min required: ${MIN_VERSION})."
+        )
+      set(${OUT_VAR}
+          FALSE
+          PARENT_SCOPE
+          )
+    else()
+      message(
+        WARNING "Unknown error while checking Python module '${MODULE_NAME}'."
+        )
+      set(${OUT_VAR}
+          FALSE
+          PARENT_SCOPE
+          )
+    endif()
+  endfunction()
+
+  check_python_module_version("numpy" "2.0.0" HAS_NUMPY)
+
+  # Phlex module to run Python algorithms
+  add_library(
+    pymodule MODULE src/pymodule.cpp src/modulewrap.cpp src/configwrap.cpp
+                    src/lifelinewrap.cpp src/errorwrap.cpp
+    )
+  include_directories(pymodule, ${Python_INCLUDE_DIRS})
+  target_link_libraries(
+    pymodule
+    PRIVATE phlex::module ${Python_LIBRARIES}
+    PUBLIC Python::Python
+    )
+
+  # numpy support if installed
+  if(HAS_NUMPY)
+
+    # locate numpy's header directory
+    execute_process(
+      COMMAND "${Python_EXECUTABLE}" -c
+              "import numpy; print(numpy.get_include(), end='')"
+      RESULT_VARIABLE NUMPY_RESULT
+      OUTPUT_VARIABLE NUMPY_INCLUDE_DIR
+      OUTPUT_STRIP_TRAILING_WHITESPACE
+      )
+
+    if(NUMPY_RESULT EQUAL 0)
+      include_directories(pymodule PRIVATE ${NUMPY_INCLUDE_DIR})
+      target_compile_definitions(
+        pymodule PRIVATE PHLEX_HAVE_NUMPY=1
+                         NPY_NO_DEPRECATED_API=NPY_1_7_API_VERSION
+        )
+    endif()
+
+  endif()
+
+endif() # Python available

plugins/python/src/configwrap.cpp

@@ -0,0 +1,213 @@
+#include <cstdint>
+#include <string>
+
+#include "phlex/configuration.hpp"
+#include "wrap.hpp"
+
+using namespace phlex::experimental;
+
+// Create a dict-like access to the configuration from Python.
+// clang-format off
+struct phlex::experimental::py_config_map {
+  PyObject_HEAD
+  phlex::experimental::configuration const* ph_config;
+  PyObject* ph_config_cache;
+};
+// clang-format on
+
+PyObject* phlex::experimental::wrap_configuration(configuration const* config)
+{
+  if (!config) {
+    PyErr_SetString(PyExc_ValueError, "provided configuration is null");
+    return nullptr;
+  }
+
+  py_config_map* pyconfig =
+    (py_config_map*)PhlexConfig_Type.tp_new(&PhlexConfig_Type, nullptr, nullptr);
+
+  pyconfig->ph_config = config;
+
+  return (PyObject*)pyconfig;
+}
+
+//= CPyCppyy low level view construction/destruction =========================
+static py_config_map* pcm_new(PyTypeObject* subtype, PyObject*, PyObject*)
+{
+  py_config_map* pcm = (py_config_map*)subtype->tp_alloc(subtype, 0);
+  if (!pcm)
+    return nullptr;
+
+  pcm->ph_config_cache = PyDict_New();
+
+  return pcm;
+}
+
+static void pcm_dealloc(py_config_map* pcm)
+{
+  Py_DECREF(pcm->ph_config_cache);
+  Py_TYPE(pcm)->tp_free((PyObject*)pcm);
+}
+
+static PyObject* pcm_subscript(py_config_map* pycmap, PyObject* pykey)
+{
+  // Retrieve a named configuration setting.
+  //
+  // Configuration should have a single in-memory representation, which is why
+  // the current approach retrieves it from the equivalent C++ object, ie. after
+  // the JSON input has been parsed, even as there are Python JSON parsers.
+  //
+  // pykey: the lookup key to retrieve the configuration value
+
+  if (!PyUnicode_Check(pykey)) {
+    PyErr_SetString(PyExc_TypeError, "__getitem__ expects a string key");
+    return nullptr;
+  }
+
+  // cached lookup
+  PyObject* pyvalue = PyDict_GetItem(pycmap->ph_config_cache, pykey);
+  if (pyvalue) {
+    Py_INCREF(pyvalue);
+    return pyvalue;
+  }
+  PyErr_Clear();
+
+  std::string ckey = PyUnicode_AsUTF8(pykey);
+
+  try {
+    auto k = pycmap->ph_config->prototype_internal_kind(ckey);
+    if (k.second /* is array */) {
+      if (k.first == boost::json::kind::bool_) {
+        auto const& cvalue = pycmap->ph_config->get<std::vector<bool>>(ckey);
+        pyvalue = PyTuple_New(cvalue.size());
+        for (Py_ssize_t i = 0; i < (Py_ssize_t)cvalue.size(); ++i) {
+          PyObject* item = PyLong_FromLong((long)cvalue[i]);
+          PyTuple_SetItem(pyvalue, i, item);
+        }
+      } else if (k.first == boost::json::kind::int64) {
+        auto const& cvalue = pycmap->ph_config->get<std::vector<std::int64_t>>(ckey);
+        pyvalue = PyTuple_New(cvalue.size());
+        for (Py_ssize_t i = 0; i < (Py_ssize_t)cvalue.size(); ++i) {
+          PyObject* item = PyLong_FromLong(cvalue[i]);
+          PyTuple_SetItem(pyvalue, i, item);
+        }
+      } else if (k.first == boost::json::kind::uint64) {
+        auto const& cvalue = pycmap->ph_config->get<std::vector<std::uint64_t>>(ckey);
+        pyvalue = PyTuple_New(cvalue.size());
+        for (Py_ssize_t i = 0; i < (Py_ssize_t)cvalue.size(); ++i) {
+          PyObject* item = PyLong_FromUnsignedLong(cvalue[i]);
+          PyTuple_SetItem(pyvalue, i, item);
+        }
+      } else if (k.first == boost::json::kind::double_) {
+        auto const& cvalue = pycmap->ph_config->get<std::vector<double>>(ckey);
+        pyvalue = PyTuple_New(cvalue.size());
+        for (Py_ssize_t i = 0; i < (Py_ssize_t)cvalue.size(); ++i) {
+          PyObject* item = PyFloat_FromDouble(cvalue[i]);
+          PyTuple_SetItem(pyvalue, i, item);
+        }
+      } else if (k.first == boost::json::kind::string) {
+        auto const& cvalue = pycmap->ph_config->get<std::vector<std::string>>(ckey);
+        pyvalue = PyTuple_New(cvalue.size());
+        for (Py_ssize_t i = 0; i < (Py_ssize_t)cvalue.size(); ++i) {
+          PyObject* item = PyUnicode_FromStringAndSize(cvalue[i].c_str(), cvalue[i].size());
+          PyTuple_SetItem(pyvalue, i, item);
+        }
+      }
+    } else {
+      if (k.first == boost::json::kind::bool_) {
+        auto cvalue = pycmap->ph_config->get<bool>(ckey);
+        pyvalue = PyBool_FromLong((long)cvalue);
+      } else if (k.first == boost::json::kind::int64) {
+        auto cvalue = pycmap->ph_config->get<std::int64_t>(ckey);
+        pyvalue = PyLong_FromLong(cvalue);
+      } else if (k.first == boost::json::kind::uint64) {
+        auto cvalue = pycmap->ph_config->get<std::uint64_t>(ckey);
+        pyvalue = PyLong_FromUnsignedLong(cvalue);
+      } else if (k.first == boost::json::kind::double_) {
+        auto cvalue = pycmap->ph_config->get<double>(ckey);
+        pyvalue = PyFloat_FromDouble(cvalue);
+      } else if (k.first == boost::json::kind::string) {
+        auto const& cvalue = pycmap->ph_config->get<std::string>(ckey);
+        pyvalue = PyUnicode_FromStringAndSize(cvalue.c_str(), cvalue.size());
+      }
+    }
+  } catch (std::runtime_error const&) {
+    PyErr_Format(PyExc_TypeError, "property \"%s\" does not exist", ckey.c_str());
+  }
+
+  // cache if found
+  if (pyvalue) {
+    PyDict_SetItem(pycmap->ph_config_cache, pykey, pyvalue);
+  }
+
+  return pyvalue;
+}
+
+static PyMappingMethods pcm_as_mapping = {nullptr, (binaryfunc)pcm_subscript, nullptr};
+
+// clang-format off
+PyTypeObject phlex::experimental::PhlexConfig_Type = {
+  PyVarObject_HEAD_INIT(&PyType_Type, 0)
+  (char*) "pyphlex.configuration",                   // tp_name
+  sizeof(py_config_map),                             // tp_basicsize
+  0,                                                 // tp_itemsize
+  (destructor)pcm_dealloc,                           // tp_dealloc
+  0,                                                 // tp_vectorcall_offset / tp_print
+  0,                                                 // tp_getattr
+  0,                                                 // tp_setattr
+  0,                                                 // tp_as_async / tp_compare
+  0,                                                 // tp_repr
+  0,                                                 // tp_as_number
+  0,                                                 // tp_as_sequence
+  &pcm_as_mapping,                                   // tp_as_mapping
+  0,                                                 // tp_hash
+  0,                                                 // tp_call
+  0,                                                 // tp_str
+  0,                                                 // tp_getattro
+  0,                                                 // tp_setattro
+  0,                                                 // tp_as_buffer
+  Py_TPFLAGS_DEFAULT,                                // tp_flags
+  (char*)"phlex configuration object-as-dictionary", // tp_doc
+  0,                                                 // tp_traverse
+  0,                                                 // tp_clear
+  0,                                                 // tp_richcompare
+  0,                                                 // tp_weaklistoffset
+  0,                                                 // tp_iter
+  0,                                                 // tp_iternext
+  0,                                                 // tp_methods
+  0,                                                 // tp_members
+  0,                                                 // tp_getset
+  0,                                                 // tp_base
+  0,                                                 // tp_dict
+  0,                                                 // tp_descr_get
+  0,                                                 // tp_descr_set
+  offsetof(py_config_map, ph_config_cache),          // tp_dictoffset
+  0,                                                 // tp_init
+  0,                                                 // tp_alloc
+  (newfunc)pcm_new,                                  // tp_new
+  0,                                                 // tp_free
+  0,                                                 // tp_is_gc
+  0,                                                 // tp_bases
+  0,                                                 // tp_mro
+  0,                                                 // tp_cache
+  0,                                                 // tp_subclasses
+  0                                                  // tp_weaklist
+#if PY_VERSION_HEX >= 0x02030000
+  , 0                                                // tp_del
+#endif
+#if PY_VERSION_HEX >= 0x02060000
+  , 0                                                // tp_version_tag
+#endif
+#if PY_VERSION_HEX >= 0x03040000
+  , 0                                                // tp_finalize
+#endif
+#if PY_VERSION_HEX >= 0x03080000
+  , 0                                                // tp_vectorcall
+#endif
+#if PY_VERSION_HEX >= 0x030c0000
+  , 0                                                // tp_watched
+#endif
+#if PY_VERSION_HEX >= 0x030d0000
+  , 0                                                // tp_versions_used
+#endif
+};
+// clang-format on