diff --git a/phlex/app/load_module.cpp b/phlex/app/load_module.cpp index a5e51bc0..e7703474 100644 --- a/phlex/app/load_module.cpp +++ b/phlex/app/load_module.cpp @@ -95,16 +95,18 @@ namespace phlex::experimental { void load_source(framework_graph& g, std::string const& label, boost::json::object raw_config) { - auto const& spec = value_to(raw_config.at("cpp")); + auto const adjusted_config = detail::adjust_config(label, std::move(raw_config)); + + auto const& spec = value_to(adjusted_config.at("cpp")); auto& creator = create_source.emplace_back(plugin_loader(spec, "create_source")); // FIXME: Should probably use the parameter name (e.g.) 'plugin_label' instead of // 'module_label', but that requires adjusting other parts of the system // (e.g. make_algorithm_name). - raw_config["module_label"] = label; + // adjusted_config["module_label"] = label; // already set by adjust_config - configuration const config{raw_config}; + configuration const config{adjusted_config}; creator(g.source_proxy(config), config); } diff --git a/plugins/python/CMakeLists.txt b/plugins/python/CMakeLists.txt index 047869d0..d2113176 100644 --- a/plugins/python/CMakeLists.txt +++ b/plugins/python/CMakeLists.txt @@ -14,6 +14,7 @@ add_library( src/pymodule.cpp src/modulewrap.cpp src/configwrap.cpp + src/dciwrap.cpp src/lifelinewrap.cpp src/errorwrap.cpp ) diff --git a/plugins/python/src/configwrap.cpp b/plugins/python/src/configwrap.cpp index f141f3d8..51cd6729 100644 --- a/plugins/python/src/configwrap.cpp +++ b/plugins/python/src/configwrap.cpp @@ -25,7 +25,6 @@ PyObject* phlex::experimental::wrap_configuration(configuration const& config) return (PyObject*)pyconfig; } -//= CPyCppyy low level view construction/destruction ========================= static py_config_map* pcm_new(PyTypeObject* subtype, PyObject*, PyObject*) { py_config_map* pcm = (py_config_map*)subtype->tp_alloc(subtype, 0); diff --git a/plugins/python/src/dciwrap.cpp b/plugins/python/src/dciwrap.cpp new file mode 100644 index 00000000..6c3371e4 --- /dev/null +++ b/plugins/python/src/dciwrap.cpp @@ -0,0 +1,101 @@ +#include "phlex/model/data_cell_index.hpp" +#include "wrap.hpp" + +using namespace phlex::experimental; +using namespace phlex; + +// Provide selected (for now) access to Phlex's data_cell_index instances. +// clang-format off +namespace phlex::experimental { + struct py_data_cell_index { + PyObject_HEAD + data_cell_index const* ph_dci; + }; +} +// clang-format on + +PyObject* phlex::experimental::wrap_dci(data_cell_index const& dci) +{ + py_data_cell_index* pydci = PyObject_New(py_data_cell_index, &PhlexDataCellIndex_Type); + pydci->ph_dci = &dci; + + return (PyObject*)pydci; +} + +// simple forwarding methods +static PyObject* dci_number(py_data_cell_index* pydci) +{ + return PyLong_FromLong((long)pydci->ph_dci->number()); +} + +static PyMethodDef dci_methods[] = { + {(char*)"number", (PyCFunction)dci_number, METH_NOARGS, (char*)"index number"}, + {(char*)nullptr, nullptr, 0, nullptr}}; + +// clang-format off +PyTypeObject phlex::experimental::PhlexDataCellIndex_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + (char*) "pyphlex.data_cell_index", // tp_name + sizeof(py_data_cell_index), // tp_basicsize + 0, // tp_itemsize + 0, // tp_dealloc + 0, // tp_vectorcall_offset / tp_print + 0, // tp_getattr + 0, // tp_setattr + 0, // tp_as_async / tp_compare + 0, // tp_repr + 0, // tp_as_number + 0, // tp_as_sequence + 0, // tp_as_mapping + 0, // tp_hash + 0, // tp_call + 0, // tp_str + 0, // tp_getattro + 0, // tp_setattro + 0, // tp_as_buffer + Py_TPFLAGS_DEFAULT, // tp_flags + (char*)"phlex data_cell_index", // tp_doc + 0, // tp_traverse + 0, // tp_clear + 0, // tp_richcompare + 0, // tp_weaklistoffset + 0, // tp_iter + 0, // tp_iternext + dci_methods, // tp_methods + 0, // tp_members + 0, // tp_getset + 0, // tp_base + 0, // tp_dict + 0, // tp_descr_get + 0, // tp_descr_set + 0, // tp_dictoffset + 0, // tp_init + 0, // tp_alloc + 0, // tp_new + 0, // tp_free + 0, // tp_is_gc + 0, // tp_bases + 0, // tp_mro + 0, // tp_cache + 0, // tp_subclasses + 0 // tp_weaklist +#if PY_VERSION_HEX >= 0x02030000 + , 0 // tp_del +#endif +#if PY_VERSION_HEX >= 0x02060000 + , 0 // tp_version_tag +#endif +#if PY_VERSION_HEX >= 0x03040000 + , 0 // tp_finalize +#endif +#if PY_VERSION_HEX >= 0x03080000 + , 0 // tp_vectorcall +#endif +#if PY_VERSION_HEX >= 0x030c0000 + , 0 // tp_watched +#endif +#if PY_VERSION_HEX >= 0x030d0000 + , 0 // tp_versions_used +#endif +}; +// clang-format on diff --git a/plugins/python/src/modulewrap.cpp b/plugins/python/src/modulewrap.cpp index 09604afb..a43f726b 100644 --- a/plugins/python/src/modulewrap.cpp +++ b/plugins/python/src/modulewrap.cpp @@ -1,12 +1,14 @@ -#include "phlex/module.hpp" #include "wrap.hpp" +#include "phlex/model/data_cell_index.hpp" + #include #include #include #include #include +#include #include #include #include @@ -38,6 +40,7 @@ // can cause a performance bottleneck (since all require the GIL). using namespace phlex::experimental; +using namespace phlex; using phlex::concurrency; using phlex::product_query; @@ -57,6 +60,22 @@ PyObject* phlex::experimental::wrap_module(phlex_module_t& module_) return (PyObject*)pymod; } +// Simple phlex source wrapper +// clang-format off +struct phlex::experimental::py_phlex_source { + PyObject_HEAD + phlex_source_t* ph_source; +}; +// clang-format on + +PyObject* phlex::experimental::wrap_source(phlex_source_t& source_) +{ + py_phlex_source* pysrc = PyObject_New(py_phlex_source, &PhlexSource_Type); + pysrc->ph_source = &source_; + + return (PyObject*)pysrc; +} + namespace { static inline std::string stringify(std::vector& v) @@ -151,7 +170,8 @@ namespace { PyGILRAII gil; - PyObject* result = PyObject_CallFunctionObjArgs(m_callable, (PyObject*)args..., nullptr); + PyObject* result = + PyObject_CallFunctionObjArgs(m_callable, lifeline_transform(args)..., nullptr); std::string error_msg; if (!result) { @@ -205,6 +225,42 @@ namespace { void operator()(intptr_t arg0, intptr_t arg1, intptr_t arg2) { callv(arg0, arg1, arg2); } }; + static inline std::optional validate_query(PyObject* pyquery) + { + if (!PyDict_Check(pyquery)) { + PyErr_Format(PyExc_TypeError, "query should be a product specification"); + return std::nullopt; + } + + PyObject* pyc = PyDict_GetItemString(pyquery, "creator"); + if (!pyc || !PyUnicode_Check(pyc)) { + PyErr_Format(PyExc_TypeError, "missing \"creator\" or not a string"); + return std::nullopt; + } + char const* c = PyUnicode_AsUTF8(pyc); + + PyObject* pyl = PyDict_GetItemString(pyquery, "layer"); + if (!pyl || !PyUnicode_Check(pyl)) { + PyErr_Format(PyExc_TypeError, "missing \"layer\" or not a string"); + return std::nullopt; + } + char const* l = PyUnicode_AsUTF8(pyl); + + std::optional s; + PyObject* pys = PyDict_GetItemString(pyquery, "suffix"); + if (pys) { + if (!PyUnicode_Check(pys)) { + PyErr_Format(PyExc_TypeError, "provided \"suffix\" is not a string"); + return std::nullopt; + } + s = identifier(PyUnicode_AsUTF8(pys)); + } else + PyErr_Clear(); + + return std::optional{ + product_query{.creator = identifier(c), .layer = identifier(l), .suffix = s}}; + } + static std::vector validate_input(PyObject* input) { std::vector cargs; @@ -222,37 +278,13 @@ namespace { for (Py_ssize_t i = 0; i < len; ++i) { PyObject* item = items[i]; // borrowed reference - if (!PyDict_Check(item)) { - PyErr_Format(PyExc_TypeError, "input item %d should be a product specifications", (int)i); - break; - } - - PyObject* pyc = PyDict_GetItemString(item, "creator"); - if (!pyc || !PyUnicode_Check(pyc)) { - PyErr_Format(PyExc_TypeError, "missing \"creator\" or not a string"); - break; - } - char const* c = PyUnicode_AsUTF8(pyc); - - PyObject* pyl = PyDict_GetItemString(item, "layer"); - if (!pyl || !PyUnicode_Check(pyl)) { - PyErr_Format(PyExc_TypeError, "missing \"layer\" or not a string"); + auto pq = validate_query(item); + if (pq.has_value()) { + cargs.push_back(pq.value()); + } else { + // validate_query will have set a python exception break; } - char const* l = PyUnicode_AsUTF8(pyl); - - std::optional s; - PyObject* pys = PyDict_GetItemString(item, "suffix"); - if (pys) { - if (!PyUnicode_Check(pys)) { - PyErr_Format(PyExc_TypeError, "provided \"suffix\" is not a string"); - break; - } - s = identifier(PyUnicode_AsUTF8(pys)); - } else - PyErr_Clear(); - - cargs.push_back(product_query{.creator = identifier(c), .layer = identifier(l), .suffix = s}); } if (PyErr_Occurred()) @@ -333,6 +365,41 @@ namespace { return ann; } + // retrieve C++ (matching) types from annotations + static void annotations_to_strings(PyObject* callable, + std::vector& input_types, + std::vector& output_types) + { + PyObject* sann = PyUnicode_FromString("__annotations__"); + PyObject* annot = PyObject_GetAttr(callable, sann); + if (!annot) { + // the callable may be an instance with a __call__ method + PyErr_Clear(); + PyObject* callm = PyObject_GetAttrString(callable, "__call__"); + if (callm) { + annot = PyObject_GetAttr(callm, sann); + Py_DECREF(callm); + } + } + Py_DECREF(sann); + + if (annot && PyDict_Check(annot)) { + // Variant guarantees OrderedDict with "return" last + PyObject *key, *value; + Py_ssize_t pos = 0; + + while (PyDict_Next(annot, &pos, &key, &value)) { + char const* key_str = PyUnicode_AsUTF8(key); + if (strcmp(key_str, "return") == 0) { + output_types.push_back(annotation_as_text(value)); + } else { + input_types.push_back(annotation_as_text(value)); + } + } + } + Py_XDECREF(annot); + } + // converters of builtin types; TODO: this is a basic subset only, b/c either // these will be generated from the IDL, or they should be picked up from an // existing place such as cppyy @@ -421,7 +488,19 @@ namespace { } \ Py_DECREF((PyObject*)pyobj); \ return i; \ - } + } \ + \ + struct provider_cb_##name : public py_callback<1> { \ + cpptype operator()(data_cell_index const& id) \ + { \ + PyGILRAII gil; \ + PyObject* arg0 = wrap_dci(id); \ + PyObject* pyres = (PyObject*)call((intptr_t)arg0); /* decrefs arg0 */ \ + cpptype cres = frompy(pyres); \ + Py_DECREF(pyres); \ + return cres; \ + } \ + }; BASIC_CONVERTER(bool, bool, PyBool_FromLong, pylong_as_bool) BASIC_CONVERTER(int, std::int32_t, PyLong_FromLong, PyLong_AsLong) @@ -526,7 +605,18 @@ namespace { \ Py_DECREF((PyObject*)pyobj); \ return vec; \ - } + } \ + \ + struct provider_cb_##name : public py_callback<1> { \ + std::shared_ptr> operator()(data_cell_index const& id) \ + { \ + PyGILRAII gil; \ + PyObject* arg0 = wrap_dci(id); \ + intptr_t pyres = call((intptr_t)arg0); /* decrefs arg0 */ \ + auto cres = py_to_##name(pyres); /* decrefs pyres */ \ + return cres; \ + } \ + }; NUMPY_ARRAY_CONVERTER(vint, std::int32_t, NPY_INT32, PyLong_AsLong) NUMPY_ARRAY_CONVERTER(vuint, std::uint32_t, NPY_UINT32, pylong_or_int_as_ulong) @@ -535,7 +625,7 @@ namespace { NUMPY_ARRAY_CONVERTER(vfloat, float, NPY_FLOAT, PyFloat_AsDouble) NUMPY_ARRAY_CONVERTER(vdouble, double, NPY_DOUBLE, PyFloat_AsDouble) - // helpers for inserting converter nodes + // helper for inserting converter nodes template void insert_converter(py_phlex_module* mod, std::string const& name, @@ -581,7 +671,7 @@ static PyObject* parse_args(PyObject* args, return nullptr; } - // retrieve function name and argument types + // retrieve function name if (!pyname) { pyname = PyObject_GetAttrString(callable, "__name__"); if (!pyname) { @@ -620,35 +710,7 @@ static PyObject* parse_args(PyObject* args, // retrieve C++ (matching) types from annotations input_types.reserve(input_queries.size()); - - PyObject* sann = PyUnicode_FromString("__annotations__"); - PyObject* annot = PyObject_GetAttr(callable, sann); - if (!annot) { - // the callable may be an instance with a __call__ method - PyErr_Clear(); - PyObject* callm = PyObject_GetAttrString(callable, "__call__"); - if (callm) { - annot = PyObject_GetAttr(callm, sann); - Py_DECREF(callm); - } - } - Py_DECREF(sann); - - if (annot && PyDict_Check(annot)) { - // Variant guarantees OrderedDict with "return" last - PyObject *key, *value; - Py_ssize_t pos = 0; - - while (PyDict_Next(annot, &pos, &key, &value)) { - char const* key_str = PyUnicode_AsUTF8(key); - if (strcmp(key_str, "return") == 0) { - output_types.push_back(annotation_as_text(value)); - } else { - input_types.push_back(annotation_as_text(value)); - } - } - } - Py_XDECREF(annot); + annotations_to_strings(callable, input_types, output_types); // ignore None as Python's conventional "void" return, which is meaningless in C++ if (output_types.size() == 1 && output_types[0] == "None") @@ -741,6 +803,55 @@ static bool insert_input_converters(py_phlex_module* mod, return true; } +static bool insert_output_converter(py_phlex_module* mod, + std::string const& cname, + product_query const& out_pq, + std::string const& out_type, + std::string const& output) +{ + // insert output converter node into the graph + if (out_type == "bool") + insert_converter(mod, cname, py_to_bool, out_pq, output); + else if (out_type == "int32_t") + insert_converter(mod, cname, py_to_int, out_pq, output); + else if (out_type == "uint32_t") + insert_converter(mod, cname, py_to_uint, out_pq, output); + else if (out_type == "int64_t") + insert_converter(mod, cname, py_to_long, out_pq, output); + else if (out_type == "uint64_t") + insert_converter(mod, cname, py_to_ulong, out_pq, output); + else if (out_type == "float") + insert_converter(mod, cname, py_to_float, out_pq, output); + else if (out_type == "double") + insert_converter(mod, cname, py_to_double, out_pq, output); + else if (out_type.compare(0, 7, "ndarray") == 0 || out_type.compare(0, 4, "list") == 0) { + // TODO: just like for input types, these are hard-coded, but should be handled by + // an IDL instead. + std::string_view dtype{out_type.begin() + out_type.rfind('['), out_type.end()}; + if (dtype == "[int32_t]") { + insert_converter(mod, cname, py_to_vint, out_pq, output); + } else if (dtype == "[uint32_t]") { + insert_converter(mod, cname, py_to_vuint, out_pq, output); + } else if (dtype == "[int64_t]") { + insert_converter(mod, cname, py_to_vlong, out_pq, output); + } else if (dtype == "[uint64_t]") { + insert_converter(mod, cname, py_to_vulong, out_pq, output); + } else if (dtype == "[float]") { + insert_converter(mod, cname, py_to_vfloat, out_pq, output); + } else if (dtype == "[double]") { + insert_converter(mod, cname, py_to_vdouble, out_pq, output); + } else { + PyErr_Format(PyExc_TypeError, "unsupported collection output type \"%s\"", out_type.c_str()); + return false; + } + } else { + PyErr_Format(PyExc_TypeError, "unsupported output type \"%s\"", out_type.c_str()); + return false; + } + + return true; +} + static PyObject* md_transform(py_phlex_module* mod, PyObject* args, PyObject* kwds) { // Register a python algorithm by adding the necessary intermediate converter @@ -842,45 +953,10 @@ static PyObject* md_transform(py_phlex_module* mod, PyObject* args, PyObject* kw auto out_pq = product_query{.creator = identifier(pyname), .layer = identifier(output_layer), .suffix = identifier(pyoutput)}; - std::string out_type = output_types[0]; - std::string output = output_suffixes[0]; - if (out_type == "bool") - insert_converter(mod, cname, py_to_bool, out_pq, output); - else if (out_type == "int32_t") - insert_converter(mod, cname, py_to_int, out_pq, output); - else if (out_type == "uint32_t") - insert_converter(mod, cname, py_to_uint, out_pq, output); - else if (out_type == "int64_t") - insert_converter(mod, cname, py_to_long, out_pq, output); - else if (out_type == "uint64_t") - insert_converter(mod, cname, py_to_ulong, out_pq, output); - else if (out_type == "float") - insert_converter(mod, cname, py_to_float, out_pq, output); - else if (out_type == "double") - insert_converter(mod, cname, py_to_double, out_pq, output); - else if (out_type.compare(0, 7, "ndarray") == 0 || out_type.compare(0, 4, "list") == 0) { - // TODO: just like for input types, these are hard-coded, but should be handled by - // an IDL instead. - std::string_view dtype{out_type.begin() + out_type.rfind('['), out_type.end()}; - if (dtype == "[int32_t]") { - insert_converter(mod, cname, py_to_vint, out_pq, output); - } else if (dtype == "[uint32_t]") { - insert_converter(mod, cname, py_to_vuint, out_pq, output); - } else if (dtype == "[int64_t]") { - insert_converter(mod, cname, py_to_vlong, out_pq, output); - } else if (dtype == "[uint64_t]") { - insert_converter(mod, cname, py_to_vulong, out_pq, output); - } else if (dtype == "[float]") { - insert_converter(mod, cname, py_to_vfloat, out_pq, output); - } else if (dtype == "[double]") { - insert_converter(mod, cname, py_to_vdouble, out_pq, output); - } else { - PyErr_Format(PyExc_TypeError, "unsupported collection output type \"%s\"", out_type.c_str()); - return nullptr; - } - } else { - PyErr_Format(PyExc_TypeError, "unsupported output type \"%s\"", out_type.c_str()); - return nullptr; + std::string const& out_type = output_types[0]; + std::string const& output = output_suffixes[0]; + if (!insert_output_converter(mod, cname, out_pq, out_type, output)) { + return nullptr; // error already set } Py_RETURN_NONE; @@ -1039,3 +1115,211 @@ PyTypeObject phlex::experimental::PhlexModule_Type = { #endif }; // clang-format on + +// +// TODO: source wrapper lives here for now to re-use the converter functions; +// this should all be refactored out into their own files +// +static PyObject* sc_provide(py_phlex_source* src, PyObject* args, PyObject* kwds) +{ + // Register a python algorithm by adding the necessary intermediate converter + // nodes going from C++ to PyObject* and back. + + static char const* kwnames[] = {"callable", "output_product", "name", nullptr}; + PyObject *callable = 0, *output = 0, *pyname = 0; + if (!PyArg_ParseTupleAndKeywords( + args, kwds, "OO|O", (char**)kwnames, &callable, &output, &pyname)) { + // error already set by argument parser + return nullptr; + } + + if (!callable || !PyCallable_Check(callable)) { + PyErr_SetString(PyExc_TypeError, "given provider is not callable"); + return nullptr; + } + + // retrieve function name + if (!pyname) { + pyname = PyObject_GetAttrString(callable, "__name__"); + if (!pyname) { + // AttributeError already set + return nullptr; + } + } else { + Py_INCREF(pyname); + } + + std::string functor_name = PyUnicode_AsUTF8(pyname); + Py_DECREF(pyname); + + // retrieve C++ (matching) types from annotations + std::vector input_types; + std::vector output_types; + annotations_to_strings(callable, input_types, output_types); + + // provider needs to take a single "data_cell_input" + if (input_types.size() != 1 || input_types[0] != "data_cell_index") { + PyErr_SetString(PyExc_TypeError, "a provider takes a single \"data_cell_index\" as input"); + return nullptr; + } + + // provider needs to have an output + if (output_types.size() != 1 || output_types[0] == "None") { + PyErr_SetString(PyExc_TypeError, "a provider must have an output"); + return nullptr; + } + + // special case of Phlex Variant wrapper + PyObject* wrapped_callable = PyObject_GetAttrString(callable, "phlex_callable"); + if (wrapped_callable) { + callable = wrapped_callable; + Py_DECREF(wrapped_callable); // safe, b/c callable holds a reference + } else { + // no wrapper, use the original callable + PyErr_Clear(); + } + + // translate and validate the output query + auto opq = validate_query(output); + if (!opq.has_value()) { + // validate_query will have set a python exception + std::string msg; + if (msg_from_py_error(msg, false)) { + throw std::runtime_error("output specification error: " + msg); + } + } + + // insert provider node (TODO: as in transform and observe, we'll leak the + // callable for now, until there's a proper shutdown procedure) + // Note: can't use a translator node here, b/c we need a module to add a + // transform, but we only have a source. However, the interface of a provider + // is fixed, so there is no combinatorics problem. + std::string const& out_type = output_types[0]; + if (out_type == "bool") { + auto* pyc = new provider_cb_bool{callable}; + src->ph_source->provide(functor_name, *pyc).output_product(opq.value()); + } else if (out_type == "int32_t") { + auto* pyc = new provider_cb_int{callable}; + src->ph_source->provide(functor_name, *pyc).output_product(opq.value()); + } else if (out_type == "uint32_t") { + auto* pyc = new provider_cb_uint{callable}; + src->ph_source->provide(functor_name, *pyc).output_product(opq.value()); + } else if (out_type == "int64_t") { + auto* pyc = new provider_cb_long{callable}; + src->ph_source->provide(functor_name, *pyc).output_product(opq.value()); + } else if (out_type == "uint64_t") { + auto* pyc = new provider_cb_ulong{callable}; + src->ph_source->provide(functor_name, *pyc).output_product(opq.value()); + } else if (out_type == "float") { + auto* pyc = new provider_cb_float{callable}; + src->ph_source->provide(functor_name, *pyc).output_product(opq.value()); + } else if (out_type == "double") { + auto* pyc = new provider_cb_double{callable}; + src->ph_source->provide(functor_name, *pyc).output_product(opq.value()); + } else if (out_type.compare(0, 7, "ndarray") == 0 || out_type.compare(0, 4, "list") == 0) { + // TODO: just like for input types, these are hard-coded, but should be handled by + // an IDL instead. + std::string_view dtype{out_type.begin() + out_type.rfind('['), out_type.end()}; + if (dtype == "[int32_t]") { + auto* pyc = new provider_cb_vint{callable}; + src->ph_source->provide(functor_name, *pyc).output_product(opq.value()); + } else if (dtype == "[uint32_t]") { + auto* pyc = new provider_cb_vuint{callable}; + src->ph_source->provide(functor_name, *pyc).output_product(opq.value()); + } else if (dtype == "[int64_t]") { + auto* pyc = new provider_cb_vlong{callable}; + src->ph_source->provide(functor_name, *pyc).output_product(opq.value()); + } else if (dtype == "[uint64_t]") { + auto* pyc = new provider_cb_vulong{callable}; + src->ph_source->provide(functor_name, *pyc).output_product(opq.value()); + } else if (dtype == "[float]") { + auto* pyc = new provider_cb_vfloat{callable}; + src->ph_source->provide(functor_name, *pyc).output_product(opq.value()); + } else if (dtype == "[double]") { + auto* pyc = new provider_cb_vdouble{callable}; + src->ph_source->provide(functor_name, *pyc).output_product(opq.value()); + } else { + PyErr_Format(PyExc_TypeError, "unsupported collection output type \"%s\"", out_type.c_str()); + return nullptr; + } + } else { + PyErr_Format(PyExc_TypeError, "unsupported output type \"%s\"", out_type.c_str()); + return nullptr; + } + + Py_RETURN_NONE; +} + +static PyMethodDef sc_methods[] = {{(char*)"provide", + (PyCFunction)sc_provide, + METH_VARARGS | METH_KEYWORDS, + (char*)"register a Python provider"}, + {(char*)nullptr, nullptr, 0, nullptr}}; + +// clang-format off +PyTypeObject phlex::experimental::PhlexSource_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + (char*)"pyphlex.source", // tp_name + sizeof(py_phlex_source), // tp_basicsize + 0, // tp_itemsize + 0, // tp_dealloc + 0, // tp_vectorcall_offset / tp_print + 0, // tp_getattr + 0, // tp_setattr + 0, // tp_as_async / tp_compare + 0, // tp_repr + 0, // tp_as_number + 0, // tp_as_sequence + 0, // tp_as_mapping + 0, // tp_hash + 0, // tp_call + 0, // tp_str + 0, // tp_getattro + 0, // tp_setattro + 0, // tp_as_buffer + Py_TPFLAGS_DEFAULT, // tp_flags + (char*)"phlex source wrapper", // tp_doc + 0, // tp_traverse + 0, // tp_clear + 0, // tp_richcompare + 0, // tp_weaklistoffset + 0, // tp_iter + 0, // tp_iternext + sc_methods, // tp_methods + 0, // tp_members + 0, // tp_getset + 0, // tp_base + 0, // tp_dict + 0, // tp_descr_get + 0, // tp_descr_set + 0, // tp_dictoffset + 0, // tp_init + 0, // tp_alloc + 0, // tp_new + 0, // tp_free + 0, // tp_is_gc + 0, // tp_bases + 0, // tp_mro + 0, // tp_cache + 0, // tp_subclasses + 0 // tp_weaklist +#if PY_VERSION_HEX >= 0x02030000 + , 0 // tp_del +#endif +#if PY_VERSION_HEX >= 0x02060000 + , 0 // tp_version_tag +#endif +#if PY_VERSION_HEX >= 0x03040000 + , 0 // tp_finalize +#endif +#if PY_VERSION_HEX >= 0x03080000 + , 0 // tp_vectorcall +#endif +#if PY_VERSION_HEX >= 0x030c0000 + , 0 // tp_watched +#endif +#if PY_VERSION_HEX >= 0x030d0000 + , 0 // tp_versions_used +#endif +}; +// clang-format on diff --git a/plugins/python/src/pymodule.cpp b/plugins/python/src/pymodule.cpp index a161f32c..41b7630a 100644 --- a/plugins/python/src/pymodule.cpp +++ b/plugins/python/src/pymodule.cpp @@ -10,41 +10,91 @@ #define PY_ARRAY_UNIQUE_SYMBOL phlex_ARRAY_API #include +#include "phlex/model/data_cell_index.hpp" +#include "phlex/source.hpp" +#include +#include + using namespace phlex::experimental; +using namespace phlex; static bool initialize(); -PHLEX_REGISTER_ALGORITHMS(m, config) -{ - initialize(); - - PyGILRAII g; - - std::string modname = config.get("py"); - PyObject* mod = PyImport_ImportModule(modname.c_str()); - if (mod) { - PyObject* reg = PyObject_GetAttrString(mod, "PHLEX_REGISTER_ALGORITHMS"); - if (reg) { - PyObject* pym = wrap_module(m); - PyObject* pyconfig = wrap_configuration(config); - if (pym && pyconfig) { - PyObject* res = PyObject_CallFunctionObjArgs(reg, pym, pyconfig, nullptr); - Py_XDECREF(res); +// the expansion of registration macros within the same file would lead to +// symbol duplication, hence the use of separate namespaces here +namespace pymodule_register_providers { + PHLEX_REGISTER_PROVIDERS(m, config) + { + initialize(); + + PyGILRAII g; + + std::string modname = config.get("py"); + PyObject* mod = PyImport_ImportModule(modname.c_str()); + if (mod) { + // register providers using conventional callback + PyObject* reg = PyObject_GetAttrString(mod, "PHLEX_REGISTER_PROVIDERS"); + if (reg) { + PyObject* pys = wrap_source(m); + PyObject* pyconfig = wrap_configuration(config); + if (pys && pyconfig) { + PyObject* res = PyObject_CallFunctionObjArgs(reg, pys, pyconfig, nullptr); + Py_XDECREF(res); + } + Py_XDECREF(pyconfig); + Py_XDECREF(pys); + Py_DECREF(reg); } - Py_XDECREF(pyconfig); - Py_XDECREF(pym); - Py_DECREF(reg); + + Py_DECREF(mod); } - Py_DECREF(mod); + + if (PyErr_Occurred()) { + std::string error_msg; + if (!msg_from_py_error(error_msg)) + error_msg = "Unknown python error"; + throw std::runtime_error(error_msg.c_str()); + } + + //m.provide("provide_i", [](data_cell_index const& id) -> int { return id.number() % 2; }) + //.output_product(product_query{.creator = "input", .layer = "event", .suffix = "i"}); } +} // namespace pymodule_register_providers + +namespace pymodule_register_algorithms { + PHLEX_REGISTER_ALGORITHMS(m, config) + { + initialize(); + + PyGILRAII g; + + std::string modname = config.get("py"); + PyObject* mod = PyImport_ImportModule(modname.c_str()); + if (mod) { + // register algorithms using conventional callback + PyObject* reg = PyObject_GetAttrString(mod, "PHLEX_REGISTER_ALGORITHMS"); + if (reg) { + PyObject* pym = wrap_module(m); + PyObject* pyconfig = wrap_configuration(config); + if (pym && pyconfig) { + PyObject* res = PyObject_CallFunctionObjArgs(reg, pym, pyconfig, nullptr); + Py_XDECREF(res); + } + Py_XDECREF(pyconfig); + Py_XDECREF(pym); + Py_DECREF(reg); + } + Py_DECREF(mod); + } - if (PyErr_Occurred()) { - std::string error_msg; - if (!msg_from_py_error(error_msg)) - error_msg = "Unknown python error"; - throw std::runtime_error(error_msg.c_str()); + if (PyErr_Occurred()) { + std::string error_msg; + if (!msg_from_py_error(error_msg)) + error_msg = "Unknown python error"; + throw std::runtime_error(error_msg.c_str()); + } } -} +} // namespace pymodule_register_algorithms static void import_numpy(bool control_interpreter) { @@ -119,13 +169,19 @@ static bool initialize() if (!Py_IsInitialized()) throw std::runtime_error("Python can not be initialized"); + // LCOV_EXCL_START // add custom types if (PyType_Ready(&PhlexConfig_Type) < 0) return false; if (PyType_Ready(&PhlexModule_Type) < 0) return false; + if (PyType_Ready(&PhlexSource_Type) < 0) + return false; + if (PyType_Ready(&PhlexDataCellIndex_Type) < 0) + return false; if (PyType_Ready(&PhlexLifeline_Type) < 0) return false; + // LCOV_EXCL_STOP // FIXME: Spack does not set PYTHONPATH or VIRTUAL_ENV, but it does set // CMAKE_PREFIX_PATH. Add site-packages directories from CMAKE_PREFIX_PATH diff --git a/plugins/python/src/wrap.hpp b/plugins/python/src/wrap.hpp index 318d5484..80d7d311 100644 --- a/plugins/python/src/wrap.hpp +++ b/plugins/python/src/wrap.hpp @@ -22,27 +22,33 @@ #include #include "phlex/configuration.hpp" +#include "phlex/model/data_cell_index.hpp" #include "phlex/module.hpp" +#include "phlex/source.hpp" namespace phlex::experimental { // Create dict-like access to the configuration from Python. - // Returns a new reference. - PyObject* wrap_configuration(configuration const& config); - - // Python wrapper for Phlex configuration + PyObject* wrap_configuration(configuration const& config); // returns new reference extern PyTypeObject PhlexConfig_Type; struct py_config_map; - // Phlex' Module wrapper to register algorithms + // Phlex' module wrapper to register algorithms typedef module_graph_proxy phlex_module_t; - // Returns a new reference. - PyObject* wrap_module(phlex_module_t& mod); - - // Python wrapper for Phlex modules + PyObject* wrap_module(phlex_module_t& mod); // returns new reference extern PyTypeObject PhlexModule_Type; struct py_phlex_module; + // Phlex' source wrapper to register providers + typedef source_graph_proxy phlex_source_t; + PyObject* wrap_source(phlex_source_t& src); // returns new reference + extern PyTypeObject PhlexSource_Type; + struct py_phlex_source; + + // Python wrapper for data cell indices (returns a new reference) + PyObject* wrap_dci(data_cell_index const& dci); + extern PyTypeObject PhlexDataCellIndex_Type; + // Python wrapper for Phlex handles extern PyTypeObject PhlexLifeline_Type; // clang-format off diff --git a/test/python/CMakeLists.txt b/test/python/CMakeLists.txt index 91db151e..aa79543c 100644 --- a/test/python/CMakeLists.txt +++ b/test/python/CMakeLists.txt @@ -180,6 +180,9 @@ list(APPEND ACTIVE_PY_CPHLEX_TESTS py:add) add_test(NAME py:suffix COMMAND phlex::phlex -c ${CMAKE_CURRENT_SOURCE_DIR}/pysuffix.jsonnet) list(APPEND ACTIVE_PY_CPHLEX_TESTS py:suffix) +add_test(NAME py:provide COMMAND phlex::phlex -c ${CMAKE_CURRENT_SOURCE_DIR}/pyprovide.jsonnet) +list(APPEND ACTIVE_PY_CPHLEX_TESTS py:provide) + add_test(NAME py:config COMMAND phlex::phlex -c ${CMAKE_CURRENT_SOURCE_DIR}/pyconfig.jsonnet) list(APPEND ACTIVE_PY_CPHLEX_TESTS py:config) diff --git a/test/python/pyprovide.jsonnet b/test/python/pyprovide.jsonnet new file mode 100644 index 00000000..b502f920 --- /dev/null +++ b/test/python/pyprovide.jsonnet @@ -0,0 +1,31 @@ +{ + driver: { + cpp: 'generate_layers', + layers: { + event: { parent: 'job', total: 10, starting_number: 1 }, + }, + }, + sources: { + provider: { + py: 'pyprovide', + }, + }, + modules: { + algorithms: { + py: 'pyprovide', + input: [ + { + creator: 'input', + layer: 'event', + suffix: 'i', + }, + { + creator: 'input', + layer: 'event', + suffix: 'j', + }, + ], + output: ['sum'], + }, + }, +} diff --git a/test/python/pyprovide.py b/test/python/pyprovide.py new file mode 100644 index 00000000..559089dc --- /dev/null +++ b/test/python/pyprovide.py @@ -0,0 +1,123 @@ +"""A most basic provider. + +This test code implements a simple provider to show that Python data +product can be injected into the execution graph and used by subsequent +algorithms. +""" + +import numpy as np +import numpy.typing as npt + +from phlex import Variant + +_specs = ((-42, np.int32, "ii32"), + (42, np.uint32, "iui32"), + (-27, np.int64, "ii64"), + (27, np.uint64, "iui64"), + (42., np.float32, "if32"), + (-42., np.float64, "if64"), + ) + +specs = [(False, np.bool_, "ib")] +for x, t, sf in _specs: + specs.append((x, t, sf)) # type: ignore + specs.append((np.array([x], dtype=t), npt.NDArray[t], "v"+sf)) # type: ignore + + +def PHLEX_REGISTER_PROVIDERS(s, config): + """Register python providers for all supported types. + + Use the standard Phlex `provide` registration to insert nodes in the + execution graph that receive a data call index and produces any of the + supported types as output. + + Args: + s (internal): Phlex source representation. + config (internal): Phlex configuration representation. + + Returns: + None + """ + def new_p(x): + def p(di): + assert 0 <= di.number() + return x + return p + + for x, t, sf in specs: + f = Variant(new_p(x), {"di": "data_cell_index", "return": t}, "input_"+sf) + s.provide(f, {"creator": "input_"+sf, "layer": "event", "suffix": sf }) + + # add a bunch of failures to check error reporting + try: + s.provide(None) + assert not "supposed to be here" + except TypeError as e: + assert "missing required argument" in str(e) + + try: + class C: + def __call__(self, di): + pass + s.provide(C(), {"creator": "a", "layer": "b", "suffix": "c" }) + assert not "supposed to be here" + except AttributeError as e: + assert "__name__" in str(e) + + try: + s.provide(42, {"creator": "a", "layer": "b", "suffix": "c" }) + assert not "supposed to be here" + except TypeError as e: + assert "given provider is not callable" in str(e) + + try: + s.provide(lambda: 42, {"creator": "a", "layer": "b", "suffix": "c" }) + assert not "supposed to be here" + except TypeError as e: + assert "data_cell_index" in str(e) + + try: + f = Variant(lambda di: 42, {"di": "data_cell_index", "return": None}, "f") + s.provide(f, {"creator": "a", "layer": "b", "suffix": "c" }) + assert not "supposed to be here" + except TypeError as e: + assert "provider must have an output" in str(e) + + try: + f = Variant(lambda di: 42, {"di": "data_cell_index", "return": "object"}, "f") + s.provide(f, {"creator": "a", "layer": "b", "suffix": "c" }) + assert not "supposed to be here" + except TypeError as e: + assert "unsupported output type" in str(e) + + try: + f = Variant(lambda di: 42, {"di": "data_cell_index", "return": npt.NDArray[np.bool_]}, "f") + s.provide(f, {"creator": "a", "layer": "b", "suffix": "c" }) + assert not "supposed to be here" + except TypeError as e: + assert "unsupported collection output type" in str(e) + + +def PHLEX_REGISTER_ALGORITHMS(m, config): + """Register python consumers as observers to check providers' output. + + Use the standard Phlex `observe` registration to insert a node in the + execution graph that receives an input from a python provider and + verifies what it receives. + + Args: + m (internal): Phlex registrar representation. + config (internal): Phlex configuration representation. + + Returns: + None + """ + def new_a(x): + def a(y): + assert y == x + return a + + for x, t, sf in specs: + f = Variant(new_a(x), {"y": t, "return": None}, sf+t.__name__) + m.observe(f, input_family=[{"creator": "input_"+sf, "layer": "event"}]) +