diff --git a/plugins/CMakeLists.txt b/plugins/CMakeLists.txt index af71df3ff..63a32a9e5 100644 --- a/plugins/CMakeLists.txt +++ b/plugins/CMakeLists.txt @@ -1,7 +1,7 @@ # Phlex provided core plugins # plugin for running Python algorithms in phlex -# add_subdirectory(python) +add_subdirectory(python) add_library(layer_generator layer_generator.cpp) target_link_libraries(layer_generator PRIVATE phlex::core) diff --git a/plugins/python/src/configwrap.cpp b/plugins/python/src/configwrap.cpp index 7484d2b45..f141f3d81 100644 --- a/plugins/python/src/configwrap.cpp +++ b/plugins/python/src/configwrap.cpp @@ -15,17 +15,12 @@ struct phlex::experimental::py_config_map { }; // clang-format on -PyObject* phlex::experimental::wrap_configuration(configuration const* config) +PyObject* phlex::experimental::wrap_configuration(configuration const& config) { - if (!config) { - PyErr_SetString(PyExc_ValueError, "provided configuration is null"); - return nullptr; - } - py_config_map* pyconfig = (py_config_map*)PhlexConfig_Type.tp_new(&PhlexConfig_Type, nullptr, nullptr); - pyconfig->ph_config = config; + pyconfig->ph_config = &config; return (PyObject*)pyconfig; } @@ -73,6 +68,11 @@ static PyObject* pcm_subscript(py_config_map* pycmap, PyObject* pykey) std::string ckey = PyUnicode_AsUTF8(pykey); + // Note: Python3.14 adds PyLong_FromInt64/PyLong_FromUInt64 to replace the + // long long variants + static_assert(sizeof(long long) >= sizeof(int64_t)); + static_assert(sizeof(unsigned long long) >= sizeof(uint64_t)); + try { auto k = pycmap->ph_config->prototype_internal_kind(ckey); if (k.second /* is array */) { @@ -87,14 +87,16 @@ static PyObject* pcm_subscript(py_config_map* pycmap, PyObject* pykey) auto const& cvalue = pycmap->ph_config->get>(ckey); pyvalue = PyTuple_New(cvalue.size()); for (Py_ssize_t i = 0; i < (Py_ssize_t)cvalue.size(); ++i) { - PyObject* item = PyLong_FromLong(cvalue[i]); + // Note Python3.14 is expected to add PyLong_FromInt64 + PyObject* item = PyLong_FromLongLong(cvalue[i]); PyTuple_SetItem(pyvalue, i, item); } } else if (k.first == boost::json::kind::uint64) { auto const& cvalue = pycmap->ph_config->get>(ckey); pyvalue = PyTuple_New(cvalue.size()); for (Py_ssize_t i = 0; i < (Py_ssize_t)cvalue.size(); ++i) { - PyObject* item = PyLong_FromUnsignedLong(cvalue[i]); + // Note Python3.14 is expected to add PyLong_FromUInt64 + PyObject* item = PyLong_FromUnsignedLongLong(cvalue[i]); PyTuple_SetItem(pyvalue, i, item); } } else if (k.first == boost::json::kind::double_) { @@ -111,6 +113,21 @@ static PyObject* pcm_subscript(py_config_map* pycmap, PyObject* pykey) PyObject* item = PyUnicode_FromStringAndSize(cvalue[i].c_str(), cvalue[i].size()); PyTuple_SetItem(pyvalue, i, item); } + } else if (k.first == boost::json::kind::object) { + auto cvalue = pycmap->ph_config->get>>(ckey); + pyvalue = PyTuple_New(cvalue.size()); + for (Py_ssize_t i = 0; i < (Py_ssize_t)cvalue.size(); ++i) { + PyObject* item = PyDict_New(); + for (auto const& kv : cvalue[i]) { + PyObject* val = PyUnicode_FromStringAndSize(kv.second.c_str(), kv.second.size()); + PyDict_SetItemString(item, kv.first.c_str(), val); + Py_DECREF(val); + } + PyTuple_SetItem(pyvalue, i, item); + } + } else if (k.first == boost::json::kind::null) { + // special case: empty array + pyvalue = PyTuple_New(0); } } else { if (k.first == boost::json::kind::bool_) { @@ -118,25 +135,37 @@ static PyObject* pcm_subscript(py_config_map* pycmap, PyObject* pykey) pyvalue = PyBool_FromLong((long)cvalue); } else if (k.first == boost::json::kind::int64) { auto cvalue = pycmap->ph_config->get(ckey); - pyvalue = PyLong_FromLong(cvalue); + // Note Python3.14 is expected to add PyLong_FromInt64 + pyvalue = PyLong_FromLongLong(cvalue); } else if (k.first == boost::json::kind::uint64) { auto cvalue = pycmap->ph_config->get(ckey); - pyvalue = PyLong_FromUnsignedLong(cvalue); + // Note Python3.14 is expected to add PyLong_FromUInt64 + pyvalue = PyLong_FromUnsignedLongLong(cvalue); } else if (k.first == boost::json::kind::double_) { auto cvalue = pycmap->ph_config->get(ckey); pyvalue = PyFloat_FromDouble(cvalue); } else if (k.first == boost::json::kind::string) { auto const& cvalue = pycmap->ph_config->get(ckey); pyvalue = PyUnicode_FromStringAndSize(cvalue.c_str(), cvalue.size()); + } else if (k.first == boost::json::kind::object) { + auto cvalue = pycmap->ph_config->get>(ckey); + pyvalue = PyDict_New(); + for (auto const& kv : cvalue) { + PyObject* val = PyUnicode_FromStringAndSize(kv.second.c_str(), kv.second.size()); + PyDict_SetItemString(pyvalue, kv.first.c_str(), val); + Py_DECREF(val); + } } } - } catch (std::runtime_error const&) { - PyErr_Format(PyExc_KeyError, "property \"%s\" does not exist", ckey.c_str()); + } catch (std::runtime_error const& e) { + PyErr_Format(PyExc_KeyError, "failed to retrieve property \"%s\" (%s)", ckey.c_str(), e.what()); } // cache if found if (pyvalue) { PyDict_SetItem(pycmap->ph_config_cache, pykey, pyvalue); + } else if (!PyErr_Occurred()) { + PyErr_Format(PyExc_KeyError, "property \"%s\" is of unknown type", ckey.c_str()); } return pyvalue; diff --git a/plugins/python/src/errorwrap.cpp b/plugins/python/src/errorwrap.cpp index c8a073ee4..77e972c91 100644 --- a/plugins/python/src/errorwrap.cpp +++ b/plugins/python/src/errorwrap.cpp @@ -2,6 +2,12 @@ #include +// This code has excluded several error checking paths from code coverage, +// because the conditions to create the errors (trace formatting problems) +// are rare and too hard to recreate in a test, while the resolution (fall +// back to a generic error messsage) is rather straightforward and thus +// does not need testing. + using namespace phlex::experimental; static bool format_traceback(std::string& msg, @@ -25,27 +31,33 @@ static bool format_traceback(std::string& msg, #endif Py_DECREF(format_exception); + // LCOV_EXCL_START if (!formatted_tb) { PyErr_Clear(); return false; } + // LCOV_EXCL_STOP PyObject* py_msg_empty = PyUnicode_FromString(""); PyObject* py_msg = PyUnicode_Join(py_msg_empty, formatted_tb); Py_DECREF(py_msg_empty); Py_DECREF(formatted_tb); + // LCOV_EXCL_START if (!py_msg) { PyErr_Clear(); return false; } + // LCOV_EXCL_STOP char const* c_msg = PyUnicode_AsUTF8(py_msg); + // LCOV_EXCL_START if (c_msg) { msg = c_msg; Py_DECREF(py_msg); return true; } + // LCOV_EXCL_STOP PyErr_Clear(); Py_DECREF(py_msg); @@ -66,11 +78,13 @@ bool phlex::experimental::msg_from_py_error(std::string& msg, bool check_error) PyErr_Fetch(&type, &value, &traceback); if (value) { bool tb_ok = format_traceback(msg, type, value, traceback); + // LCOV_EXCL_START if (!tb_ok) { PyObject* pymsg = PyObject_Str(value); msg = PyUnicode_AsUTF8(pymsg); Py_DECREF(pymsg); } + // LCOV_EXCL_STOP } else { msg = "unknown Python error occurred"; } diff --git a/plugins/python/src/modulewrap.cpp b/plugins/python/src/modulewrap.cpp index 9517e095c..c48973621 100644 --- a/plugins/python/src/modulewrap.cpp +++ b/plugins/python/src/modulewrap.cpp @@ -1,9 +1,13 @@ #include "phlex/module.hpp" #include "wrap.hpp" +#include +#include + +#include #include #include -#include +#include #include #include @@ -11,14 +15,28 @@ #define PY_ARRAY_UNIQUE_SYMBOL phlex_ARRAY_API #include +// Python algorithms are supported by inserting nodes from C++ -> Python, +// followed by the intended call, and another from Python -> C++. +// +// Since product_query inputs, list the creator name, the suffix can remain +// the same through out the chain (as does the layer), distinguishing the +// stage with the creator name (and thus the node names) only. +// +// The chain is as follows (last step not added for observers): +// C++ -> Python: creator: +// name: _arg_py +// output: py_ +// Python algoritm: creator: _arg>_py (xN) +// name: py_ +// output: _py +// Python -> C++: creator: py_ +// name: +// output: + using namespace phlex::experimental; using phlex::concurrency; using phlex::product_query; -// TODO: the layer is currently hard-wired and should come from the product -// specification instead, but that doesn't exist in Python yet. -static std::string const LAYER = "event"; - // Simple phlex module wrapper // clang-format off struct phlex::experimental::py_phlex_module { @@ -27,32 +45,29 @@ struct phlex::experimental::py_phlex_module { }; // clang-format on -PyObject* phlex::experimental::wrap_module(phlex_module_t* module_) +PyObject* phlex::experimental::wrap_module(phlex_module_t& module_) { - if (!module_) { - PyErr_SetString(PyExc_ValueError, "provided module is null"); - return nullptr; - } - py_phlex_module* pymod = PyObject_New(py_phlex_module, &PhlexModule_Type); - pymod->ph_module = module_; + pymod->ph_module = &module_; return (PyObject*)pymod; } namespace { - // TODO: wishing for std::views::join_with() in C++23, but until then: - static std::string stringify(std::vector& v) + static inline std::string stringify(std::vector& v) { - std::ostringstream oss; - if (!v.empty()) { - oss << v.front(); - for (std::size_t i = 1; i < v.size(); ++i) { - oss << ", " << v[i]; - } - } - return oss.str(); + return fmt::format("{:n}", v); + } + + static inline std::string stringify(std::vector& v) + { + return fmt::format("{:n}", std::ranges::views::transform(v, &product_query::to_string)); + } + + static inline std::string input_converter_name(std::string const& algname, size_t arg) + { + return fmt::format("{}_arg{}_py", algname, arg); } static inline PyObject* lifeline_transform(intptr_t arg) @@ -186,15 +201,70 @@ namespace { void operator()(intptr_t arg0, intptr_t arg1, intptr_t arg2) { callv(arg0, arg1, arg2); } }; - static std::vector cseq(PyObject* coll) + static std::vector validate_input(PyObject* input) { - if (!coll) { - return std::vector{}; - } + std::vector cargs; + if (!input) + return cargs; + + PyObject* coll = PySequence_Fast(input, "input_family must be a sequence"); + if (!coll) + return cargs; - // coll is guaranteed to be a list or tuple (from PySequence_Fast) Py_ssize_t len = PySequence_Fast_GET_SIZE(coll); + cargs.reserve(static_cast(len)); + + PyObject** items = PySequence_Fast_ITEMS(coll); + for (Py_ssize_t i = 0; i < len; ++i) { + PyObject* item = items[i]; // borrowed reference + + if (!PyDict_Check(item)) { + PyErr_Format(PyExc_TypeError, "input item %d should be a product specifications", (int)i); + break; + } + + PyObject* pyc = PyDict_GetItemString(item, "creator"); + if (!pyc || !PyUnicode_Check(pyc)) { + PyErr_Format(PyExc_ValueError, "missing \"creator\" for input specification"); + break; + } + char const* c = PyUnicode_AsUTF8(pyc); + + PyObject* pyl = PyDict_GetItemString(item, "layer"); + if (!pyl || !PyUnicode_Check(pyl)) { + PyErr_Format(PyExc_ValueError, "missing \"layer\" for input specification"); + break; + } + char const* l = PyUnicode_AsUTF8(pyl); + + PyObject* pys = PyDict_GetItemString(item, "suffix"); + if (!pys || !PyUnicode_Check(pys)) { + PyErr_Format(PyExc_ValueError, "missing \"suffix\" for input specification"); + break; + } + char const* s = PyUnicode_AsUTF8(pys); + + cargs.push_back( + product_query{.creator = identifier(c), .layer = identifier(l), .suffix = identifier(s)}); + } + + if (PyErr_Occurred()) + cargs.clear(); // error handled through Python + + return cargs; + } + + static std::vector validate_output(PyObject* output) + { std::vector cargs; + if (!output) + return cargs; + + PyObject* coll = PySequence_Fast(output, "output_products must be a sequence"); + if (!coll) + return cargs; + + Py_ssize_t len = PySequence_Fast_GET_SIZE(coll); cargs.reserve(static_cast(len)); PyObject** items = PySequence_Fast_ITEMS(coll); @@ -202,18 +272,23 @@ namespace { PyObject* item = items[i]; // borrowed reference if (!PyUnicode_Check(item)) { PyErr_Format(PyExc_TypeError, "item %d must be a string", (int)i); - return std::vector{}; // Error set + break; } char const* p = PyUnicode_AsUTF8(item); if (!p) { - return std::vector{}; // Error already set + break; } Py_ssize_t sz = PyUnicode_GetLength(item); cargs.emplace_back(p, static_cast(sz)); } + Py_DECREF(coll); + + if (PyErr_Occurred()) + cargs.clear(); // error handled through Python + return cargs; } @@ -449,22 +524,25 @@ namespace { NUMPY_ARRAY_CONVERTER(vfloat, float, NPY_FLOAT, PyFloat_AsDouble) NUMPY_ARRAY_CONVERTER(vdouble, double, NPY_DOUBLE, PyFloat_AsDouble) -} // unnamed namespace - -#define INSERT_INPUT_CONVERTER(name, alg, inp) \ - mod->ph_module->transform("py" #name "_" + inp + "_" + alg, name##_to_py, concurrency::serial) \ - .input_family(product_query{product_specification::create(inp), LAYER}) \ - .output_products(alg + "_" + inp + "py") + // helpers for inserting converter nodes + template + void insert_converter(py_phlex_module* mod, + std::string const& name, + R (*converter)(Args...), + product_query pq_in, + std::string const& output) + { + mod->ph_module->transform(name, converter, concurrency::serial) + .input_family(pq_in) + .output_products(output); + } -#define INSERT_OUTPUT_CONVERTER(name, alg, outp) \ - mod->ph_module->transform(#name "py_" + outp + "_" + alg, py_to_##name, concurrency::serial) \ - .input_family(product_query{product_specification::create("py" + outp + "_" + alg), LAYER}) \ - .output_products(outp) +} // unnamed namespace static PyObject* parse_args(PyObject* args, PyObject* kwds, std::string& functor_name, - std::vector& input_labels, + std::vector& input_queries, std::vector& input_types, std::vector& output_labels, std::vector& output_types) @@ -511,36 +589,26 @@ static PyObject* parse_args(PyObject* args, return nullptr; } - // Accept any sequence type (list, tuple, custom sequences) - PyObject* input_fast = PySequence_Fast(input, "input_family must be a sequence"); - if (!input_fast) { - return nullptr; // TypeError already set by PySequence_Fast - } - - PyObject* output_fast = nullptr; - if (output) { - output_fast = PySequence_Fast(output, "output_products must be a sequence"); - if (!output_fast) { - Py_DECREF(input_fast); - return nullptr; + // convert input declarations, to be able to pass them to Phlex + input_queries = validate_input(input); + if (input_queries.empty()) { + if (!PyErr_Occurred()) { + PyErr_Format(PyExc_ValueError, + "no input provided for %s; node can not be scheduled", + functor_name.c_str()); } + return nullptr; } - // convert input and output declarations, to be able to pass them to Phlex - input_labels = cseq(input_fast); - output_labels = cseq(output_fast); - - // Clean up fast sequences - Py_DECREF(input_fast); - Py_XDECREF(output_fast); - + // convert output declarations, to be able to pass them to Phlex + output_labels = validate_output(output); if (output_labels.size() > 1) { PyErr_SetString(PyExc_TypeError, "only a single output supported"); return nullptr; } // retrieve C++ (matching) types from annotations - input_types.reserve(input_labels.size()); + input_types.reserve(input_queries.size()); PyObject* sann = PyUnicode_FromString("__annotations__"); PyObject* annot = PyObject_GetAttr(callable, sann); @@ -577,11 +645,11 @@ static PyObject* parse_args(PyObject* args, // if annotations were correct (and correctly parsed), there should be as many // input types as input labels - if (input_types.size() != input_labels.size()) { + if (input_types.size() != input_queries.size()) { PyErr_Format(PyExc_TypeError, "number of inputs (%d; %s) does not match number of annotation types (%d; %s)", - input_labels.size(), - stringify(input_labels).c_str(), + input_queries.size(), + stringify(input_queries).c_str(), input_types.size(), stringify(input_types).c_str()); return nullptr; @@ -604,106 +672,72 @@ static PyObject* parse_args(PyObject* args, static bool insert_input_converters(py_phlex_module* mod, std::string const& cname, // TODO: shared_ptr - std::vector const& input_labels, + std::vector const& input_queries, std::vector const& input_types) { // insert input converter nodes into the graph - for (size_t i = 0; i < (size_t)input_labels.size(); ++i) { + for (size_t i = 0; i < (size_t)input_queries.size(); ++i) { // TODO: this seems overly verbose and inefficient, but the function needs // to be properly types, so every option is made explicit - auto const& inp = input_labels[i]; + auto const& inp_pq = input_queries[i]; auto const& inp_type = input_types[i]; + std::string const& pyname = input_converter_name(cname, i); + std::string output = "py_" + std::string{static_cast(*inp_pq.suffix)}; + if (inp_type == "bool") - INSERT_INPUT_CONVERTER(bool, cname, inp); + insert_converter(mod, pyname, bool_to_py, inp_pq, output); else if (inp_type == "int") - INSERT_INPUT_CONVERTER(int, cname, inp); + insert_converter(mod, pyname, int_to_py, inp_pq, output); else if (inp_type == "unsigned int") - INSERT_INPUT_CONVERTER(uint, cname, inp); + insert_converter(mod, pyname, uint_to_py, inp_pq, output); else if (inp_type == "long") - INSERT_INPUT_CONVERTER(long, cname, inp); + insert_converter(mod, pyname, long_to_py, inp_pq, output); else if (inp_type == "unsigned long") - INSERT_INPUT_CONVERTER(ulong, cname, inp); + insert_converter(mod, pyname, ulong_to_py, inp_pq, output); else if (inp_type == "float") - INSERT_INPUT_CONVERTER(float, cname, inp); + insert_converter(mod, pyname, float_to_py, inp_pq, output); else if (inp_type == "double") - INSERT_INPUT_CONVERTER(double, cname, inp); - else if (inp_type.compare(0, 13, "numpy.ndarray") == 0) { + insert_converter(mod, pyname, double_to_py, inp_pq, output); + else if (inp_type.compare(0, 13, "numpy.ndarray") == 0 || inp_type.compare(0, 4, "list") == 0) { // TODO: these are hard-coded std::vector <-> numpy array mappings, which is // way too simplistic for real use. It only exists for demonstration purposes, // until we have an IDL auto pos = inp_type.rfind("numpy.dtype"); if (pos == std::string::npos) { - PyErr_Format( - PyExc_TypeError, "could not determine dtype of input type \"%s\"", inp_type.c_str()); - return false; + if (inp_type[0] == 'l') { + pos = 0; + } else { + PyErr_Format( + PyExc_TypeError, "could not determine dtype of input type \"%s\"", inp_type.c_str()); + return false; + } + } else { + pos += 18; } - pos += 18; - std::string py_out = cname + "_" + inp + "py"; - - if (inp_type.compare(pos, 8, "uint32]]") == 0) { - mod->ph_module->transform("pyvuint_" + inp + "_" + cname, vuint_to_py, concurrency::serial) - .input_family(product_query{product_specification::create(inp), LAYER}) - .output_products(py_out); - } else if (inp_type.compare(pos, 7, "int32]]") == 0) { - mod->ph_module->transform("pyvint_" + inp + "_" + cname, vint_to_py, concurrency::serial) - .input_family(product_query{product_specification::create(inp), LAYER}) - .output_products(py_out); - } else if (inp_type.compare(pos, 8, "uint64]]") == 0) { // id. - mod->ph_module - ->transform("pyvulong_" + inp + "_" + cname, vulong_to_py, concurrency::serial) - .input_family(product_query{product_specification::create(inp), LAYER}) - .output_products(py_out); - } else if (inp_type.compare(pos, 7, "int64]]") == 0) { // need not be true - mod->ph_module->transform("pyvlong_" + inp + "_" + cname, vlong_to_py, concurrency::serial) - .input_family(product_query{product_specification::create(inp), LAYER}) - .output_products(py_out); - } else if (inp_type.compare(pos, 9, "float32]]") == 0) { - mod->ph_module - ->transform("pyvfloat_" + inp + "_" + cname, vfloat_to_py, concurrency::serial) - .input_family(product_query{product_specification::create(inp), LAYER}) - .output_products(py_out); - } else if (inp_type.compare(pos, 9, "float64]]") == 0) { - mod->ph_module - ->transform("pyvdouble_" + inp + "_" + cname, vdouble_to_py, concurrency::serial) - .input_family(product_query{product_specification::create(inp), LAYER}) - .output_products(py_out); + if (inp_type.compare(pos, std::string::npos, "uint32]]") == 0 || + inp_type == "list[unsigned int]" || inp_type == "list['unsigned int']") { + insert_converter(mod, pyname, vuint_to_py, inp_pq, output); + } else if (inp_type.compare(pos, std::string::npos, "int32]]") == 0 || + inp_type == "list[int]") { + insert_converter(mod, pyname, vint_to_py, inp_pq, output); + } else if (inp_type.compare(pos, std::string::npos, "uint64]]") == 0 || // need not be true + inp_type == "list[unsigned long]" || inp_type == "list['unsigned long']") { + insert_converter(mod, pyname, vulong_to_py, inp_pq, output); + } else if (inp_type.compare(pos, std::string::npos, "int64]]") == 0 || // id. + inp_type == "list[long]" || inp_type == "list['long']") { + insert_converter(mod, pyname, vlong_to_py, inp_pq, output); + } else if (inp_type.compare(pos, std::string::npos, "float32]]") == 0 || + inp_type == "list[float]") { + insert_converter(mod, pyname, vfloat_to_py, inp_pq, output); + } else if (inp_type.compare(pos, std::string::npos, "float64]]") == 0 || + inp_type == "list[double]" || inp_type == "list['double']") { + insert_converter(mod, pyname, vdouble_to_py, inp_pq, output); } else { - PyErr_Format(PyExc_TypeError, "unsupported array input type \"%s\"", inp_type.c_str()); + PyErr_Format(PyExc_TypeError, "unsupported collection input type \"%s\"", inp_type.c_str()); return false; } - } else if (inp_type == "list[int]") { - std::string py_out = cname + "_" + inp + "py"; - mod->ph_module->transform("pyvint_" + inp + "_" + cname, vint_to_py, concurrency::serial) - .input_family(product_query{product_specification::create(inp), LAYER}) - .output_products(py_out); - } else if (inp_type == "list[unsigned int]" || inp_type == "list['unsigned int']") { - std::string py_out = cname + "_" + inp + "py"; - mod->ph_module->transform("pyvuint_" + inp + "_" + cname, vuint_to_py, concurrency::serial) - .input_family(product_query{product_specification::create(inp), LAYER}) - .output_products(py_out); - } else if (inp_type == "list[long]" || inp_type == "list['long']") { - std::string py_out = cname + "_" + inp + "py"; - mod->ph_module->transform("pyvlong_" + inp + "_" + cname, vlong_to_py, concurrency::serial) - .input_family(product_query{product_specification::create(inp), LAYER}) - .output_products(py_out); - } else if (inp_type == "list[unsigned long]" || inp_type == "list['unsigned long']") { - std::string py_out = cname + "_" + inp + "py"; - mod->ph_module->transform("pyvulong_" + inp + "_" + cname, vulong_to_py, concurrency::serial) - .input_family(product_query{product_specification::create(inp), LAYER}) - .output_products(py_out); - } else if (inp_type == "list[float]") { - std::string py_out = cname + "_" + inp + "py"; - mod->ph_module->transform("pyvfloat_" + inp + "_" + cname, vfloat_to_py, concurrency::serial) - .input_family(product_query{product_specification::create(inp), LAYER}) - .output_products(py_out); - } else if (inp_type == "list[double]" || inp_type == "list['double']") { - std::string py_out = cname + "_" + inp + "py"; - mod->ph_module - ->transform("pyvdouble_" + inp + "_" + cname, vdouble_to_py, concurrency::serial) - .input_family(product_query{product_specification::create(inp), LAYER}) - .output_products(py_out); } else { PyErr_Format(PyExc_TypeError, "unsupported input type \"%s\"", inp_type.c_str()); return false; @@ -719,152 +753,154 @@ static PyObject* md_transform(py_phlex_module* mod, PyObject* args, PyObject* kw // nodes going from C++ to PyObject* and back. std::string cname; - std::vector input_labels, input_types, output_labels, output_types; + std::vector input_queries; + std::vector input_types, output_labels, output_types; PyObject* callable = - parse_args(args, kwds, cname, input_labels, input_types, output_labels, output_types); + parse_args(args, kwds, cname, input_queries, input_types, output_labels, output_types); if (!callable) return nullptr; // error already set if (output_types.empty()) { - PyErr_Format(PyExc_TypeError, "a transform should have an output type"); + PyErr_Format(PyExc_TypeError, "transform %s should have an output type", cname.c_str()); Py_DECREF(callable); return nullptr; } - // TODO: only support single output type for now, as there has to be a mapping - // onto a std::tuple otherwise, which is a typed object, thus complicating the - // template instantiation - std::string output = output_labels[0]; - std::string output_type = output_types[0]; + // TODO: it's not clear what the output layer will be if the input layers are not + // all the same, so for now, simply raise an error if their is any ambiguity + auto output_layer = static_cast(input_queries[0].layer); + if (1 < input_queries.size()) { + for (std::vector::size_type iq = 1; iq < input_queries.size(); ++iq) { + if (static_cast(input_queries[iq].layer) != output_layer) { + PyErr_Format(PyExc_ValueError, "transform %s output layer is ambiguous", cname.c_str()); + Py_DECREF(callable); + return nullptr; + } + } + } - if (!insert_input_converters(mod, cname, input_labels, input_types)) { + if (!insert_input_converters(mod, cname, input_queries, input_types)) { Py_DECREF(callable); return nullptr; // error already set } // register Python transform - std::string py_out = "py" + output + "_" + cname; - if (input_labels.size() == 1) { + + // TODO: only support single output type for now, as there has to be a mapping + // onto a std::tuple otherwise, which is a typed object, thus complicating the + // template instantiation + std::string pyname = "py_" + cname; + std::string pyoutput = output_labels[0] + "_py"; + + auto pq0 = input_queries[0]; + std::string c0 = input_converter_name(cname, 0); + std::string suff0 = "py_" + std::string{static_cast(*pq0.suffix)}; + + switch (input_queries.size()) { + case 1: { auto* pyc = new py_callback_1{callable}; // TODO: leaks, but has program lifetime - mod->ph_module->transform(cname, *pyc, concurrency::serial) + mod->ph_module->transform(pyname, *pyc, concurrency::serial) .input_family( - product_query{product_specification::create(cname + "_" + input_labels[0] + "py"), LAYER}) - .output_products(py_out); - Py_DECREF(callable); - } else if (input_labels.size() == 2) { + product_query{.creator = identifier(c0), .layer = pq0.layer, .suffix = identifier(suff0)}) + .output_products(pyoutput); + break; + } + case 2: { auto* pyc = new py_callback_2{callable}; - mod->ph_module->transform(cname, *pyc, concurrency::serial) + auto pq1 = input_queries[1]; + std::string suff1 = "py_" + std::string{static_cast(*pq1.suffix)}; + + std::string c1 = input_converter_name(cname, 1); + mod->ph_module->transform(pyname, *pyc, concurrency::serial) .input_family( - product_query{product_specification::create(cname + "_" + input_labels[0] + "py"), LAYER}, - product_query{product_specification::create(cname + "_" + input_labels[1] + "py"), LAYER}) - .output_products(py_out); - Py_DECREF(callable); - } else if (input_labels.size() == 3) { + product_query{.creator = identifier(c0), .layer = pq0.layer, .suffix = identifier(suff0)}, + product_query{.creator = identifier(c1), .layer = pq1.layer, .suffix = identifier(suff1)}) + .output_products(pyoutput); + break; + } + case 3: { auto* pyc = new py_callback_3{callable}; - mod->ph_module->transform(cname, *pyc, concurrency::serial) + auto pq1 = input_queries[1]; + std::string c1 = input_converter_name(cname, 1); + std::string suff1 = "py_" + std::string{static_cast(*pq1.suffix)}; + auto pq2 = input_queries[2]; + std::string c2 = input_converter_name(cname, 2); + std::string suff2 = "py_" + std::string{static_cast(*pq2.suffix)}; + mod->ph_module->transform(pyname, *pyc, concurrency::serial) .input_family( - product_query{product_specification::create(cname + "_" + input_labels[0] + "py"), LAYER}, - product_query{product_specification::create(cname + "_" + input_labels[1] + "py"), LAYER}, - product_query{product_specification::create(cname + "_" + input_labels[2] + "py"), LAYER}) - .output_products(py_out); - Py_DECREF(callable); - } else { + product_query{.creator = identifier(c0), .layer = pq0.layer, .suffix = identifier(suff0)}, + product_query{.creator = identifier(c1), .layer = pq1.layer, .suffix = identifier(suff1)}, + product_query{.creator = identifier(c2), .layer = pq2.layer, .suffix = identifier(suff2)}) + .output_products(pyoutput); + break; + } + default: { PyErr_SetString(PyExc_TypeError, "unsupported number of inputs"); Py_DECREF(callable); return nullptr; } + } - // insert output converter node into the graph (TODO: same as above; these - // are explicit b/c of the templates only) + // insert output converter node into the graph + auto out_pq = product_query{.creator = identifier(pyname), + .layer = identifier(output_layer), + .suffix = identifier(pyoutput)}; + std::string output_type = output_types[0]; + std::string output = output_labels[0]; if (output_type == "bool") - INSERT_OUTPUT_CONVERTER(bool, cname, output); + insert_converter(mod, cname, py_to_bool, out_pq, output); else if (output_type == "int") - INSERT_OUTPUT_CONVERTER(int, cname, output); + insert_converter(mod, cname, py_to_int, out_pq, output); else if (output_type == "unsigned int") - INSERT_OUTPUT_CONVERTER(uint, cname, output); + insert_converter(mod, cname, py_to_uint, out_pq, output); else if (output_type == "long") - INSERT_OUTPUT_CONVERTER(long, cname, output); + insert_converter(mod, cname, py_to_long, out_pq, output); else if (output_type == "unsigned long") - INSERT_OUTPUT_CONVERTER(ulong, cname, output); + insert_converter(mod, cname, py_to_ulong, out_pq, output); else if (output_type == "float") - INSERT_OUTPUT_CONVERTER(float, cname, output); + insert_converter(mod, cname, py_to_float, out_pq, output); else if (output_type == "double") - INSERT_OUTPUT_CONVERTER(double, cname, output); - else if (output_type.compare(0, 13, "numpy.ndarray") == 0) { + insert_converter(mod, cname, py_to_double, out_pq, output); + else if (output_type.compare(0, 13, "numpy.ndarray") == 0 || + output_type.compare(0, 4, "list") == 0) { // TODO: just like for input types, these are hard-coded, but should be handled by // an IDL instead. auto pos = output_type.rfind("numpy.dtype"); if (pos == std::string::npos) { - PyErr_Format( - PyExc_TypeError, "could not determine dtype of input type \"%s\"", output_type.c_str()); - return nullptr; + if (output_type[0] == 'l') { + pos = 0; + } else { + PyErr_Format( + PyExc_TypeError, "could not determine dtype of output type \"%s\"", output_type.c_str()); + return nullptr; + } + } else { + pos += 18; } - pos += 18; - - auto py_in = "py" + output + "_" + cname; - if (output_type.compare(pos, 7, "int32]]") == 0) { - mod->ph_module->transform("pyvint_" + output + "_" + cname, py_to_vint, concurrency::serial) - .input_family(product_query{product_specification::create(py_in), LAYER}) - .output_products(output); - } else if (output_type.compare(pos, 8, "uint32]]") == 0) { - mod->ph_module->transform("pyvuint_" + output + "_" + cname, py_to_vuint, concurrency::serial) - .input_family(product_query{product_specification::create(py_in), LAYER}) - .output_products(output); - } else if (output_type.compare(pos, 7, "int64]]") == 0) { // need not be true - mod->ph_module->transform("pyvlong_" + output + "_" + cname, py_to_vlong, concurrency::serial) - .input_family(product_query{product_specification::create(py_in), LAYER}) - .output_products(output); - } else if (output_type.compare(pos, 8, "uint64]]") == 0) { // id. - mod->ph_module - ->transform("pyvulong_" + output + "_" + cname, py_to_vulong, concurrency::serial) - .input_family(product_query{product_specification::create(py_in), LAYER}) - .output_products(output); - } else if (output_type.compare(pos, 9, "float32]]") == 0) { - mod->ph_module - ->transform("pyvfloat_" + output + "_" + cname, py_to_vfloat, concurrency::serial) - .input_family(product_query{product_specification::create(py_in), LAYER}) - .output_products(output); - } else if (output_type.compare(pos, 9, "float64]]") == 0) { - mod->ph_module - ->transform("pyvdouble_" + output + "_" + cname, py_to_vdouble, concurrency::serial) - .input_family(product_query{product_specification::create(py_in), LAYER}) - .output_products(output); + if (output_type.compare(pos, std::string::npos, "uint32]]") == 0 || + output_type == "list[unsigned int]" || output_type == "list['unsigned int']") { + insert_converter(mod, cname, py_to_vuint, out_pq, output); + } else if (output_type.compare(pos, std::string::npos, "int32]]") == 0 || + output_type == "list[int]") { + insert_converter(mod, cname, py_to_vint, out_pq, output); + } else if (output_type.compare(pos, std::string::npos, "uint64]]") == 0 || // need not be true + output_type == "list[unsigned long]" || output_type == "list['unsigned long']") { + insert_converter(mod, cname, py_to_vulong, out_pq, output); + } else if (output_type.compare(pos, std::string::npos, "int64]]") == 0 || // id. + output_type == "list[long]" || output_type == "list['long']") { + insert_converter(mod, cname, py_to_vlong, out_pq, output); + } else if (output_type.compare(pos, std::string::npos, "float32]]") == 0 || + output_type == "list[float]") { + insert_converter(mod, cname, py_to_vfloat, out_pq, output); + } else if (output_type.compare(pos, std::string::npos, "float64]]") == 0 || + output_type == "list[double]" || output_type == "list['double']") { + insert_converter(mod, cname, py_to_vdouble, out_pq, output); } else { - PyErr_Format(PyExc_TypeError, "unsupported array output type \"%s\"", output_type.c_str()); + PyErr_Format( + PyExc_TypeError, "unsupported collection output type \"%s\"", output_type.c_str()); return nullptr; } - } else if (output_type == "list[int]") { - auto py_in = "py" + output + "_" + cname; - mod->ph_module->transform("pyvint_" + output + "_" + cname, py_to_vint, concurrency::serial) - .input_family(product_query{product_specification::create(py_in), LAYER}) - .output_products(output); - } else if (output_type == "list[unsigned int]" || output_type == "list['unsigned int']") { - auto py_in = "py" + output + "_" + cname; - mod->ph_module->transform("pyvuint_" + output + "_" + cname, py_to_vuint, concurrency::serial) - .input_family(product_query{product_specification::create(py_in), LAYER}) - .output_products(output); - } else if (output_type == "list[long]" || output_type == "list['long']") { - auto py_in = "py" + output + "_" + cname; - mod->ph_module->transform("pyvlong_" + output + "_" + cname, py_to_vlong, concurrency::serial) - .input_family(product_query{product_specification::create(py_in), LAYER}) - .output_products(output); - } else if (output_type == "list[unsigned long]" || output_type == "list['unsigned long']") { - auto py_in = "py" + output + "_" + cname; - mod->ph_module->transform("pyvulong_" + output + "_" + cname, py_to_vulong, concurrency::serial) - .input_family(product_query{product_specification::create(py_in), LAYER}) - .output_products(output); - } else if (output_type == "list[float]") { - auto py_in = "py" + output + "_" + cname; - mod->ph_module->transform("pyvfloat_" + output + "_" + cname, py_to_vfloat, concurrency::serial) - .input_family(product_query{product_specification::create(py_in), LAYER}) - .output_products(output); - } else if (output_type == "list[double]" || output_type == "list['double']") { - auto py_in = "py" + output + "_" + cname; - mod->ph_module - ->transform("pyvdouble_" + output + "_" + cname, py_to_vdouble, concurrency::serial) - .input_family(product_query{product_specification::create(py_in), LAYER}) - .output_products(output); } else { PyErr_Format(PyExc_TypeError, "unsupported output type \"%s\"", output_type.c_str()); return nullptr; @@ -879,9 +915,10 @@ static PyObject* md_observe(py_phlex_module* mod, PyObject* args, PyObject* kwds // nodes going from C++ to PyObject* and back. std::string cname; - std::vector input_labels, input_types, output_labels, output_types; + std::vector input_queries; + std::vector input_types, output_labels, output_types; PyObject* callable = - parse_args(args, kwds, cname, input_labels, input_types, output_labels, output_types); + parse_args(args, kwds, cname, input_queries, input_types, output_labels, output_types); if (!callable) return nullptr; // error already set @@ -890,38 +927,56 @@ static PyObject* md_observe(py_phlex_module* mod, PyObject* args, PyObject* kwds return nullptr; } - if (!insert_input_converters(mod, cname, input_labels, input_types)) { + if (!insert_input_converters(mod, cname, input_queries, input_types)) { Py_DECREF(callable); return nullptr; // error already set } // register Python observer - if (input_labels.size() == 1) { - auto* pyc = new py_callback_1v{callable}; // id. + auto pq0 = input_queries[0]; + std::string c0 = input_converter_name(cname, 0); + std::string suff0 = "py_" + std::string{static_cast(*pq0.suffix)}; + + switch (input_queries.size()) { + case 1: { + auto* pyc = new py_callback_1v{callable}; mod->ph_module->observe(cname, *pyc, concurrency::serial) .input_family( - product_query{product_specification::create(cname + "_" + input_labels[0] + "py"), LAYER}); - Py_DECREF(callable); - } else if (input_labels.size() == 2) { + product_query{.creator = identifier(c0), .layer = pq0.layer, .suffix = identifier(suff0)}); + break; + } + case 2: { auto* pyc = new py_callback_2v{callable}; + auto pq1 = input_queries[1]; + std::string c1 = input_converter_name(cname, 1); + std::string suff1 = "py_" + std::string{static_cast(*pq1.suffix)}; mod->ph_module->observe(cname, *pyc, concurrency::serial) .input_family( - product_query{product_specification::create(cname + "_" + input_labels[0] + "py"), LAYER}, - product_query{product_specification::create(cname + "_" + input_labels[1] + "py"), LAYER}); - Py_DECREF(callable); - } else if (input_labels.size() == 3) { + product_query{.creator = identifier(c0), .layer = pq0.layer, .suffix = identifier(suff0)}, + product_query{.creator = identifier(c1), .layer = pq1.layer, .suffix = identifier(suff1)}); + break; + } + case 3: { auto* pyc = new py_callback_3v{callable}; + auto pq1 = input_queries[1]; + std::string c1 = input_converter_name(cname, 1); + std::string suff1 = "py_" + std::string{static_cast(*pq1.suffix)}; + auto pq2 = input_queries[2]; + std::string c2 = input_converter_name(cname, 2); + std::string suff2 = "py_" + std::string{static_cast(*pq2.suffix)}; mod->ph_module->observe(cname, *pyc, concurrency::serial) .input_family( - product_query{product_specification::create(cname + "_" + input_labels[0] + "py"), LAYER}, - product_query{product_specification::create(cname + "_" + input_labels[1] + "py"), LAYER}, - product_query{product_specification::create(cname + "_" + input_labels[2] + "py"), LAYER}); - Py_DECREF(callable); - } else { + product_query{.creator = identifier(c0), .layer = pq0.layer, .suffix = identifier(suff0)}, + product_query{.creator = identifier(c1), .layer = pq1.layer, .suffix = identifier(suff1)}, + product_query{.creator = identifier(c2), .layer = pq2.layer, .suffix = identifier(suff2)}); + break; + } + default: { PyErr_SetString(PyExc_TypeError, "unsupported number of inputs"); Py_DECREF(callable); return nullptr; } + } Py_RETURN_NONE; } diff --git a/plugins/python/src/pymodule.cpp b/plugins/python/src/pymodule.cpp index c8185ea42..a161f32cb 100644 --- a/plugins/python/src/pymodule.cpp +++ b/plugins/python/src/pymodule.cpp @@ -25,8 +25,8 @@ PHLEX_REGISTER_ALGORITHMS(m, config) if (mod) { PyObject* reg = PyObject_GetAttrString(mod, "PHLEX_REGISTER_ALGORITHMS"); if (reg) { - PyObject* pym = wrap_module(&m); - PyObject* pyconfig = wrap_configuration(&config); + PyObject* pym = wrap_module(m); + PyObject* pyconfig = wrap_configuration(config); if (pym && pyconfig) { PyObject* res = PyObject_CallFunctionObjArgs(reg, pym, pyconfig, nullptr); Py_XDECREF(res); diff --git a/plugins/python/src/wrap.hpp b/plugins/python/src/wrap.hpp index f0818dd16..0f5e9385e 100644 --- a/plugins/python/src/wrap.hpp +++ b/plugins/python/src/wrap.hpp @@ -28,7 +28,7 @@ namespace phlex::experimental { // Create dict-like access to the configuration from Python. // Returns a new reference. - PyObject* wrap_configuration(configuration const* config); + PyObject* wrap_configuration(configuration const& config); // Python wrapper for Phlex configuration extern PyTypeObject PhlexConfig_Type; @@ -37,7 +37,7 @@ namespace phlex::experimental { // Phlex' Module wrapper to register algorithms typedef module_graph_proxy phlex_module_t; // Returns a new reference. - PyObject* wrap_module(phlex_module_t* mod); + PyObject* wrap_module(phlex_module_t& mod); // Python wrapper for Phlex modules extern PyTypeObject PhlexModule_Type; diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 6b418e98b..8724e21b8 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -255,7 +255,7 @@ add_subdirectory(plugins) add_subdirectory(utilities) add_subdirectory(mock-workflow) add_subdirectory(demo-giantdata) -# add_subdirectory(python) +add_subdirectory(python) if(PHLEX_USE_FORM) add_subdirectory(form) diff --git a/test/python/CMakeLists.txt b/test/python/CMakeLists.txt index 48b06b678..20fd97abd 100644 --- a/test/python/CMakeLists.txt +++ b/test/python/CMakeLists.txt @@ -204,7 +204,7 @@ add_test( ) set_tests_properties( py:failure - PROPERTIES PASS_REGULAR_EXPRESSION "property \"input\" does not exist" + PROPERTIES PASS_REGULAR_EXPRESSION "failed to retrieve property \"input\"" ) list(APPEND ACTIVE_PY_CPHLEX_TESTS py:failure) diff --git a/test/python/all_config.py b/test/python/all_config.py index 9e6c00d6a..aeee3b373 100644 --- a/test/python/all_config.py +++ b/test/python/all_config.py @@ -42,6 +42,29 @@ def __init__(self, config): assert config["some_floats"] == (3.1415, 2.71828) assert config["some_strings"] == ("aap", "noot", "mies") + # aggregate type + assert len(config["some_object"]) == 2 + assert config["some_object"]["here"] == "here" + assert config["some_object"]["there"] == "there" + + assert len(config["some_objects"]) == 3 + expected = [ + {'a': 'b', 'c': 'd', 'e': 'f'}, + {'g': 'h', 'i': 'j', 'k': 'l'}, + {'m': 'n', 'o': 'p', 'q': 'r'}, + ] + for i in range(3): + assert config["some_objects"][i] == expected[i] + + # special case of empty collection + assert config["empty"] == () + + try: + config[42] # should raise + assert not "did not raise TypeError" + except TypeError: + pass # all good as exception was raised + def __call__(self, i: int, j: int) -> None: """Dummy routine to do something. diff --git a/test/python/check_sys_path.py b/test/python/check_sys_path.py index 9d7a7acc4..89a17c907 100644 --- a/test/python/check_sys_path.py +++ b/test/python/check_sys_path.py @@ -39,6 +39,8 @@ def __call__(self, i: int) -> None: f"{sys.prefix}/lib/python{sys.version_info.major}." f"{sys.version_info.minor}/site-packages" ) + print("HAVE:", sys.path, flush=True) + print(venv_site_packages, flush=True) assert any(p == venv_site_packages for p in sys.path) diff --git a/test/python/pyadd.jsonnet b/test/python/pyadd.jsonnet index ec08d2719..25dd4209d 100644 --- a/test/python/pyadd.jsonnet +++ b/test/python/pyadd.jsonnet @@ -13,12 +13,29 @@ modules: { pyadd: { py: 'adder', - input: ['i', 'j'], + input: [ + { + creator: 'input', + layer: 'event', + suffix: 'i', + }, + { + creator: 'input', + layer: 'event', + suffix: 'j', + }, + ], output: ['sum'], }, pyverify: { py: 'verify', - input: ['sum'], + input: [ + { + creator: 'iadd', + layer: 'event', + suffix: 'sum', + }, + ], sum_total: 1, }, }, diff --git a/test/python/pybadbool.jsonnet b/test/python/pybadbool.jsonnet index 97bd2821f..d56d86fbc 100644 --- a/test/python/pybadbool.jsonnet +++ b/test/python/pybadbool.jsonnet @@ -14,12 +14,12 @@ test_bad_bool: { py: 'test_callbacks', mode: 'bad_bool', - input: ['i'], + input: [{ creator: 'input', layer: 'event', suffix: 'i' }], output: ['out_bool'], }, verify_bool: { py: 'verify', - input: ['out_bool'], + input: [{ creator: 'test_bad_bool', layer: 'event', suffix: 'out_bool' }], expected_bool: true, }, }, diff --git a/test/python/pybadint.jsonnet b/test/python/pybadint.jsonnet index 7bfbb8659..1d7f751d8 100644 --- a/test/python/pybadint.jsonnet +++ b/test/python/pybadint.jsonnet @@ -14,7 +14,7 @@ test_bad_long: { py: 'test_callbacks', mode: 'bad_long', - input: ['i'], + input: [{ creator: 'input', layer: 'event', suffix: 'i' }], output: ['out_long'], }, }, diff --git a/test/python/pybaduint.jsonnet b/test/python/pybaduint.jsonnet index 0616e7fdc..e1044da5e 100644 --- a/test/python/pybaduint.jsonnet +++ b/test/python/pybaduint.jsonnet @@ -14,7 +14,7 @@ test_bad_uint: { py: 'test_callbacks', mode: 'bad_uint', - input: ['i'], + input: [{ creator: 'input', layer: 'event', suffix: 'i' }], output: ['out_uint'], }, }, diff --git a/test/python/pycallback3.jsonnet b/test/python/pycallback3.jsonnet index c6893fd80..b9f712cb6 100644 --- a/test/python/pycallback3.jsonnet +++ b/test/python/pycallback3.jsonnet @@ -15,12 +15,18 @@ test_three_args: { py: 'test_callbacks', mode: 'three_args', - input: ['i', 'j', 'k'], + input: [ + { creator: 'input', layer: 'event', suffix: 'i' }, + { creator: 'input', layer: 'event', suffix: 'j' }, + { creator: 'input', layer: 'event', suffix: 'k' }, + ], output: ['sum_ijk'], }, verify_three: { py: 'verify', - input: ['sum_ijk'], + input: [ + { creator: 'test_three_args', layer: 'event', suffix: 'sum_ijk' }, + ], sum_total: 1, // 1 event * (0+0+0? wait, i=event_num-1. event1->0. sum=0. ) // provider generates i, j starting at 0? // cppsource4py probably uses event number. diff --git a/test/python/pyconfig.jsonnet b/test/python/pyconfig.jsonnet index e2debf327..bca4f9420 100644 --- a/test/python/pyconfig.jsonnet +++ b/test/python/pyconfig.jsonnet @@ -13,7 +13,10 @@ modules: { pyconfig: { py: 'all_config', - input: ['i', 'j'], + input: [ + { creator: 'input', layer: 'event', suffix: 'i' }, + { creator: 'input', layer: 'event', suffix: 'j' }, + ], a_bool: false, an_int: -37, a_uint: 18446744073709551616, @@ -24,6 +27,13 @@ some_uints: [18446744073709551616, 29, 137], some_floats: [3.1415, 2.71828], some_strings: ['aap', 'noot', 'mies'], + some_object: { here: 'here', there: 'there' }, + some_objects: [ + { a: 'b', c: 'd', e: 'f' }, + { g: 'h', i: 'j', k: 'l' }, + { m: 'n', o: 'p', q: 'r' }, + ], + empty: [], }, }, } diff --git a/test/python/pyfailure.jsonnet b/test/python/pyfailure.jsonnet index 9b849f4fd..6be9c52bd 100644 --- a/test/python/pyfailure.jsonnet +++ b/test/python/pyfailure.jsonnet @@ -13,7 +13,7 @@ modules: { pyadd: { py: 'adder', - //input: ['i', 'j'], # commented out to cause a failure + //input: [...], # commented out to cause a failure output: ['sum'], }, }, diff --git a/test/python/pymismatch_variant.jsonnet b/test/python/pymismatch_variant.jsonnet index 07823340b..1cd132ca1 100644 --- a/test/python/pymismatch_variant.jsonnet +++ b/test/python/pymismatch_variant.jsonnet @@ -15,7 +15,11 @@ py: 'test_callbacks', mode: 'mismatch', // Providing 3 inputs for a 2-arg function - input: ['i', 'j', 'k'], + input: [ + { creator: 'input', layer: 'event', suffix: 'i' }, + { creator: 'input', layer: 'event', suffix: 'j' }, + { creator: 'input', layer: 'event', suffix: 'k' }, + ], output: ['sum_out'], }, }, diff --git a/test/python/pyraise.jsonnet b/test/python/pyraise.jsonnet index cd08ce5b5..1055d78e0 100644 --- a/test/python/pyraise.jsonnet +++ b/test/python/pyraise.jsonnet @@ -14,7 +14,7 @@ test_exception: { py: 'test_callbacks', mode: 'exception', - input: ['i'], + input: [{ creator: 'input', layer: 'event', suffix: 'i' }], output: ['out'], }, }, diff --git a/test/python/pyreduce.jsonnet b/test/python/pyreduce.jsonnet index ddabf9873..0a563cd0b 100644 --- a/test/python/pyreduce.jsonnet +++ b/test/python/pyreduce.jsonnet @@ -13,11 +13,14 @@ modules: { pyreduce: { py: 'reducer', - input: ['i', 'j'], + input: [ + { creator: 'input', layer: 'event', suffix: 'i' }, + { creator: 'input', layer: 'event', suffix: 'j' }, + ], }, pyverify: { py: 'verify', - input: ['sum'], + input: [{ creator: 'reduce', layer: 'event', suffix: 'sum' }], sum_total: 4, }, }, diff --git a/test/python/pysyspath.jsonnet.in b/test/python/pysyspath.jsonnet.in index 22c0c13e6..8bcd0e885 100644 --- a/test/python/pysyspath.jsonnet.in +++ b/test/python/pysyspath.jsonnet.in @@ -13,7 +13,7 @@ modules: { pysyspath: { py: 'check_sys_path', - input: ['i'], + input: [{creator: 'input', layer: 'event', suffix: 'i'}], venv: '@PY_VIRTUAL_ENV_DIR@', }, }, diff --git a/test/python/pytypes.jsonnet b/test/python/pytypes.jsonnet index 4c401a1a7..6f6286a13 100644 --- a/test/python/pytypes.jsonnet +++ b/test/python/pytypes.jsonnet @@ -13,13 +13,25 @@ modules: { pytypes: { py: 'test_types', - input_float: ['f1', 'f2'], + input_float: [ + { creator: 'input', layer: 'event', suffix: 'f1' }, + { creator: 'input', layer: 'event', suffix: 'f2' }, + ], output_float: ['sum_f'], - input_double: ['d1', 'd2'], + input_double: [ + { creator: 'input', layer: 'event', suffix: 'd1' }, + { creator: 'input', layer: 'event', suffix: 'd2' }, + ], output_double: ['sum_d'], - input_uint: ['u1', 'u2'], + input_uint: [ + { creator: 'input', layer: 'event', suffix: 'u1' }, + { creator: 'input', layer: 'event', suffix: 'u2' }, + ], output_uint: ['sum_u'], - input_bool: ['b1', 'b2'], + input_bool: [ + { creator: 'input', layer: 'event', suffix: 'b1' }, + { creator: 'input', layer: 'event', suffix: 'b2' }, + ], output_bool: ['and_b'], output_vfloat: ['vec_f'], output_vdouble: ['vec_d'], diff --git a/test/python/pyvec.jsonnet b/test/python/pyvec.jsonnet index ffc775e58..52da94dc9 100644 --- a/test/python/pyvec.jsonnet +++ b/test/python/pyvec.jsonnet @@ -13,12 +13,29 @@ modules: { pysum: { py: 'sumit', - input: ['i', 'j'], + input: [ + { + creator: 'input', + layer: 'event', + suffix: 'i', + }, + { + creator: 'input', + layer: 'event', + suffix: 'j', + }, + ], output: ['sum'], }, pyverify: { py: 'verify', - input: ['sum'], + input: [ + { + creator: 'sum_array', + layer: 'event', + suffix: 'sum', + }, + ], sum_total: 1, }, }, diff --git a/test/python/pyveclists.jsonnet b/test/python/pyveclists.jsonnet index 4d09979fd..4374e4c57 100644 --- a/test/python/pyveclists.jsonnet +++ b/test/python/pyveclists.jsonnet @@ -14,17 +14,35 @@ vectypes: { py: 'vectypes', use_lists: true, - input_int32: ['i', 'j'], + input_int32: [ + { creator: 'input', layer: 'event', suffix: 'i' }, + { creator: 'input', layer: 'event', suffix: 'j' }, + ], output_int32: ['sum_int32'], - input_uint32: ['u1', 'u2'], + input_uint32: [ + { creator: 'input', layer: 'event', suffix: 'u1' }, + { creator: 'input', layer: 'event', suffix: 'u2' }, + ], output_uint32: ['sum_uint32'], - input_int64: ['l1', 'l2'], + input_int64: [ + { creator: 'input', layer: 'event', suffix: 'l1' }, + { creator: 'input', layer: 'event', suffix: 'l2' }, + ], output_int64: ['sum_int64'], - input_uint64: ['ul1', 'ul2'], + input_uint64: [ + { creator: 'input', layer: 'event', suffix: 'ul1' }, + { creator: 'input', layer: 'event', suffix: 'ul2' }, + ], output_uint64: ['sum_uint64'], - input_float32: ['f1', 'f2'], + input_float32: [ + { creator: 'input', layer: 'event', suffix: 'f1' }, + { creator: 'input', layer: 'event', suffix: 'f2' }, + ], output_float32: ['sum_float32'], - input_float64: ['d1', 'd2'], + input_float64: [ + { creator: 'input', layer: 'event', suffix: 'd1' }, + { creator: 'input', layer: 'event', suffix: 'd2' }, + ], output_float64: ['sum_float64'], }, verify_int32: { diff --git a/test/python/pyvectypes.jsonnet b/test/python/pyvectypes.jsonnet index 3740cd802..a1856b7f4 100644 --- a/test/python/pyvectypes.jsonnet +++ b/test/python/pyvectypes.jsonnet @@ -13,17 +13,35 @@ modules: { vectypes: { py: 'vectypes', - input_int32: ['i', 'j'], + input_int32: [ + { creator: 'input', layer: 'event', suffix: 'i' }, + { creator: 'input', layer: 'event', suffix: 'j' }, + ], output_int32: ['sum_int32'], - input_uint32: ['u1', 'u2'], + input_uint32: [ + { creator: 'input', layer: 'event', suffix: 'u1' }, + { creator: 'input', layer: 'event', suffix: 'u2' }, + ], output_uint32: ['sum_uint32'], - input_int64: ['l1', 'l2'], + input_int64: [ + { creator: 'input', layer: 'event', suffix: 'l1' }, + { creator: 'input', layer: 'event', suffix: 'l2' }, + ], output_int64: ['sum_int64'], - input_uint64: ['ul1', 'ul2'], + input_uint64: [ + { creator: 'input', layer: 'event', suffix: 'ul1' }, + { creator: 'input', layer: 'event', suffix: 'ul2' }, + ], output_uint64: ['sum_uint64'], - input_float32: ['f1', 'f2'], + input_float32: [ + { creator: 'input', layer: 'event', suffix: 'f1' }, + { creator: 'input', layer: 'event', suffix: 'f2' }, + ], output_float32: ['sum_float32'], - input_float64: ['d1', 'd2'], + input_float64: [ + { creator: 'input', layer: 'event', suffix: 'd1' }, + { creator: 'input', layer: 'event', suffix: 'd2' }, + ], output_float64: ['sum_float64'], }, verify_int32: { diff --git a/test/python/reducer.py b/test/python/reducer.py index 855d35313..b32fe0395 100644 --- a/test/python/reducer.py +++ b/test/python/reducer.py @@ -63,19 +63,32 @@ def PHLEX_REGISTER_ALGORITHMS(m, config): Returns: None """ - # first recieve the same input x4 but return "different" output + # first receive the same input x4 but return "different" output for i in range(4): m.transform( add, name="reduce%d" % i, input_family=config["input"], output_products=["sum%d" % i] ) # now reduce them pair-wise + input_family01 = [ + {"creator": "reduce0", "layer": "event", "suffix": "sum0"}, + {"creator": "reduce1", "layer": "event", "suffix": "sum1"}, + ] m.transform( - add_sum01, name="reduce01", input_family=["sum0", "sum1"], output_products=["sum01"] + add_sum01, name="reduce01", input_family=input_family01, output_products=["sum01"] ) + + input_family01 = [ + {"creator": "reduce2", "layer": "event", "suffix": "sum2"}, + {"creator": "reduce3", "layer": "event", "suffix": "sum3"}, + ] m.transform( - add_sum23, name="reduce23", input_family=["sum2", "sum3"], output_products=["sum23"] + add_sum23, name="reduce23", input_family=input_family01, output_products=["sum23"] ) # once more (and the configuration will add a verifier) - m.transform(add_final, name="reduce", input_family=["sum01", "sum23"], output_products=["sum"]) + input_family_final = [ + {"creator": "reduce01", "layer": "event", "suffix": "sum01"}, + {"creator": "reduce23", "layer": "event", "suffix": "sum23"}, + ] + m.transform(add_final, name="reduce", input_family=input_family_final, output_products=["sum"]) diff --git a/test/python/sumit.py b/test/python/sumit.py index be4940141..17eaef93c 100644 --- a/test/python/sumit.py +++ b/test/python/sumit.py @@ -63,4 +63,8 @@ def PHLEX_REGISTER_ALGORITHMS(m, config): None """ m.transform(collectify, input_family=config["input"], output_products=["my_pyarray"]) - m.transform(sum_array, input_family=["my_pyarray"], output_products=config["output"]) + m.transform(sum_array, + input_family=[ + {"creator" : "collectify", "layer" : "event", "suffix" : "my_pyarray"} + ], + output_products=config["output"]) diff --git a/test/python/test_coverage.py b/test/python/test_coverage.py index 40f890899..33f82d33b 100644 --- a/test/python/test_coverage.py +++ b/test/python/test_coverage.py @@ -42,10 +42,14 @@ def PHLEX_REGISTER_ALGORITHMS(m, config): """Register algorithms.""" # We need to transform scalar inputs to lists first # i, f1, d1 come from cppsource4py - m.transform(collect_int, input_family=["i"], output_products=["l_int"]) - m.transform(collect_float, input_family=["f1"], output_products=["l_float"]) - m.transform(collect_double, input_family=["d1"], output_products=["l_double"]) - - m.transform(list_int_func, input_family=["l_int"], output_products=["sum_int"]) - m.transform(list_float_func, input_family=["l_float"], output_products=["sum_float"]) - m.transform(list_double_func, input_family=["l_double"], output_products=["sum_double"]) + tfs = ((collect_int, "input", "i", "l_int"), + (collect_float, "input", "f1", "l_float"), + (collect_double, "input", "d1", "l_double"), + (list_int_func, collect_int.__name__, "l_int", "sum_int"), + (list_float_func, collect_float.__name__, "l_float", "sum_float"), + (list_double_func, collect_double.__name__, "l_double", "sum_double") + ) + + for func, creator, suffix, output in tfs: + input_family = [{"creator": creator, "layer": "event", "suffix": suffix}] + m.transform(func, input_family=input_family, output_products=[output]) diff --git a/test/python/test_mismatch.py b/test/python/test_mismatch.py index d84f27e8c..2d188496c 100644 --- a/test/python/test_mismatch.py +++ b/test/python/test_mismatch.py @@ -10,4 +10,6 @@ def PHLEX_REGISTER_ALGORITHMS(m, config): """Register algorithms.""" # input_family has 1 element, but function takes 2 arguments # This should trigger the error in modulewrap.cpp - m.transform(mismatch_func, input_family=["a"], output_products=["sum"]) + m.transform(mismatch_func, + input_family=[{"creator": "input", "layer": "event", "suffix": "a"}], + output_products=["sum"]) diff --git a/test/python/vectypes.py b/test/python/vectypes.py index 4763621b7..de182aabb 100644 --- a/test/python/vectypes.py +++ b/test/python/vectypes.py @@ -259,7 +259,11 @@ def PHLEX_REGISTER_ALGORITHMS(m, config): ) sum_kwargs = { - "input_family": [arr_name], + "input_family": [{ + "creator": list_collect.__name__ if use_lists else arr_collect.__name__, + "layer": "event", + "suffix": arr_name + }], "output_products": config[out_key], } if sum_name: