diff --git a/phlex/app/load_module.cpp b/phlex/app/load_module.cpp index 247f6ab71..352d78a11 100644 --- a/phlex/app/load_module.cpp +++ b/phlex/app/load_module.cpp @@ -47,20 +47,45 @@ namespace phlex::experimental { } } + namespace detail { + boost::json::object adjust_config(std::string const& label, boost::json::object raw_config) + { + raw_config["module_label"] = label; + + // Automatically specify the 'pymodule' Phlex plugin if the 'py' parameter is specified + if (auto const* py = raw_config.if_contains("py")) { + if (auto const* cpp = raw_config.if_contains("cpp")) { + std::string msg = fmt::format("Both 'cpp' and 'py' parameters specified for {}", label); + if (auto const* cpp_value = cpp->if_string()) { + msg += fmt::format("\n - cpp: {}", *cpp_value); + } + if (auto const* py_value = py->if_string()) { + msg += fmt::format("\n - py: {}", *py_value); + } + throw std::runtime_error(msg); + } + raw_config["cpp"] = "pymodule"; + } + + return raw_config; + } + } + void load_module(framework_graph& g, std::string const& label, boost::json::object raw_config) { - auto const& spec = value_to(raw_config.at("plugin")); + auto const adjusted_config = detail::adjust_config(label, std::move(raw_config)); + + auto const& spec = value_to(adjusted_config.at("cpp")); auto& creator = create_module.emplace_back(plugin_loader(spec, "create_module")); - raw_config["module_label"] = label; - configuration const config{raw_config}; + configuration const config{adjusted_config}; creator(g.module_proxy(config), config); } void load_source(framework_graph& g, std::string const& label, boost::json::object raw_config) { - auto const& spec = value_to(raw_config.at("plugin")); + auto const& spec = value_to(raw_config.at("cpp")); auto& creator = create_source.emplace_back(plugin_loader(spec, "create_source")); @@ -76,7 +101,7 @@ namespace phlex::experimental { detail::next_index_t load_driver(boost::json::object const& raw_config) { configuration const config{raw_config}; - auto const& spec = config.get("plugin"); + auto const& spec = config.get("cpp"); create_driver = plugin_loader(spec, "create_driver"); return create_driver(config); } diff --git a/phlex/app/load_module.hpp b/phlex/app/load_module.hpp index 32a3fab3e..14d3ba1f4 100644 --- a/phlex/app/load_module.hpp +++ b/phlex/app/load_module.hpp @@ -9,6 +9,12 @@ #include namespace phlex::experimental { + namespace detail { + // Adjust_config adds the module_label as a parameter, and it checks if the 'py' + // parameter exists, inserting the 'cpp: "pymodule"' configuration if necessary. + boost::json::object adjust_config(std::string const& label, boost::json::object raw_config); + } + void load_module(framework_graph& g, std::string const& label, boost::json::object config); void load_source(framework_graph& g, std::string const& label, boost::json::object config); detail::next_index_t load_driver(boost::json::object const& config); diff --git a/plugins/python/src/pymodule.cpp b/plugins/python/src/pymodule.cpp index c27f9e6ad..ef3123c01 100644 --- a/plugins/python/src/pymodule.cpp +++ b/plugins/python/src/pymodule.cpp @@ -22,7 +22,7 @@ PHLEX_EXPERIMENTAL_REGISTER_ALGORITHMS(m, config) PyGILRAII g; - std::string modname = config.get("pyplugin"); + std::string modname = config.get("py"); PyObject* mod = PyImport_ImportModule(modname.c_str()); if (mod) { PyObject* reg = PyObject_GetAttrString(mod, "PHLEX_EXPERIMENTAL_REGISTER_ALGORITHMS"); diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index df6b645e7..30353586f 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -6,6 +6,15 @@ include(CetTest) cet_test_env(SPDLOG_LEVEL=debug) cet_test(concepts SOURCE concepts.cpp LIBRARIES phlex::core) +cet_test( + adjust_config + USE_CATCH2_MAIN + SOURCE + adjust_config.cpp + LIBRARIES + run_phlex + Boost::json + ) cet_test(config_test USE_CATCH2_MAIN SOURCE configuration.cpp LIBRARIES phlex::configuration ) diff --git a/test/adjust_config.cpp b/test/adjust_config.cpp new file mode 100644 index 000000000..6cd0d6db6 --- /dev/null +++ b/test/adjust_config.cpp @@ -0,0 +1,61 @@ +#include "phlex/app/load_module.hpp" + +#include "catch2/catch_test_macros.hpp" +#include "catch2/matchers/catch_matchers_string.hpp" + +#include "boost/json.hpp" + +using namespace phlex::experimental::detail; + +TEST_CASE("Adjust empty config", "[config]") +{ + auto const config = adjust_config("empty", {}); + CHECK(config.at("module_label").as_string() == "empty"); +} + +TEST_CASE("Adjust config with py only", "[config]") +{ + boost::json::object obj; + obj["py"] = "my_python_module"; + + auto const config = adjust_config("", std::move(obj)); + CHECK(config.at("py").as_string() == "my_python_module"); + CHECK(config.at("cpp").as_string() == "pymodule"); +} + +TEST_CASE("Both py and cpp specified as strings", "[config]") +{ + boost::json::object obj; + obj["py"] = "my_python_module"; + obj["cpp"] = "my_other_python_phlex_module"; + + auto const err_msg = R"""(Both 'cpp' and 'py' parameters specified for malformed1 + - cpp: my_other_python_phlex_module + - py: my_python_module)"""; + CHECK_THROWS_WITH(adjust_config("malformed1", std::move(obj)), + Catch::Matchers::ContainsSubstring(err_msg)); +} + +TEST_CASE("Both py and cpp specified, py as string", "[config]") +{ + boost::json::object obj; + obj["py"] = "my_python_module"; + obj["cpp"] = 1; + + auto const err_msg = R"""(Both 'cpp' and 'py' parameters specified for malformed2 + - py: my_python_module)"""; + CHECK_THROWS_WITH(adjust_config("malformed2", std::move(obj)), + Catch::Matchers::ContainsSubstring(err_msg)); +} + +TEST_CASE("Both py and cpp specified, cpp as string", "[config]") +{ + boost::json::object obj; + obj["py"] = 2; + obj["cpp"] = "my_other_python_phlex_module"; + + auto const err_msg = R"""(Both 'cpp' and 'py' parameters specified for malformed3 + - cpp: my_other_python_phlex_module)"""; + CHECK_THROWS_WITH(adjust_config("malformed3", std::move(obj)), + Catch::Matchers::ContainsSubstring(err_msg)); +} diff --git a/test/benchmarks/benchmark-01.jsonnet b/test/benchmarks/benchmark-01.jsonnet index 14ef01f5b..3859967dd 100644 --- a/test/benchmarks/benchmark-01.jsonnet +++ b/test/benchmarks/benchmark-01.jsonnet @@ -1,18 +1,18 @@ { driver: { - plugin: 'generate_layers', + cpp: 'generate_layers', layers: { event: { total: 100000 } } }, sources: { provider: { - plugin: 'benchmarks_provider' + cpp: 'benchmarks_provider' } }, modules: { a_creator: { - plugin: 'last_index', + cpp: 'last_index', }, }, } diff --git a/test/benchmarks/benchmark-02.jsonnet b/test/benchmarks/benchmark-02.jsonnet index db3764ca2..968db98d1 100644 --- a/test/benchmarks/benchmark-02.jsonnet +++ b/test/benchmarks/benchmark-02.jsonnet @@ -1,22 +1,22 @@ { driver: { - plugin: 'generate_layers', + cpp: 'generate_layers', layers: { event: { total: 100000 } } }, sources: { provider: { - plugin: 'benchmarks_provider' + cpp: 'benchmarks_provider' } }, modules: { a1_creator: { - plugin: 'last_index', + cpp: 'last_index', product_name: "a1" }, a2_creator: { - plugin: 'last_index', + cpp: 'last_index', product_name: "a2" }, }, diff --git a/test/benchmarks/benchmark-03.jsonnet b/test/benchmarks/benchmark-03.jsonnet index 71a36187a..bba3585cd 100644 --- a/test/benchmarks/benchmark-03.jsonnet +++ b/test/benchmarks/benchmark-03.jsonnet @@ -1,18 +1,18 @@ { driver: { - plugin: 'generate_layers', + cpp: 'generate_layers', layers: { event: { total: 100000 } } }, sources: { provider: { - plugin: 'benchmarks_provider' + cpp: 'benchmarks_provider' } }, modules: { read_id: { - plugin: 'read_id', + cpp: 'read_id', }, }, } diff --git a/test/benchmarks/benchmark-04.jsonnet b/test/benchmarks/benchmark-04.jsonnet index a8676307e..5d5a37020 100644 --- a/test/benchmarks/benchmark-04.jsonnet +++ b/test/benchmarks/benchmark-04.jsonnet @@ -1,21 +1,21 @@ { driver: { - plugin: 'generate_layers', + cpp: 'generate_layers', layers: { event: { total: 100000 } } }, sources: { provider: { - plugin: 'benchmarks_provider' + cpp: 'benchmarks_provider' } }, modules: { a_creator: { - plugin: 'last_index', + cpp: 'last_index', }, read_index: { - plugin: 'read_index', + cpp: 'read_index', consumes: { product: 'a', layer: "event" } }, }, diff --git a/test/benchmarks/benchmark-05.jsonnet b/test/benchmarks/benchmark-05.jsonnet index 98eafad47..495995e1e 100644 --- a/test/benchmarks/benchmark-05.jsonnet +++ b/test/benchmarks/benchmark-05.jsonnet @@ -1,26 +1,26 @@ { driver: { - plugin: 'generate_layers', + cpp: 'generate_layers', layers: { event: { total: 100000 } } }, sources: { provider: { - plugin: 'benchmarks_provider' + cpp: 'benchmarks_provider' } }, modules: { b_creator: { - plugin: 'last_index', + cpp: 'last_index', produces: 'b', }, c_creator: { - plugin: 'last_index', + cpp: 'last_index', produces: 'c', }, d: { - plugin: 'verify_difference', + cpp: 'verify_difference', expected: 0 }, }, diff --git a/test/benchmarks/benchmark-06.jsonnet b/test/benchmarks/benchmark-06.jsonnet index 2d74f29a2..ee6c9a50c 100644 --- a/test/benchmarks/benchmark-06.jsonnet +++ b/test/benchmarks/benchmark-06.jsonnet @@ -1,27 +1,27 @@ { driver: { - plugin: 'generate_layers', + cpp: 'generate_layers', layers: { event: { total: 100000 } } }, sources: { provider: { - plugin: 'benchmarks_provider' + cpp: 'benchmarks_provider' } }, modules: { a_creator: { - plugin: 'last_index', + cpp: 'last_index', }, b_creator: { - plugin: 'plus_one', + cpp: 'plus_one', }, c_creator: { - plugin: 'plus_101', + cpp: 'plus_101', }, d: { - plugin: 'verify_difference', + cpp: 'verify_difference', }, }, } diff --git a/test/benchmarks/benchmark-07.jsonnet b/test/benchmarks/benchmark-07.jsonnet index 2807cb2df..bfaf895ef 100644 --- a/test/benchmarks/benchmark-07.jsonnet +++ b/test/benchmarks/benchmark-07.jsonnet @@ -1,32 +1,32 @@ { driver: { - plugin: 'generate_layers', + cpp: 'generate_layers', layers: { event: { total: 100000 } } }, sources: { provider: { - plugin: 'benchmarks_provider' + cpp: 'benchmarks_provider' } }, modules: { even_filter: { - plugin: 'accept_even_ids', + cpp: 'accept_even_ids', input: { product: 'id', layer: 'event' }, }, b_creator: { - plugin: 'last_index', + cpp: 'last_index', when: ['even_filter:accept_even_ids'], produces: 'b', }, c_creator: { - plugin: 'last_index', + cpp: 'last_index', when: ['even_filter:accept_even_ids'], produces: 'c', }, d: { - plugin: 'verify_difference', + cpp: 'verify_difference', expected: 0 }, }, diff --git a/test/benchmarks/benchmark-08.jsonnet b/test/benchmarks/benchmark-08.jsonnet index 368600737..2de2e1cbc 100644 --- a/test/benchmarks/benchmark-08.jsonnet +++ b/test/benchmarks/benchmark-08.jsonnet @@ -2,32 +2,32 @@ local max_number = 100000; { driver: { - plugin: 'generate_layers', + cpp: 'generate_layers', layers: { event: { total: max_number } } }, sources: { provider: { - plugin: 'benchmarks_provider' + cpp: 'benchmarks_provider' } }, modules: { a_creator: { - plugin: 'last_index', + cpp: 'last_index', produces: 'a', }, even_filter: { - plugin: 'accept_even_numbers', + cpp: 'accept_even_numbers', consumes: { product: 'a', layer: 'event' } }, fibonacci_filter: { - plugin: 'accept_fibonacci_numbers', + cpp: 'accept_fibonacci_numbers', consumes: { product: 'a', layer: "event" }, max_number: max_number, }, d: { - plugin: 'verify_even_fibonacci_numbers', + cpp: 'verify_even_fibonacci_numbers', when: ['even_filter:accept_even_numbers', 'fibonacci_filter:accept'], consumes: { product: 'a', layer: "event" }, max_number: max_number, diff --git a/test/benchmarks/benchmark-09.jsonnet b/test/benchmarks/benchmark-09.jsonnet index 20f63dea9..66fc935aa 100644 --- a/test/benchmarks/benchmark-09.jsonnet +++ b/test/benchmarks/benchmark-09.jsonnet @@ -1,28 +1,28 @@ { driver: { - plugin: 'generate_layers', + cpp: 'generate_layers', layers: { event: { total: 100000 } } }, sources: { provider: { - plugin: 'benchmarks_provider' + cpp: 'benchmarks_provider' } }, modules: { a_creator: { - plugin: 'last_index', + cpp: 'last_index', }, b_creator: { - plugin: 'plus_one', + cpp: 'plus_one', }, even_filter: { - plugin: 'accept_even_numbers', + cpp: 'accept_even_numbers', consumes: { product: 'a', layer: "event" } }, d: { - plugin: 'read_index', + cpp: 'read_index', when: ['even_filter:accept_even_numbers'], consumes: { product: 'b', layer: "event" } }, diff --git a/test/form/form_test.jsonnet b/test/form/form_test.jsonnet index 121545e5a..9527a1d51 100644 --- a/test/form/form_test.jsonnet +++ b/test/form/form_test.jsonnet @@ -1,21 +1,21 @@ { driver: { - plugin: 'generate_layers', + cpp: 'generate_layers', layers: { event: { total: 10 } } }, sources: { provider: { - plugin: 'ij_source' + cpp: 'ij_source' } }, modules: { add: { - plugin: 'module', + cpp: 'module', }, form_output: { - plugin: 'form_module', + cpp: 'form_module', products: ["sum"], }, }, diff --git a/test/max-parallelism/check_parallelism_default.jsonnet.in b/test/max-parallelism/check_parallelism_default.jsonnet.in index 9424f96eb..1abf6ff75 100644 --- a/test/max-parallelism/check_parallelism_default.jsonnet.in +++ b/test/max-parallelism/check_parallelism_default.jsonnet.in @@ -1,15 +1,15 @@ { driver: { - plugin: 'generate_layers', + cpp: 'generate_layers', }, sources: { parallelism_provider: { - plugin: 'provide_parallelism', + cpp: 'provide_parallelism', } }, modules: { verify: { - plugin: 'check_parallelism', + cpp: 'check_parallelism', expected_parallelism: @NPROC@, }, }, diff --git a/test/mock-workflow/G4Stage1.libsonnet b/test/mock-workflow/G4Stage1.libsonnet index 39aa200ff..df22de141 100644 --- a/test/mock-workflow/G4Stage1.libsonnet +++ b/test/mock-workflow/G4Stage1.libsonnet @@ -3,7 +3,7 @@ local generators = import 'SinglesGen.libsonnet'; { largeant: { - plugin: 'largeant', + cpp: 'largeant', duration_usec: 156, # Typical: 15662051 inputs: [ev.event_product(f + "/MCTruths") for f in std.objectFields(generators)], outputs: ["ParticleAncestryMap", "Assns", "SimEnergyDeposits", "AuxDetHits", "MCParticles"], diff --git a/test/mock-workflow/G4Stage2.libsonnet b/test/mock-workflow/G4Stage2.libsonnet index 26881ae07..535e90a77 100644 --- a/test/mock-workflow/G4Stage2.libsonnet +++ b/test/mock-workflow/G4Stage2.libsonnet @@ -3,13 +3,13 @@ local g4stage1 = import 'G4Stage1.libsonnet'; { IonAndScint: { - plugin: 'ion_and_scint', + cpp: 'ion_and_scint', duration_usec: 546, # Typical: 5457973 inputs: [ev.event_product(f + "/SimEnergyDeposits") for f in std.objectFields(g4stage1)], outputs: ["SimEnergyDeposits", "SimEnergyDeposits_priorSCE"], }, PDFastSim: { - plugin: 'pd_fast_sim', + cpp: 'pd_fast_sim', duration_usec: 69, # Typical: 69681950 inputs: [ev.event_product('SimEnergyDeposits_priorSCE')], outputs: ['SimPhotonLites', 'OpDetBacktrackerRecords'], diff --git a/test/mock-workflow/SinglesGen.libsonnet b/test/mock-workflow/SinglesGen.libsonnet index cad274b25..de065307f 100644 --- a/test/mock-workflow/SinglesGen.libsonnet +++ b/test/mock-workflow/SinglesGen.libsonnet @@ -2,37 +2,37 @@ local ev = import 'event_product.libsonnet'; { rn222: { - plugin: "MC_truth_algorithm", + cpp: "MC_truth_algorithm", duration_usec: 39, inputs: [ev.event_product("id")], outputs: ["MCTruths"] }, ar39: { - plugin: "MC_truth_algorithm", + cpp: "MC_truth_algorithm", duration_usec: 12410, inputs: [ev.event_product("id")], outputs: ["MCTruths"] }, cosmicgenerator: { - plugin: "MC_truth_algorithm", + cpp: "MC_truth_algorithm", duration_usec: 492, # Typical: 4926215 inputs: [ev.event_product("id")], outputs: ["MCTruths"] }, kr85: { - plugin: "MC_truth_algorithm", + cpp: "MC_truth_algorithm", duration_usec: 1643, inputs: [ev.event_product("id")], outputs: ["MCTruths"] }, generator: { - plugin: "three_tuple_algorithm", + cpp: "three_tuple_algorithm", duration_usec: 69616, inputs: [ev.event_product("id")], outputs: ["MCTruths", "BeamEvents", "beamsims"] }, ar42: { - plugin: "MC_truth_algorithm", + cpp: "MC_truth_algorithm", duration_usec: 148, inputs: [ev.event_product("id")], outputs: ["MCTruths"] diff --git a/test/mock-workflow/mock-workflow.jsonnet b/test/mock-workflow/mock-workflow.jsonnet index 20c7f48f4..c4c15adaf 100644 --- a/test/mock-workflow/mock-workflow.jsonnet +++ b/test/mock-workflow/mock-workflow.jsonnet @@ -4,14 +4,14 @@ local g4stage2 = import 'G4Stage2.libsonnet'; { driver: { - plugin: 'generate_layers', + cpp: 'generate_layers', layers: { event: { total: 1 } } }, sources: { provider: { - plugin: 'id_provider' + cpp: 'id_provider' } }, modules: singlesgen + g4stage1 + g4stage2, diff --git a/test/plugins/add.jsonnet b/test/plugins/add.jsonnet index 85c6558f8..08c3637e0 100644 --- a/test/plugins/add.jsonnet +++ b/test/plugins/add.jsonnet @@ -1,21 +1,21 @@ { driver: { - plugin: 'generate_layers', + cpp: 'generate_layers', layers: { event: { parent: "job", total: 10, starting_number: 1 } } }, sources: { provider: { - plugin: 'ij_source' + cpp: 'ij_source' } }, modules: { add: { - plugin: 'module', + cpp: 'module', }, output: { - plugin: 'output', + cpp: 'output', }, }, } diff --git a/test/python/pyadd.jsonnet b/test/python/pyadd.jsonnet index 85441647f..c54c8eabb 100644 --- a/test/python/pyadd.jsonnet +++ b/test/python/pyadd.jsonnet @@ -1,25 +1,23 @@ { driver: { - plugin: 'generate_layers', + cpp: 'generate_layers', layers: { event: { parent: 'job', total: 10, starting_number: 1 } } }, sources: { provider: { - plugin: 'cppsource4py', + cpp: 'cppsource4py', } }, modules: { pyadd: { - plugin: 'pymodule', - pyplugin: 'adder', + py: 'adder', input: ['i', 'j'], output: ['sum'], }, pyverify: { - plugin: 'pymodule', - pyplugin: 'verify', + py: 'verify', input: ['sum'], sum_total: 1, }, diff --git a/test/python/pyconfig.jsonnet b/test/python/pyconfig.jsonnet index 4d545ecd9..2effc8c4c 100644 --- a/test/python/pyconfig.jsonnet +++ b/test/python/pyconfig.jsonnet @@ -1,19 +1,18 @@ { driver: { - plugin: 'generate_layers', + cpp: 'generate_layers', layers: { event: { parent: 'job', total: 10, starting_number: 1 } } }, sources: { provider: { - plugin: 'cppsource4py', + cpp: 'cppsource4py', } }, modules: { pyconfig: { - plugin: 'pymodule', - pyplugin: 'all_config', + py: 'all_config', input: ['i', 'j'], a_bool: false, an_int: -37, diff --git a/test/python/pyfailure.jsonnet b/test/python/pyfailure.jsonnet index f7649e0b8..63b6c6086 100644 --- a/test/python/pyfailure.jsonnet +++ b/test/python/pyfailure.jsonnet @@ -1,19 +1,18 @@ { driver: { - plugin: 'generate_layers', + cpp: 'generate_layers', layers: { event: { parent: 'job', total: 10, starting_number: 1 } } }, sources: { provider: { - plugin: 'cppsource4py', + cpp: 'cppsource4py', } }, modules: { pyadd: { - plugin: 'pymodule', - pyplugin: 'adder', + py: 'adder', #input: ['i', 'j'], # commented out to cause a failure output: ['sum'], }, diff --git a/test/python/pyreduce.jsonnet b/test/python/pyreduce.jsonnet index 1331593a6..af5c87786 100644 --- a/test/python/pyreduce.jsonnet +++ b/test/python/pyreduce.jsonnet @@ -1,24 +1,22 @@ { driver: { - plugin: 'generate_layers', + cpp: 'generate_layers', layers: { event: { parent: 'job', total: 10, starting_number: 1 } } }, sources: { provider: { - plugin: 'cppsource4py', + cpp: 'cppsource4py', } }, modules: { pyreduce: { - plugin: 'pymodule', - pyplugin: 'reducer', + py: 'reducer', input: ['i', 'j'], }, pyverify: { - plugin: 'pymodule', - pyplugin: 'verify', + py: 'verify', input: ['sum'], sum_total: 4, }, diff --git a/test/python/pyvec.jsonnet b/test/python/pyvec.jsonnet index 5a0c55e28..29e3b9a14 100644 --- a/test/python/pyvec.jsonnet +++ b/test/python/pyvec.jsonnet @@ -1,25 +1,23 @@ { driver: { - plugin: 'generate_layers', + cpp: 'generate_layers', layers: { event: { parent: 'job', total: 10, starting_number: 1 } } }, sources: { provider: { - plugin: 'cppsource4py', + cpp: 'cppsource4py', } }, modules: { pysum: { - plugin: 'pymodule', - pyplugin: 'sumit', + py: 'sumit', input: ['i', 'j'], output: ['sum'], }, pyverify: { - plugin: 'pymodule', - pyplugin: 'verify', + py: 'verify', input: ['sum'], sum_total: 1, },