Driver that generates layers according to configuration

CMakeLists.txt

@@ -180,6 +180,7 @@ if(NOT CMAKE_BUILD_TYPE)
 endif()
 
 add_subdirectory(phlex)
+add_subdirectory(plugins)
 
 if(PHLEX_USE_FORM)
   set(BUILD_SHARED_LIBS OFF) # Temporary

phlex/model/data_cell_index.cpp

@@ -2,12 +2,18 @@
 #include "phlex/utilities/hashing.hpp"
 
 #include "boost/algorithm/string.hpp"
+#include "fmt/format.h"
+#include "fmt/ranges.h"
 
 #include <algorithm>
 #include <iterator>
 #include <map>
 #include <numeric>
+#include <ranges>
 #include <stdexcept>
+#include <string>
+
+using namespace std::string_literals;
 
 namespace {
 
@@ -56,6 +62,18 @@ namespace phlex::experimental {
   }
 
   std::string const& data_cell_index::layer_name() const noexcept { return layer_name_; }
+
+  std::string data_cell_index::layer_path() const
+  {
+    std::vector layers_in_reverse{layer_name_};
+    auto next_parent = parent();
+    while (next_parent) {
+      layers_in_reverse.push_back(next_parent->layer_name());
+      next_parent = next_parent->parent();
+    }
+    return fmt::format("/{}", fmt::join(std::views::reverse(layers_in_reverse), "/"));
+  }
+
   std::size_t data_cell_index::depth() const noexcept { return depth_; }
 
   data_cell_index_ptr data_cell_index::make_child(std::size_t const data_cell_number,

phlex/model/data_cell_index.hpp

@@ -20,6 +20,7 @@ namespace phlex::experimental {
     using hash_type = std::size_t;
     data_cell_index_ptr make_child(std::size_t data_cell_number, std::string layer_name) const;
     std::string const& layer_name() const noexcept;
+    std::string layer_path() const;
     std::size_t depth() const noexcept;
     data_cell_index_ptr parent(std::string const& layer_name) const;
     data_cell_index_ptr parent() const noexcept;

plugins/CMakeLists.txt

@@ -0,0 +1,5 @@
+add_library(layer_generator layer_generator.cpp)
+target_link_libraries(layer_generator PRIVATE phlex::core)
+
+add_library(generate_layers MODULE generate_layers.cpp)
+target_link_libraries(generate_layers PRIVATE phlex::module layer_generator)

plugins/generate_layers.cpp

@@ -0,0 +1,55 @@
+// ==============================================================================================
+// See notes in plugins/layer_generator.hpp.  To configure the generate_layers driver to
+// produce the tree shown in plugins/layer_generator.hpp, the user would configure the driver
+// like (e.g.):
+//
+//   driver: {
+//     plugin: "generate_layers",
+//     layers: {
+//       spill: { parent: "job", total: 16 },
+//       CRU: { parent: "spill", total: 256 },
+//       run: { parent: "job", total: 16},
+//       APA: { parent: "run", total: 150, starting_number: 1 }
+//     }
+//   }
+//
+// Note that 'total' refers to the total number of data cells *per* parent.
+// ==============================================================================================
+
+#include "phlex/source.hpp"
+#include "plugins/layer_generator.hpp"
+
+#include "phlex/core/framework_graph.hpp"
+
+#include "fmt/ranges.h"
+#include "spdlog/spdlog.h"
+
+#include <string>
+
+using namespace phlex::experimental;
+
+namespace {
+  class generate_layers {
+  public:
+    generate_layers(configuration const& config)
+    {
+      auto const layers = config.get<configuration>("layers", {});
+      for (auto const& key : layers.keys()) {
+        auto const layer_config = layers.get<configuration>(key);
+        auto const parent = layer_config.get<std::string>("parent", "job");
+        auto const total_number = layer_config.get<unsigned int>("total");
+        auto const starting_number = layer_config.get<unsigned int>("starting_number", 0);
+        gen_.add_layer(key, {parent, total_number, starting_number});
+      }
+
+      // FIXME: Print out statement?
+    }
+
+    void next(framework_driver& driver) { gen_(driver); }
+
+  private:
+    layer_generator gen_;
+  };
+}
+
+PHLEX_EXPERIMENTAL_REGISTER_SOURCE(generate_layers)

plugins/layer_generator.cpp

@@ -0,0 +1,137 @@
+#include "plugins/layer_generator.hpp"
+#include "phlex/model/data_cell_index.hpp"
+
+#include "fmt/format.h"
+#include "spdlog/spdlog.h"
+
+#include <algorithm>
+#include <ranges>
+
+namespace phlex::experimental {
+
+  layer_generator::layer_generator()
+  {
+    // Always seed the "job" in case only the job is desired
+    parent_to_children_["/job"] = {};
+    emitted_cells_["/job"] = 0ull;
+  }
+
+  std::size_t layer_generator::emitted_cells(std::string layer_path) const
+  {
+    // Check if the count of all emitted cells is requested
+    if (layer_path.empty()) {
+      // For C++23, we can use std::ranges::fold_left
+      std::size_t total{};
+      for (auto const& [_, count] : emitted_cells_) {
+        total += count;
+      }
+      return total;
+    }
+
+    if (auto it = emitted_cells_.find(layer_path); it != emitted_cells_.end()) {
+      return emitted_cells_.at(layer_path);
+    }
+
+    throw std::runtime_error("No emitted cells corresponding to layer path '" + layer_path + "'");
+  }
+
+  std::string layer_generator::parent_path(std::string const& layer_name,
+                                           std::string const& parent_layer_spec) const
+  {
+    // Seed result with the specified parent_layer_spec
+    std::string result{"/" + parent_layer_spec};
+    std::string const* found_parent{nullptr};
+    for (auto const& path : layer_paths_) {
+      if (path.ends_with(parent_layer_spec)) {
+        if (found_parent) {
+          auto const msg =
+            fmt::format("Ambiguous: two parent layers found for data layer '{}':\n  - {}\n  - {}"
+                        "\nTo disambiguate, specify a parent layer path that is more complete.",
+                        layer_name,
+                        *found_parent,
+                        path);
+          throw std::runtime_error(msg);
+        }
+        found_parent = &path;
+      }
+    }
+    return found_parent ? *found_parent : result;
+  }
+
+  void layer_generator::maybe_rebase_layer_paths(std::string const& layer_name,
+                                                 std::string const& parent_full_path)
+  {
+    // First check if layer paths need to be rebased
+    std::vector<size_t> indices_for_rebasing;
+    for (std::size_t i = 0ull, n = layer_paths_.size(); i != n; ++i) {
+      auto const& layer = layer_paths_[i];
+      if (layer.starts_with("/" + layer_name)) {
+        indices_for_rebasing.push_back(i);
+      }
+    }
+
+    // Do the rebase
+    for (std::size_t const i : indices_for_rebasing) {
+      auto const old_layer_path = layer_paths_[i];
+      auto const& new_layer_path = layer_paths_[i] = parent_full_path + old_layer_path;
+
+      auto layer_handle = layers_.extract(old_layer_path);
+      layer_handle.key() = new_layer_path;
+      auto const old_parent_path = layer_handle.mapped().parent_layer_name;
+      auto const new_parent_path = new_layer_path.substr(0, new_layer_path.find_last_of("/"));
+      layer_handle.mapped().parent_layer_name = new_parent_path;
+      layers_.insert(std::move(layer_handle));
+
+      auto emitted_handle = emitted_cells_.extract(old_layer_path);
+      emitted_handle.key() = new_layer_path;
+      emitted_cells_.insert(std::move(emitted_handle));
+
+      auto reverse_handle = parent_to_children_.extract(old_parent_path);
+      reverse_handle.key() = new_parent_path;
+      parent_to_children_.insert(std::move(reverse_handle));
+    }
+  }
+
+  void layer_generator::add_layer(std::string layer_name, layer_spec lspec)
+  {
+    auto const parent_full_path = parent_path(layer_name, lspec.parent_layer_name);
+
+    // We need to make sure that we can distinguish between (e.g.) /events and /run/events.
+    // When a layer is added, the parent layers are also included as part of the path.
+    maybe_rebase_layer_paths(layer_name, parent_full_path);
+
+    auto full_path = parent_full_path + "/" + layer_name;
+
+    lspec.parent_layer_name = parent_full_path;
+    layers_[full_path] = std::move(lspec);
+    emitted_cells_[full_path] = 0ull;
+    parent_to_children_[parent_full_path].push_back(std::move(layer_name));
+    layer_paths_.push_back(full_path);
+  }
+
+  void layer_generator::execute(framework_driver& driver, data_cell_index_ptr index, bool recurse)
+  {
+    auto cell_it = emitted_cells_.find(index->layer_path());
+    assert(cell_it != emitted_cells_.cend());
+    ++cell_it->second;
+
+    driver.yield(index);
+
+    if (not recurse) {
+      return;
+    }
+
+    auto it = parent_to_children_.find(index->layer_path());
+    assert(it != parent_to_children_.cend());
+
+    for (auto const& child : it->second) {
+      auto const full_child_path = index->layer_path() + "/" + child;
+      bool const recurse = parent_to_children_.contains(full_child_path);
+      auto const& [_, total_per_parent, starting_value] = layers_.at(full_child_path);
+      for (unsigned int i : std::views::iota(starting_value, total_per_parent + starting_value)) {
+        execute(driver, index->make_child(i, child), recurse);
+      }
+    }
+  }
+
+}

plugins/layer_generator.hpp

@@ -0,0 +1,80 @@
+#ifndef PLUGINS_LAYER_GENERATOR_HPP
+#define PLUGINS_LAYER_GENERATOR_HPP
+
+// ==============================================================================================
+// The layer_generator class enables the creation of a tree of layers, such as
+//
+//   job
+//    │
+//    ├ spill: 16
+//    │  │
+//    │  └ CRU: 4096
+//    │
+//    └ run: 16
+//       │
+//       └ APA: 2500
+//
+// To create the above tree of layers, the following function calls could be made:
+//
+//   layer_generator gen;
+//   gen.add_layer("spill", {"job", 16});     // 16 spill data cells with job as parent
+//   gen.add_layer("CRU",  {"spill", 256});   // 256 CRU data cells per spill parent
+//   gen.add_layer("run", {"job", 16});       // 16 run data cells with job as parent
+//   gen.add_layer("APA",  {"run", 150, 1});  // 150 APA data cells per run parent
+//                                            // with first APA data cell number starting at 1
+//
+// ----------------------------------------------------------------------------------------------
+// N.B. The layer generator can create data-layer hierarchies that are trees, and not
+//      more general DAGs, where a data layer may have more than one parent.
+// ==============================================================================================
+
+#include "phlex/core/framework_graph.hpp"
+#include "phlex/model/data_cell_index.hpp"
+
+#include <string>
+
+namespace phlex::experimental {
+  struct layer_spec {
+    std::string parent_layer_name;
+    std::size_t total_per_parent_data_cell;
+    std::size_t starting_value = 0;
+  };
+
+  class layer_generator {
+  public:
+    layer_generator();
+
+    layer_generator(layer_generator const&) = delete;
+    layer_generator& operator=(layer_generator const&) = delete;
+    layer_generator(layer_generator&&) = default;
+    layer_generator& operator=(layer_generator&&) = default;
+
+    void add_layer(std::string layer_name, layer_spec lspec);
+
+    void operator()(framework_driver& driver) { execute(driver, data_cell_index::base_ptr()); }
+
+    std::size_t emitted_cells(std::string layer_path = {}) const;
+
+  private:
+    void execute(framework_driver& driver, data_cell_index_ptr index, bool recurse = true);
+    std::string parent_path(std::string const& layer_name,
+                            std::string const& parent_layer_spec) const;
+    void maybe_rebase_layer_paths(std::string const& layer_name,
+                                  std::string const& parent_full_path);
+
+    std::map<std::string, layer_spec> layers_;
+    std::map<std::string, std::size_t> emitted_cells_;
+    std::vector<std::string> layer_paths_{"/job"};
+
+    using reverse_map_t = std::map<std::string, std::vector<std::string>>;
+    reverse_map_t parent_to_children_;
+  };
+
+  // N.B. The layer_generator object must outlive any whatever uses it.
+  std::function<void(framework_driver&)> driver_for_test(layer_generator& generator)
+  {
+    return [&generator](framework_driver& driver) mutable { generator(driver); };
+  }
+}
+
+#endif // PLUGINS_LAYER_GENERATOR_HPP

scripts/normalize_coverage_lcov.py

@@ -65,7 +65,7 @@ def _is_repo_content(relative_path: Path) -> bool:
     """
     parts = relative_path.parts
     # Accept phlex/**, form/**, build-clang/**, .coverage-generated
-    if len(parts) >= 1 and parts[0] in ("phlex", "form", "build-clang"):
+    if len(parts) >= 1 and parts[0] in ("phlex", "plugins", "form", "build-clang"):
         return True
     if ".coverage-generated" in parts:
         return True