Initial changes for XDP runlist

Signed-off-by: Rahul Bramandlapalli <[email protected]>

Author: rbramand-xilinx

src/runtime_src/core/common/api/hw_context_int.h

@@ -8,6 +8,7 @@
 // This file defines implementation extensions to the XRT XCLBIN APIs.
 #include "core/include/xrt/xrt_hw_context.h"
 #include "core/include/xrt/experimental/xrt_module.h"
+#include "core/include/xrt/xrt_kernel.h"
 
 #include <cstdint>
 
@@ -70,6 +71,23 @@ get_scratchpad_mem_buf(const xrt::hw_context& hwctx, size_t size_per_col);
 void
 dump_scratchpad_mem(const xrt::hw_context& hwctx);
 
+// APIs used by XDP to register init and exit runs
+// These runs will be used to initialize AI array and collect profile/trace data
+// These runs are added to runlist at the beginning and end respectively
+XRT_CORE_COMMON_EXPORT
+void
+register_xdp_init_run(const xrt::hw_context& ctx, const xrt::run& run);
+
+XRT_CORE_COMMON_EXPORT
+void
+register_xdp_exit_run(const xrt::hw_context& ctx, const xrt::run& run);
+
+const std::vector<xrt::run>&
+get_xdp_init_runs(const xrt::hw_context& ctx);
+
+const std::vector<xrt::run>&
+get_xdp_exit_runs(const xrt::hw_context& ctx);
+
 }} // hw_context_int, xrt_core
 
 #endif

src/runtime_src/core/common/api/xrt_hw_context.cpp

@@ -12,6 +12,7 @@
 #include "core/common/time.h"
 #include "core/common/utils.h"
 #include "core/include/xrt/xrt_hw_context.h"
+#include "core/include/xrt/xrt_kernel.h"
 #include "core/include/xrt/experimental/xrt_ext.h"
 #include "core/include/xrt/experimental/xrt_module.h"
 #include "bo_int.h"
@@ -141,6 +142,11 @@ class hw_context_impl : public std::enable_shared_from_this<hw_context_impl>
   // are saved to a scratchpad memory allocated specifically for that context.
   std::once_flag m_scratchpad_init_flag; // used for thread safe lazy init of scratchpad
   xrt::bo m_scratchpad_buf;
+  // Vector of XDP runs to be added at beginning and end of runlist
+  // These runs initializes/configures AI array and collects the profile/trace data
+  std::vector<xrt::run> m_xdp_init_runs;
+  std::vector<xrt::run> m_xdp_exit_runs;
+
   std::shared_ptr<xrt_core::usage_metrics::base_logger> m_usage_logger =
       xrt_core::usage_metrics::get_usage_metrics_logger();
   bool m_elf_flow = false;
@@ -454,6 +460,33 @@ class hw_context_impl : public std::enable_shared_from_this<hw_context_impl>
     msg.append(dump_file_name);
     xrt_core::message::send(xrt_core::message::severity_level::debug, "xrt_hw_context", msg);
   }
+
+  // Callback Functions for XDP to register init and exit runs
+  // that are added to xrt::runlist at start and end respectively
+  void
+  register_xdp_init_run(const xrt::run& run)
+  {
+    m_xdp_init_runs.push_back(run);
+  }
+
+  void
+  register_xdp_exit_run(const xrt::run& run)
+  {
+    m_xdp_exit_runs.push_back(run);
+  }
+
+  // Functions to get XDP init and exit run vectors
+  const std::vector<xrt::run>&
+  get_xdp_init_runs() const
+  {
+    return m_xdp_init_runs;
+  }
+
+  const std::vector<xrt::run>&
+  get_xdp_exit_runs() const
+  {
+    return m_xdp_exit_runs;
+  }
 };
 
 } // xrt
@@ -521,6 +554,31 @@ dump_scratchpad_mem(const xrt::hw_context& hwctx)
   return hwctx.get_handle()->dump_scratchpad_mem();
 }
 
+void
+register_xdp_init_run(const xrt::hw_context& ctx, const xrt::run& run)
+{
+  ctx.get_handle()->register_xdp_init_run(run);
+}
+
+XRT_CORE_COMMON_EXPORT
+void
+register_xdp_exit_run(const xrt::hw_context& ctx, const xrt::run& run)
+{
+  ctx.get_handle()->register_xdp_exit_run(run);
+}
+
+const std::vector<xrt::run>&
+get_xdp_init_runs(const xrt::hw_context& ctx)
+{
+  return ctx.get_handle()->get_xdp_init_runs();
+}
+
+const std::vector<xrt::run>&
+get_xdp_exit_runs(const xrt::hw_context& ctx)
+{
+  return ctx.get_handle()->get_xdp_exit_runs();
+}
+
 } // xrt_core::hw_context_int
 
 ////////////////////////////////////////////////////////////////

src/runtime_src/core/common/api/xrt_kernel.cpp

@@ -3350,6 +3350,46 @@ class runlist_impl
     }
   }
 
+  void
+  add_run_helper(xrt::run run)
+  {
+    // Get the potentially throwing action out of the way first
+    auto runidx = m_runlist.size();
+    m_runlist.reserve(runidx + 1);
+    m_bos.reserve(runidx + 1);
+
+    auto execbuf = get_cmd_chain_for_run_at_index(runidx);
+    auto [cmd, pkt] = unpack(execbuf);
+    auto chain_data = get_ert_cmd_chain_data(pkt);
+    
+    auto run_impl = run.get_handle();
+    auto run_cmd = run_impl->get_cmd();
+    auto run_bo = run_cmd->get_exec_bo();
+    auto run_bo_props = run_bo->get_properties();
+
+    auto data_idx = chain_data->command_count;
+    chain_data->data[data_idx] = run_bo_props.kmhdl;
+
+    // Let shim handle binding of run_bo arguments to the command
+    // that cahins the run_bo.  This allows pinning if necessary.
+    // May throw, but so far no state change, so still safe.
+    cmd->bind_at(data_idx, run_bo, 0, run_bo_props.size); 
+
+    // Once a run object is added to a list it will be in a state that
+    // makes it impossible to add to another list or to same list
+    // twice.  This state is managed by the run object itself by
+    // recording this runlist with the run object, but it doesn't 
+    // proctect against caller manually controlling the run object,
+    // which is undefined behavior.  No exceptions after this point.
+    run_impl->set_runlist(this);  // throws or changes state of run
+
+    // Non throwing state change
+    chain_data->command_count++;
+    pkt->count += sizeof(uint64_t) / word_size; // account for added command
+    m_runlist.push_back(std::move(run));  // move of shared_ptr is noexcept
+    m_bos.push_back(run_bo);              // ptr noexcept
+  }
+
   // Wait for runlist to complete, then check each chained command
   // submitted to determine potential error within chunk.  Locate the
   // first failing command if any and mark all subsequent commands as
@@ -3454,41 +3494,19 @@ class runlist_impl
     if (m_state != state::idle)
       throw xrt_core::error("runlist must be idle before adding run objects, current state: " + state_to_string(m_state));
 
-    // Get the potentially throwing action out of the way first
-    auto runidx = m_runlist.size();
-    m_runlist.reserve(runidx + 1);
-    m_bos.reserve(runidx + 1);
-
-    auto execbuf = get_cmd_chain_for_run_at_index(runidx);
-    auto [cmd, pkt] = unpack(execbuf);
-    auto chain_data = get_ert_cmd_chain_data(pkt);
-    
-    auto run_impl = run.get_handle();
-    auto run_cmd = run_impl->get_cmd();
-    auto run_bo = run_cmd->get_exec_bo();
-    auto run_bo_props = run_bo->get_properties();
-
-    auto data_idx = chain_data->command_count;
-    chain_data->data[data_idx] = run_bo_props.kmhdl;
-
-    // Let shim handle binding of run_bo arguments to the command
-    // that cahins the run_bo.  This allows pinning if necessary.
-    // May throw, but so far no state change, so still safe.
-    cmd->bind_at(data_idx, run_bo, 0, run_bo_props.size); 
-
-    // Once a run object is added to a list it will be in a state that
-    // makes it impossible to add to another list or to same list
-    // twice.  This state is managed by the run object itself by
-    // recording this runlist with the run object, but it doesn't 
-    // proctect against caller manually controlling the run object,
-    // which is undefined behavior.  No exceptions after this point.
-    run_impl->set_runlist(this);  // throws or changes state of run
+    // Get XDP init runs registered with the hwctx and add to runlist
+    // if not already added.
+    // These runs initializes AI array for profile/trace data
+    // The list can be empty if profile/trace is not enabled
+    if (m_runlist.empty()) {
+      const auto& xdp_init_runs = xrt_core::hw_context_int::get_xdp_init_runs(m_hwctx);
+      for (const auto& init_run : xdp_init_runs) {
+        add_run_helper(init_run);
+      }
+    }
 
-    // Non throwing state change
-    chain_data->command_count++;
-    pkt->count += sizeof(uint64_t) / word_size; // account for added command
-    m_runlist.push_back(std::move(run));  // move of shared_ptr is noexcept
-    m_bos.push_back(run_bo);              // ptr noexcept
+    // Now add the original run object
+    add_run_helper(std::move(run));
   }
 
   void
@@ -3500,6 +3518,14 @@ class runlist_impl
     if (m_runlist.empty())
       return;
 
+    // Add XDP exit runs if any at the end of runlist before submitting
+    // These runs collect profile/trace data
+    // The list can be empty if profile/trace is not enabled
+    const auto& xdp_exit_runs = xrt_core::hw_context_int::get_xdp_exit_runs(m_hwctx);
+    for (const auto& exit_run : xdp_exit_runs) {
+      add_run_helper(exit_run);
+    }
+
     // Prep each run object
     for (auto& run : m_runlist)
       run.get_handle()->prep_start();
@@ -3539,6 +3565,20 @@ class runlist_impl
 
     // On succesful wait, the runlist becomes idle
     m_state = state::idle;
+
+    // Remove any XDP exit runs added during execute
+    // This is done because runlist can be reused
+    // and XDP exit runs should be added at end
+    const auto& xdp_exit_runs = xrt_core::hw_context_int::get_xdp_exit_runs(m_hwctx);
+    if (xdp_exit_runs.size() > 0) {
+      // remove the exit runs from the runlist
+      m_runlist.erase(m_runlist.end() - xdp_exit_runs.size(), m_runlist.end());
+
+      // clear the runlist associated with these exit runs
+      for (auto& run : xdp_exit_runs)
+        run.get_handle()->clear_runlist();
+    }
+
     return std::cv_status::no_timeout;
   }