add thread_affinity_exclude_list

.gitignore

@@ -18,4 +18,6 @@
 **/__pycache__
 
 # mac os
-.DS_Store
+.DS_Store
+
+.cache

CMakeLists.txt

@@ -124,7 +124,7 @@ if (NOT EXISTS ${Hiactor_BUILD_SEASTAR_SOURCE_DIR})
   execute_process (
     COMMAND git config --local user.name hiactor
     COMMAND git config --local user.email [email protected]
-    COMMAND git am -q ${Hiactor_SOURCE_DIR}/seastar/patch/hiactor-hackings.patch
+    COMMAND git am -q ${Hiactor_SOURCE_DIR}/seastar/patch/hiactor-hackings.patch ${Hiactor_SOURCE_DIR}/seastar/patch/thread_affinity_exclude_list.patch
     WORKING_DIRECTORY ${Hiactor_BUILD_SEASTAR_SOURCE_DIR})
   if (Hiactor_COOK_DEPENDENCIES_ALL_STATIC)
     execute_process (

seastar/patch/thread_affinity_exclude_list.patch

@@ -0,0 +1,77 @@
+From 6784d8a218ba0627360dc8f24aa91d98bef26d5f Mon Sep 17 00:00:00 2001
+From: Lixin Wei <[email protected]>
+Date: Fri, 16 Aug 2024 14:49:38 +0800
+Subject: [PATCH] Add schedule-affinity-exclude-list
+
+---
+ src/core/reactor.cc  | 8 ++++++--
+ src/core/resource.cc | 3 ---
+ 2 files changed, 6 insertions(+), 5 deletions(-)
+
+diff --git a/src/core/reactor.cc b/src/core/reactor.cc
+index 22dc21b1..5069a65e 100644
+--- a/src/core/reactor.cc
++++ b/src/core/reactor.cc
+@@ -19,6 +19,7 @@
+  * Copyright 2014 Cloudius Systems
+  */
+
++#include <unordered_set>
+ #define __user /* empty */  // for xfs includes, below
+
+ #include <cinttypes>
+@@ -3557,6 +3558,7 @@ smp::get_options_description()
+         ("hugepages", bpo::value<std::string>(), "path to accessible hugetlbfs mount (typically /dev/hugepages/something)")
+         ("lock-memory", bpo::value<bool>(), "lock all memory (prevents swapping)")
+         ("thread-affinity", bpo::value<bool>()->default_value(true), "pin threads to their cpus (disable for overprovisioning)")
++        ("thread-affinity-exclude-list", bpo::value<std::vector<unsigned>>()->composing(), "don't pin these threads.")
+ #ifdef SEASTAR_HAVE_HWLOC
+         ("num-io-queues", bpo::value<unsigned>(), "Number of IO queues. Each IO unit will be responsible for a fraction of the IO requests. Defaults to the number of threads")
+         ("num-io-groups", bpo::value<unsigned>(), "Number of IO groups. Each IO group will be responsible for a fraction of the IO requests. Defaults to the number of NUMA nodes")
+@@ -3920,6 +3922,7 @@ void smp::configure(boost::program_options::variables_map configuration, reactor
+     _using_dpdk = configuration.count("dpdk-pmd");
+ #endif
+     auto thread_affinity = configuration["thread-affinity"].as<bool>();
++    auto thread_affinity_exclude_list = configuration["thread-affinity-exclude-list"].as<std::vector<unsigned>>();
+     if (configuration.count("overprovisioned")
+            && configuration["thread-affinity"].defaulted()) {
+         thread_affinity = false;
+@@ -4126,18 +4129,19 @@ void smp::configure(boost::program_options::variables_map configuration, reactor
+     _all_event_loops_done.emplace(smp::count);
+
+     auto backend_selector = configuration["reactor-backend"].as<reactor_backend_selector>();
++    std::unordered_set<unsigned> thread_affinity_exclude_set(thread_affinity_exclude_list.begin(), thread_affinity_exclude_list.end());
+
+     unsigned i;
+     auto smp_tmain = smp::_tmain;
+     for (i = 1; i < smp::count; i++) {
+         auto allocation = allocations[i];
+-        create_thread([this, smp_tmain, inited, &reactors_registered, &smp_queues_constructed, configuration, &reactors, hugepages_path, i, allocation, assign_io_queues, alloc_io_queues, thread_affinity, heapprof_enabled, mbind, backend_selector, reactor_cfg] {
++        create_thread([this, smp_tmain, inited, &reactors_registered, &smp_queues_constructed, configuration, &reactors, hugepages_path, i, allocation, assign_io_queues, alloc_io_queues, thread_affinity, &thread_affinity_exclude_set, heapprof_enabled, mbind, backend_selector, reactor_cfg] {
+           try {
+             // initialize thread_locals that are equal across all reacto threads of this smp instance
+             smp::_tmain = smp_tmain;
+             auto thread_name = seastar::format("reactor-{}", i);
+             pthread_setname_np(pthread_self(), thread_name.c_str());
+-            if (thread_affinity) {
++            if (thread_affinity && thread_affinity_exclude_set.count(i) != 0) {
+                 smp::pin(allocation.cpu_id);
+             }
+             memory::configure(allocation.mem, mbind, hugepages_path);
+diff --git a/src/core/resource.cc b/src/core/resource.cc
+index a8f9796c..67689047 100644
+--- a/src/core/resource.cc
++++ b/src/core/resource.cc
+@@ -529,9 +529,6 @@ resources allocate(configuration& c) {
+                                               cgroup::memory_limit()));
+     unsigned available_procs = hwloc_get_nbobjs_by_type(topology, HWLOC_OBJ_PU);
+     unsigned procs = c.cpus.value_or(available_procs);
+-    if (procs > available_procs) {
+-        throw std::runtime_error("insufficient processing units");
+-    }
+     // limit memory address to fit in 36-bit, see core/memory.cc:Memory map
+     constexpr size_t max_mem_per_proc = 1UL << 36;
+     auto mem_per_proc = std::min(align_down<size_t>(mem / procs, 2 << 20), max_mem_per_proc);
+-- 
+2.43.0
+