Merge branch 'master' into niek-fix-sink

.github/workflows/python-lint.yaml

@@ -0,0 +1,16 @@
+name: Python format
+
+on: [push, pull_request]
+
+jobs:
+  python-format:
+    name: Enforce python format
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: psf/[email protected]
+        with:
+            version: "24.8.0"
+            src: ./scripts
+            # override options so that we can specify only specific files for now
+            options: "--check --diff --include=.*addr2line.*"

CMakeLists.txt

@@ -73,11 +73,11 @@ set (Seastar_API_LEVEL
   "7"
   CACHE
   STRING
-  "Seastar compatibility API level (5=future<T>::get() returns T&&, 6=future is not variadic, 7=unified CPU/IO scheduling groups")
+  "Seastar compatibility API level (7=unified CPU/IO scheduling groups")
 
 set_property (CACHE Seastar_API_LEVEL
   PROPERTY
-  STRINGS 6)
+  STRINGS 7)
 
 set (Seastar_SCHEDULING_GROUPS_COUNT
   "16"
@@ -821,6 +821,14 @@ if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
     -Wno-error=deprecated-declarations)
 endif ()
 
+if (CMAKE_CXX_STANDARD GREATER_EQUAL 23)
+  include (CheckP2582R1)
+  if (Cxx_Compiler_IMPLEMENTS_P2581R1)
+    target_compile_definitions (seastar
+      PUBLIC SEASTAR_P2581R1)
+  endif ()
+endif ()
+
 if (BUILD_SHARED_LIBS)
   # use initial-exec TLS, as it puts the TLS variables in the static TLS space
   # instead of allocating them using malloc. otherwise intercepting mallocs and
@@ -883,7 +891,7 @@ if (condition)
   if (NOT Sanitizers_FOUND)
     message (FATAL_ERROR "Sanitizers not found!")
   endif ()
-  set (Seastar_Sanitizers_OPTIONS ${Sanitizers_COMPILER_OPTIONS})
+  set (Seastar_Sanitizers_OPTIONS ${Sanitizers_COMPILE_OPTIONS})
   target_link_libraries (seastar
     PUBLIC
       $<${condition}:Sanitizers::address>

README.md

@@ -164,13 +164,16 @@ The documentation is available on the [web](http://docs.seastar.io/master/index.
 
 Resources
 ---------
-Ask questions and post patches on the development mailing list. Subscription
-information and archives are available [here](https://groups.google.com/forum/#!forum/seastar-dev),
-or just send an email to [email protected].
 
-Information can be found on the main [project website](http://seastar.io).
+* Seasatar Development Mailing List: Discuss challenges, propose improvements with
+  sending code contributions (patches), and get help from experienced developers.
+  Subscribe or browse archives: [here](https://groups.google.com/forum/#!forum/seastar-dev)
+  (or email [email protected]).
+* GitHub Discussions: For more casual conversations and quick questions, consider
+  using the Seastar project's [discussions on Github](https://github.com/scylladb/seastar/discussions).
+* Issue Tracker: File bug reports on the project's [issue tracker](https://github.com/scylladb/seastar/issues).
 
-File bug reports on the project [issue tracker](https://github.com/scylladb/seastar/issues).
+Learn more about Seastar on the main [project website](http://seastar.io).
 
 The Native TCP/IP Stack
 -----------------------
@@ -199,3 +202,4 @@ Projects using Seastar
 * [redpanda](https://vectorized.io/): A Kafka replacement for mission critical systems
 * [Scylla](https://github.com/scylladb/scylla): A fast and reliable NoSQL data store compatible with Cassandra and DynamoDB
 * [smf](https://github.com/smfrpc/smf): The fastest RPC in the West
+* [Ceph - Crimson](https://github.com/ceph/ceph): Next-generation OSD (Object Storage Daemon) implementation based on the Seastar framework

apps/io_tester/io_tester.cc

@@ -219,6 +219,7 @@ class shard_config {
 struct shard_info {
     unsigned parallelism = 0;
     unsigned rps = 0;
+    unsigned batch = 1;
     unsigned limit = std::numeric_limits<unsigned>::max();
     unsigned shares = 10;
     std::string sched_class = "";
@@ -326,49 +327,44 @@ class class_data {
         }
     }
 
-    future<> issue_requests_in_parallel(std::chrono::steady_clock::time_point stop, unsigned parallelism) {
-        return parallel_for_each(boost::irange(0u, parallelism), [this, stop] (auto dummy) mutable {
+    future<> issue_request(char* buf, io_intent* intent, std::chrono::steady_clock::time_point start, std::chrono::steady_clock::time_point stop) {
+        return issue_request(buf, intent).then([this, start, stop] (auto size) {
+            auto now = std::chrono::steady_clock::now();
+            if (now < stop) {
+                this->add_result(size, std::chrono::duration_cast<std::chrono::microseconds>(now - start));
+            }
+            return make_ready_future<>();
+        });
+    }
+
+    future<> issue_requests_in_parallel(std::chrono::steady_clock::time_point stop) {
+        return parallel_for_each(boost::irange(0u, parallelism()), [this, stop] (auto dummy) mutable {
             auto bufptr = allocate_aligned_buffer<char>(this->req_size(), _alignment);
             auto buf = bufptr.get();
             return do_until([this, stop] { return std::chrono::steady_clock::now() > stop || requests() > limit(); }, [this, buf, stop] () mutable {
                 auto start = std::chrono::steady_clock::now();
-                return issue_request(buf, nullptr).then([this, start, stop] (auto size) {
-                    auto now = std::chrono::steady_clock::now();
-                    if (now < stop) {
-                        this->add_result(size, std::chrono::duration_cast<std::chrono::microseconds>(now - start));
-                    }
+                return issue_request(buf, nullptr, start, stop).then([this] {
                     return think();
                 });
             }).finally([bufptr = std::move(bufptr)] {});
         });
     }
 
-    future<> issue_requests_at_rate(std::chrono::steady_clock::time_point stop, unsigned rps, unsigned parallelism) {
-        return do_with(io_intent{}, 0u, [this, stop, rps, parallelism] (io_intent& intent, unsigned& in_flight) {
-            return parallel_for_each(boost::irange(0u, parallelism), [this, stop, rps, &intent, &in_flight, parallelism] (auto dummy) mutable {
+    future<> issue_requests_at_rate(std::chrono::steady_clock::time_point stop) {
+        return do_with(io_intent{}, 0u, [this, stop] (io_intent& intent, unsigned& in_flight) {
+            return parallel_for_each(boost::irange(0u, parallelism()), [this, stop, &intent, &in_flight] (auto dummy) mutable {
                 auto bufptr = allocate_aligned_buffer<char>(this->req_size(), _alignment);
                 auto buf = bufptr.get();
-                auto pause = std::chrono::duration_cast<std::chrono::microseconds>(1s) / rps;
+                auto pause = std::chrono::duration_cast<std::chrono::microseconds>(1s) / rps();
                 auto pause_dist = _config.options.pause_fn(pause);
-                return seastar::sleep((pause / parallelism) * dummy).then([this, buf, stop, pause = pause_dist.get(), &intent, &in_flight] () mutable {
+                return seastar::sleep((pause / parallelism()) * dummy).then([this, buf, stop, pause = pause_dist.get(), &intent, &in_flight] () mutable {
                     return do_until([this, stop] { return std::chrono::steady_clock::now() > stop || requests() > limit(); }, [this, buf, stop, pause, &intent, &in_flight] () mutable {
                         auto start = std::chrono::steady_clock::now();
                         in_flight++;
-                        return issue_request(buf, &intent).then_wrapped([this, start, pause, stop, &in_flight] (auto size_f) {
-                            size_t size;
-                            try {
-                                size = size_f.get();
-                            } catch (...) {
-                                // cancelled
-                                in_flight--;
-                                return make_ready_future<>();
-                            }
-
+                        return parallel_for_each(boost::irange(0u, batch()), [this, buf, &intent, start, stop] (auto dummy) {
+                            return issue_request(buf, &intent, start, stop);
+                        }).then([this, start, pause] {
                             auto now = std::chrono::steady_clock::now();
-                            if (now < stop) {
-                                this->add_result(size, std::chrono::duration_cast<std::chrono::microseconds>(now - start));
-                            }
-                            in_flight--;
                             auto p = pause->template get_as<std::chrono::microseconds>();
                             auto next = start + p;
 
@@ -378,12 +374,16 @@ class class_data {
                                 // probably the system cannot keep-up with this rate
                                 return make_ready_future<>();
                             }
+                        }).handle_exception_type([] (const cancelled_error&) {
+                            // expected
+                        }).finally([&in_flight] {
+                            in_flight--;
                         });
                     });
-                }).then([&intent, &in_flight] {
-                    intent.cancel();
-                    return do_until([&in_flight] { return in_flight == 0; }, [] { return seastar::sleep(100ms /* ¯\_(ツ)_/¯ */); });
                 }).finally([bufptr = std::move(bufptr), pause = std::move(pause_dist)] {});
+            }).then([&intent, &in_flight] {
+                intent.cancel();
+                return do_until([&in_flight] { return in_flight == 0; }, [] { return seastar::sleep(100ms /* ¯\_(ツ)_/¯ */); });
             });
         });
     }
@@ -393,9 +393,9 @@ class class_data {
         _start = std::chrono::steady_clock::now();
         return with_scheduling_group(_sg, [this, stop] {
             if (rps() == 0) {
-                return issue_requests_in_parallel(stop, parallelism());
+                return issue_requests_in_parallel(stop);
             } else {
-                return issue_requests_at_rate(stop, rps(), parallelism());
+                return issue_requests_at_rate(stop);
             }
         }).then([this] {
             _total_duration = std::chrono::steady_clock::now() - _start;
@@ -477,6 +477,10 @@ class class_data {
         return _config.shard_info.rps;
     }
 
+    unsigned batch() const {
+        return _config.shard_info.batch;
+    }
+
     unsigned limit() const noexcept {
         return _config.shard_info.limit;
     }
@@ -923,6 +927,9 @@ struct convert<shard_info> {
         if (node["rps"]) {
             sl.rps = node["rps"].as<unsigned>();
         }
+        if (node["batch"]) {
+            sl.batch = node["batch"].as<unsigned>();
+        }
         if (node["limit"]) {
             sl.limit = node["limit"].as<unsigned>();
         }
@@ -966,7 +973,7 @@ struct convert<options> {
             } else if (st == "steady") {
                 op.sleep_fn = timer_sleep<std::chrono::steady_clock>;
             } else {
-                throw std::runtime_error(format("Unknown sleep_type {}", st));
+                throw std::runtime_error(seastar::format("Unknown sleep_type {}", st));
             }
         }
         if (node["pause_distribution"]) {
@@ -976,7 +983,7 @@ struct convert<options> {
             } else if (pd == "poisson") {
                 op.pause_fn = make_poisson_pause;
             } else {
-                throw std::runtime_error(format("Unknown pause_distribution {}", pd));
+                throw std::runtime_error(seastar::format("Unknown pause_distribution {}", pd));
             }
         }
         return true;

cmake/CheckP2582R1.cmake

@@ -0,0 +1,26 @@
+include (CheckCXXSourceCompiles)
+include (CMakePushCheckState)
+
+cmake_push_check_state (RESET)
+
+set (CMAKE_REQUIRED_FLAGS "-std=c++23")
+
+# check if the compiler implements the inherited vs non-inherited guide
+# tiebreaker specified by P2582R1, see https://wg21.link/P2582R1
+check_cxx_source_compiles ("
+template <typename T> struct B {
+    B(T) {}
+};
+
+template <typename T> struct C : public B<T> {
+    using B<T>::B;
+};
+
+B(int) -> B<char>;
+C c2(42);
+
+int main() {}
+"
+  Cxx_Compiler_IMPLEMENTS_P2581R1)
+
+cmake_pop_check_state ()

cmake/FindSanitizers.cmake

@@ -20,54 +20,61 @@
 # Copyright (C) 2018 Scylladb, Ltd.
 #
 
-include (CheckCXXSourceCompiles)
+if(NOT Sanitizers_FIND_COMPONENTS)
+  set(Sanitizers_FIND_COMPONENTS
+    address
+    undefined_behavior)
+endif()
 
-set (CMAKE_REQUIRED_FLAGS -fsanitize=address)
-check_cxx_source_compiles ("int main() {}" Sanitizers_ADDRESS_FOUND)
-
-if (Sanitizers_ADDRESS_FOUND)
-  set (Sanitizers_ADDRESS_COMPILER_OPTIONS -fsanitize=address)
-endif ()
+foreach (component ${Sanitizers_FIND_COMPONENTS})
+  string (TOUPPER ${component} COMPONENT)
+  set (compile_options "Sanitizers_${COMPONENT}_COMPILE_OPTIONS")
+  if (component STREQUAL "address")
+    list (APPEND ${compile_options} -fsanitize=address)
+  elseif (component STREQUAL "undefined_behavior")
+    list (APPEND ${compile_options} -fsanitize=undefined)
+  else ()
+    message (FATAL_ERROR "Unsupported sanitizer: ${component}")
+  endif ()
+  list(APPEND Sanitizers_COMPILE_OPTIONS "${${compile_options}}")
+endforeach ()
 
-set (CMAKE_REQUIRED_FLAGS -fsanitize=undefined)
-check_cxx_source_compiles ("int main() {}" Sanitizers_UNDEFINED_BEHAVIOR_FOUND)
+include(CheckCXXSourceCompiles)
+include(CMakePushCheckState)
 
-if (Sanitizers_UNDEFINED_BEHAVIOR_FOUND)
-  # Disable vptr because of https://gcc.gnu.org/bugzilla/show_bug.cgi?id=88684
-  set (Sanitizers_UNDEFINED_BEHAVIOR_COMPILER_OPTIONS "-fsanitize=undefined;-fno-sanitize=vptr")
+# -fsanitize=address cannot be combined with -fsanitize=thread, so let's test
+# the combination of the compiler options.
+cmake_push_check_state()
+string (REPLACE ";" " " CMAKE_REQUIRED_FLAGS "${Sanitizers_COMPILE_OPTIONS}")
+set(CMAKE_REQUIRED_FLAGS ${Sanitizers_COMPILE_OPTIONS})
+check_cxx_source_compiles("int main() {}"
+  Sanitizers_SUPPORTED)
+if (Sanitizers_SUPPORTED)
+  if ("address" IN_LIST Sanitizers_FIND_COMPONENTS)
+    file (READ ${CMAKE_CURRENT_LIST_DIR}/code_tests/Sanitizers_fiber_test.cc _sanitizers_fiber_test_code)
+    check_cxx_source_compiles ("${_sanitizers_fiber_test_code}"
+      Sanitizers_FIBER_SUPPORT)
+  endif ()
 endif ()
-
-set (Sanitizers_COMPILER_OPTIONS
-  ${Sanitizers_ADDRESS_COMPILER_OPTIONS}
-  ${Sanitizers_UNDEFINED_BEHAVIOR_COMPILER_OPTIONS})
-
-file (READ ${CMAKE_CURRENT_LIST_DIR}/code_tests/Sanitizers_fiber_test.cc _sanitizers_fiber_test_code)
-set (CMAKE_REQUIRED_FLAGS ${Sanitizers_COMPILER_OPTIONS})
-check_cxx_source_compiles ("${_sanitizers_fiber_test_code}" Sanitizers_FIBER_SUPPORT)
+cmake_pop_check_state()
 
 include (FindPackageHandleStandardArgs)
 
 find_package_handle_standard_args (Sanitizers
   REQUIRED_VARS
-    Sanitizers_ADDRESS_COMPILER_OPTIONS
-    Sanitizers_UNDEFINED_BEHAVIOR_COMPILER_OPTIONS)
+    Sanitizers_COMPILE_OPTIONS
+    Sanitizers_SUPPORTED)
 
 if (Sanitizers_FOUND)
-  if (NOT (TARGET Sanitizers::address))
-    add_library (Sanitizers::address INTERFACE IMPORTED)
-
-    set_target_properties (Sanitizers::address
-      PROPERTIES
-        INTERFACE_COMPILE_OPTIONS ${Sanitizers_ADDRESS_COMPILER_OPTIONS}
-        INTERFACE_LINK_LIBRARIES ${Sanitizers_ADDRESS_COMPILER_OPTIONS})
-  endif ()
-
-  if (NOT (TARGET Sanitizers::undefined_behavior))
-    add_library (Sanitizers::undefined_behavior INTERFACE IMPORTED)
-
-    set_target_properties (Sanitizers::undefined_behavior
-      PROPERTIES
-        INTERFACE_COMPILE_OPTIONS "${Sanitizers_UNDEFINED_BEHAVIOR_COMPILER_OPTIONS}"
-        INTERFACE_LINK_LIBRARIES "${Sanitizers_UNDEFINED_BEHAVIOR_COMPILER_OPTIONS}")
-  endif ()
+  foreach (component ${Sanitizers_FIND_COMPONENTS})
+    string (TOUPPER ${component} COMPONENT)
+    set (library Sanitizers::${component})
+    if (NOT TARGET ${library})
+      add_library (${library} INTERFACE IMPORTED)
+      set_target_properties (${library}
+        PROPERTIES
+          INTERFACE_COMPILE_OPTIONS "${Sanitizers_${COMPONENT}_COMPILE_OPTIONS}"
+          INTERFACE_LINK_LIBRARIES "${Sanitizers_${COMPONENT}_COMPILE_OPTIONS}")
+    endif ()
+  endforeach ()
 endif ()

cmake/Finddpdk.cmake

@@ -129,6 +129,13 @@ find_package_handle_standard_args (dpdk
   REQUIRED_VARS
     ${dpdk_REQUIRED})
 
+# Depending on whether libbsd-dev exists at build time, DPDK may be built with a dependency on
+# libbsd. Note that other libraries in dpdk_PC_LIBRARIES are handled using separate logic
+# (see rte_libs above), thus the additional dependencies must be handled on a case by case basis.
+if ("bsd" IN_LIST dpdk_PC_LIBRARIES)
+  list (APPEND dpdk_dependencies "bsd")
+endif ()
+
 if (dpdk_FOUND AND NOT (TARGET dpdk))
   get_filename_component (library_suffix "${dpdk_EAL_LIBRARY}" LAST_EXT)
   # strictly speaking, we should have being using check_c_compiler_flag()
@@ -163,6 +170,7 @@ if (dpdk_FOUND AND NOT (TARGET dpdk))
     set_target_properties (dpdk
       PROPERTIES
         INTERFACE_INCLUDE_DIRECTORIES ${dpdk_INCLUDE_DIR}
+        INTERFACE_LINK_LIBRARIES "${dpdk_dependencies}"
         IMPORTED_OBJECTS ${dpdk_object_path}
         ${compile_options})
     # we include dpdk in seastar already, so no need to expose it with
@@ -175,7 +183,7 @@ if (dpdk_FOUND AND NOT (TARGET dpdk))
     set_target_properties (DPDK::dpdk
       PROPERTIES
         INTERFACE_INCLUDE_DIRECTORIES "${dpdk_PC_INCLUDE_DIRS}"
-        INTERFACE_LINK_LIBRARIES "${_dpdk_libraries}"
+        INTERFACE_LINK_LIBRARIES "${_dpdk_libraries};${dpdk_dependencies}"
         ${compile_options})
   endif()
 endif ()