From 3e4d3ebce7600fc5721acb56c7518a469ab0e96c Mon Sep 17 00:00:00 2001 From: tomershafir Date: Wed, 11 Dec 2024 21:23:46 +0200 Subject: [PATCH] smp: disable mlock on overcommit Memory locking keeps pages in physical memory, and decreases the amount of pages available for swapping on an overcommit setup. Thus, it can decrease fairness, increase memory pressure, and cause OOMs. So, it is less suitable for a cooperative overcommit setup. This change disables memory locking on overcommit (overprovisioned - I hope I can rename that config). Also, it reuses the mlock variable. I assume that this change shouldn't break reasonable configs. --- include/seastar/core/reactor_config.hh | 1 + include/seastar/core/smp_options.hh | 2 ++ src/core/reactor.cc | 8 ++++---- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/include/seastar/core/reactor_config.hh b/include/seastar/core/reactor_config.hh index 048298d654..6c2c6ac0ee 100644 --- a/include/seastar/core/reactor_config.hh +++ b/include/seastar/core/reactor_config.hh @@ -131,6 +131,7 @@ struct reactor_options : public program_options::option_group { /// * \ref idle_poll_time_us = 0 /// * \ref smp_options::thread_affinity = 0 /// * \ref poll_aio = 0 + /// * \ref smp_options::lock_memory = 0 program_options::value<> overprovisioned; /// \brief Abort when seastar allocator cannot allocate memory. program_options::value<> abort_on_seastar_bad_alloc; diff --git a/include/seastar/core/smp_options.hh b/include/seastar/core/smp_options.hh index 8cbee25f08..5cff44bb82 100644 --- a/include/seastar/core/smp_options.hh +++ b/include/seastar/core/smp_options.hh @@ -57,6 +57,8 @@ struct smp_options : public program_options::option_group { /// Path to accessible hugetlbfs mount (typically /dev/hugepages/something). program_options::value hugepages; /// Lock all memory (prevents swapping). + /// + /// Disable for overprovisioned environments. program_options::value lock_memory; /// Pin threads to their cpus (disable for overprovisioning). /// diff --git a/src/core/reactor.cc b/src/core/reactor.cc index a8b98e3e23..8f9db00351 100644 --- a/src/core/reactor.cc +++ b/src/core/reactor.cc @@ -3778,7 +3778,7 @@ reactor_options::reactor_options(program_options::option_group* parent_group) , kernel_page_cache(*this, "kernel-page-cache", false, "Use the kernel page cache. This disables DMA (O_DIRECT)." " Useful for short-lived functional tests with a small data set.") - , overprovisioned(*this, "overprovisioned", "run in an overprovisioned environment (such as docker or a laptop); equivalent to --idle-poll-time-us 0 --thread-affinity 0 --poll-aio 0") + , overprovisioned(*this, "overprovisioned", "run in an overprovisioned environment (such as docker or a laptop); equivalent to --idle-poll-time-us 0 --thread-affinity 0 --poll-aio 0 --lock-memory 0") , abort_on_seastar_bad_alloc(*this, "abort-on-seastar-bad-alloc", "abort when seastar allocator cannot allocate memory") , force_aio_syscalls(*this, "force-aio-syscalls", false, "Force io_getevents(2) to issue a system call, instead of bypassing the kernel when possible." @@ -3814,7 +3814,7 @@ smp_options::smp_options(program_options::option_group* parent_group) , memory(*this, "memory", std::nullopt, "memory to use, in bytes (ex: 4G) (default: all)") , reserve_memory(*this, "reserve-memory", {}, "memory reserved to OS (if --memory not specified)") , hugepages(*this, "hugepages", {}, "path to accessible hugetlbfs mount (typically /dev/hugepages/something)") - , lock_memory(*this, "lock-memory", {}, "lock all memory (prevents swapping)") + , lock_memory(*this, "lock-memory", {}, "lock all memory (prevents swapping). Disable for overprovisioned environments") , thread_affinity(*this, "thread-affinity", true, "pin threads to their cpus (disable for overprovisioning)") #ifdef SEASTAR_HAVE_HWLOC , num_io_groups(*this, "num-io-groups", {}, "Number of IO groups. Each IO group will be responsible for a fraction of the IO requests. Defaults to the number of NUMA nodes") @@ -4332,7 +4332,7 @@ void smp::configure(const smp_options& smp_opts, const reactor_options& reactor_ hugepages_path = smp_opts.hugepages.get_value(); } auto mlock = false; - if (smp_opts.lock_memory) { + if (smp_opts.lock_memory && !reactor_opts.overprovisioned) { mlock = smp_opts.lock_memory.get_value(); } if (mlock) { @@ -4603,7 +4603,7 @@ void smp::configure(const smp_options& smp_opts, const reactor_options& reactor_ engine().configure(reactor_opts); - if (smp_opts.lock_memory && smp_opts.lock_memory.get_value() && layout && !layout->ranges.empty()) { + if (mlock && layout && !layout->ranges.empty()) { smp::setup_prefaulter(resources, std::move(*layout)); } }