From 5f7baef755d282e7bc44ff7bbe439c11b7fb1e6d Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Sat, 22 Feb 2025 19:53:27 +0000 Subject: [PATCH] Refactor: Remove unused features and functions, and move most allocator operations to a global namespace. (#750) * Factor out explicit Config type Instead of using snmalloc::Alloc::Config, expose snmalloc::Config, which is then used to derive the allocator type. * Move globalalloc to front end. * Remove unneed template parameter from global snmalloc functions. * Remove SNMALLOC_PASS_THROUGH VeronaRT now has an abstraction layer which can easily replace the allocator. Having such a complex integration still in snmalloc does not make sense. * Take some global functions off of local alloc. * Drop comparison overloads on atomic Capptr. Performing a comparison on two atomic ptr is a complex operation, and should not be implicit. The memory model order and such things needs to be considered by the caller. * Remove function_ref and use templates The implementation prefers to use templates over the function_ref. This now only exists in the Pal for a currently unused feature. * Removing function_ref reduces stl needs. * Remove use of __is_convertible to support older g++ * Inline function that is only used once. * Remove unused function * Restrict ThreadAlloc usage to globalalloc This commit introduces various inline functions on snmalloc:: that perform allocation/deallocation using the thread local allocator. They remove all usage from a particular test. * Move cheri checks to own file. * Refactor is_owned checks. * Move alloc_size and check_size to globalalloc. * Minor simplification of dealloc path * Fix up is_owned to take a config * Improve usage of scoped allocator. * Handle Config_ in globalalloc. --- CMakeLists.txt | 15 +- docs/release/0.7/README.md | 6 +- fuzzing/snmalloc-fuzzer.cpp | 2 +- src/snmalloc/README.md | 2 +- src/snmalloc/ds_core/cheri.h | 139 ++++++ src/snmalloc/ds_core/ds_core.h | 1 + src/snmalloc/ds_core/helpers.h | 49 -- src/snmalloc/ds_core/ptrwrap.h | 15 - src/snmalloc/global/bounds_checks.h | 15 +- src/snmalloc/global/global.h | 1 + src/snmalloc/global/globalalloc.h | 378 +++++++++++++++ src/snmalloc/global/libc.h | 52 +- src/snmalloc/global/scopedalloc.h | 8 +- src/snmalloc/mem/external_alloc.h | 89 ---- src/snmalloc/mem/freelist_queue.h | 2 +- src/snmalloc/mem/globalalloc.h | 137 ------ src/snmalloc/mem/localalloc.h | 453 +----------------- src/snmalloc/mem/mem.h | 2 - src/snmalloc/mem/metadata.h | 14 +- src/snmalloc/override/jemalloc_compat.cc | 47 +- src/snmalloc/override/malloc.cc | 2 +- src/snmalloc/override/rust.cc | 12 +- src/snmalloc/pal/pal_ds.h | 3 +- src/snmalloc/snmalloc.h | 10 +- src/snmalloc/stl/cxx/type_traits.h | 6 - src/snmalloc/stl/gnu/type_traits.h | 90 ---- src/test/func/cheri/cheri.cc | 2 +- src/test/func/client_meta/client_meta.cc | 15 +- src/test/func/domestication/domestication.cc | 41 +- .../func/external_pagemap/external_pagemap.cc | 3 +- .../func/first_operation/first_operation.cc | 87 +--- src/test/func/fixed_region/fixed_region.cc | 12 +- src/test/func/jemalloc/jemalloc.cc | 3 - src/test/func/malloc/malloc.cc | 16 +- src/test/func/memcpy/func-memcpy.cc | 13 +- src/test/func/memory/memory.cc | 176 ++++--- src/test/func/memory_usage/memory_usage.cc | 2 - src/test/func/miracle_ptr/miracle_ptr.cc | 17 +- src/test/func/sandbox/sandbox.cc | 6 +- src/test/func/statistics/stats.cc | 41 +- src/test/func/teardown/teardown.cc | 100 +--- .../thread_alloc_external.cc | 12 +- src/test/func/two_alloc_types/alloc1.cc | 6 +- src/test/perf/contention/contention.cc | 13 +- .../perf/external_pointer/externalpointer.cc | 23 +- src/test/perf/low_memory/low-memory.cc | 4 +- src/test/perf/memcpy/memcpy.cc | 12 +- src/test/perf/msgpass/msgpass.cc | 54 +-- src/test/perf/singlethread/singlethread.cc | 12 +- src/test/perf/startup/startup.cc | 3 +- 50 files changed, 819 insertions(+), 1404 deletions(-) create mode 100644 src/snmalloc/ds_core/cheri.h create mode 100644 src/snmalloc/global/globalalloc.h delete mode 100644 src/snmalloc/mem/external_alloc.h delete mode 100644 src/snmalloc/mem/globalalloc.h diff --git a/CMakeLists.txt b/CMakeLists.txt index e09a26160..da7d02fd6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -574,22 +574,9 @@ endif() endif() if (SNMALLOC_BUILD_TESTING) - if (WIN32 - OR (CMAKE_SYSTEM_NAME STREQUAL NetBSD) - OR (CMAKE_SYSTEM_NAME STREQUAL OpenBSD) - OR (CMAKE_SYSTEM_NAME STREQUAL SunOS)) - # Windows does not support aligned allocation well enough - # for pass through. - # NetBSD, OpenBSD and DragonFlyBSD do not support malloc*size calls. - set(FLAVOURS fast;check) - else() - set(FLAVOURS fast;check;malloc) - endif() + set(FLAVOURS fast;check) foreach(FLAVOUR ${FLAVOURS}) - if (${FLAVOUR} STREQUAL "malloc") - set(DEFINES SNMALLOC_PASS_THROUGH) - endif() if (${FLAVOUR} STREQUAL "check") set(DEFINES SNMALLOC_CHECK_CLIENT) endif() diff --git a/docs/release/0.7/README.md b/docs/release/0.7/README.md index 123bbc422..33aa68587 100644 --- a/docs/release/0.7/README.md +++ b/docs/release/0.7/README.md @@ -109,10 +109,10 @@ The additional meta-data size in snmalloc 0.6 was fixed and under a cache line i In snmalloc 0.7, we have made this meta-data size configurable. This allows developers to build new security features on top of snmalloc. -For instance, building snmalloc with the following definition of `Alloc` will allow you to store a 64-bit counter for each allocation: +For instance, building snmalloc with the following definition of `Config` will allow you to store a 64-bit counter for each allocation: ```cpp - using Alloc = snmalloc::LocalAllocator>>>; + using Config = snmalloc::StandardConfigClientMeta< + ArrayClientMetaDataProvider>>; ``` This does not affect the underlying alignment of the allocations. diff --git a/fuzzing/snmalloc-fuzzer.cpp b/fuzzing/snmalloc-fuzzer.cpp index f2e5f1ff7..6295b4569 100644 --- a/fuzzing/snmalloc-fuzzer.cpp +++ b/fuzzing/snmalloc-fuzzer.cpp @@ -148,7 +148,7 @@ struct Result { auto alloc = snmalloc::get_scoped_allocator(); if (ptr) - alloc->dealloc(ptr, size); + alloc->dealloc(ptr); ptr = nullptr; } }; diff --git a/src/snmalloc/README.md b/src/snmalloc/README.md index 05347b4d1..9f593ac1d 100644 --- a/src/snmalloc/README.md +++ b/src/snmalloc/README.md @@ -20,7 +20,7 @@ These are arranged in a hierarchy such that each of the directories may include This includes data structures such as pagemap implementations (efficient maps from a chunk address to associated metadata) and buddy allocators for managing address-space ranges. - `backend/` provides some example implementations for snmalloc embeddings that provide a global memory allocator for an address space. Users may ignore this entirely and use the types in `mem/` with a custom back end to expose an snmalloc instance with specific behaviour. - Layers above this can be used with a custom configuration by defining `SNMALLOC_PROVIDE_OWN_CONFIG` and exporting a type as `snmalloc::Alloc` that defines the type of an `snmalloc::LocalAllocator` template specialisation. + Layers above this can be used with a custom configuration by defining `SNMALLOC_PROVIDE_OWN_CONFIG` and exporting a type as `snmalloc::Config` that defines the configuration. - `global/` provides some front-end components that assume that snmalloc is available in a global configuration. - `override/` builds on top of `global/` to provide specific implementations with compatibility with external specifications (for example C `malloc`, C++ `operator new`, jemalloc's `*allocx`, or Rust's `std::alloc`). diff --git a/src/snmalloc/ds_core/cheri.h b/src/snmalloc/ds_core/cheri.h new file mode 100644 index 000000000..8d5bd13e7 --- /dev/null +++ b/src/snmalloc/ds_core/cheri.h @@ -0,0 +1,139 @@ +#include "mitigations.h" + +namespace snmalloc +{ + /* + * Many of these tests come with an "or is null" branch that they'd need to + * add if we did them up front. Instead, defer them until we're past the + * point where we know, from the pagemap, or by explicitly testing, that the + * pointer under test is not nullptr. + */ + SNMALLOC_FAST_PATH_INLINE void dealloc_cheri_checks(void* p) + { +#if defined(__CHERI_PURE_CAPABILITY__) + /* + * Enforce the use of an unsealed capability. + * + * TODO In CHERI+MTE, this, is part of the CAmoCDecVersion instruction; + * elide this test in that world. + */ + snmalloc_check_client( + mitigations(cheri_checks), + !__builtin_cheri_sealed_get(p), + "Sealed capability in deallocation"); + + /* + * Enforce permissions on the returned pointer. These pointers end up in + * free queues and will be cycled out to clients again, so try to catch + * erroneous behavior now, rather than later. + * + * TODO In the CHERI+MTE case, we must reconstruct the pointer for the + * free queues as part of the discovery of the start of the object (so + * that it has the correct version), and the CAmoCDecVersion call imposes + * its own requirements on the permissions (to ensure that it's at least + * not zero). They are somewhat more lax than we might wish, so this test + * may remain, guarded by SNMALLOC_CHECK_CLIENT, but no explicit + * permissions checks are required in the non-SNMALLOC_CHECK_CLIENT case + * to defend ourselves or other clients against a misbehaving client. + */ + static const size_t reqperm = CHERI_PERM_LOAD | CHERI_PERM_STORE | + CHERI_PERM_LOAD_CAP | CHERI_PERM_STORE_CAP; + snmalloc_check_client( + mitigations(cheri_checks), + (__builtin_cheri_perms_get(p) & reqperm) == reqperm, + "Insufficient permissions on capability in deallocation"); + + /* + * We check for a valid tag here, rather than in domestication, because + * domestication might be answering a slightly different question, about + * the plausibility of addresses rather than of exact pointers. + * + * TODO Further, in the CHERI+MTE case, the tag check will be implicit in + * a future CAmoCDecVersion instruction, and there should be no harm in + * the lookups we perform along the way to get there. In that world, + * elide this test. + */ + snmalloc_check_client( + mitigations(cheri_checks), + __builtin_cheri_tag_get(p), + "Untagged capability in deallocation"); + + /* + * Verify that the capability is not zero-length, ruling out the other + * edge case around monotonicity. + */ + snmalloc_check_client( + mitigations(cheri_checks), + __builtin_cheri_length_get(p) > 0, + "Zero-length capability in deallocation"); + + /* + * At present we check for the pointer also being the start of an + * allocation closer to dealloc; for small objects, that happens in + * dealloc_local_object_fast, either below or *on the far end of message + * receipt*. For large objects, it happens below by directly rounding to + * power of two rather than using the is_start_of_object helper. + * (XXX This does mean that we might end up threading our remote queue + * state somewhere slightly unexpected rather than at the head of an + * object. That is perhaps fine for now?) + */ + + /* + * TODO + * + * We could enforce other policies here, including that the length exactly + * match the sizeclass. At present, we bound caps we give for allocations + * to the underlying sizeclass, so even malloc(0) will have a non-zero + * length. Monotonicity would then imply that the pointer must be the + * head of an object (modulo, perhaps, temporal aliasing if we somehow + * introduced phase shifts in heap layout like some allocators do). + * + * If we switched to bounding with upwards-rounded representable bounds + * (c.f., CRRL) rather than underlying object size, then we should, + * instead, in general require plausibility of p_raw by checking that its + * length is nonzero and the snmalloc size class associated with its + * length is the one for the slab in question... except for the added + * challenge of malloc(0). Since 0 rounds up to 0, we might end up + * constructing zero-length caps to hand out, which we would then reject + * upon receipt. Instead, as part of introducing CRRL bounds, we should + * introduce a sizeclass for slabs holding zero-size objects. All told, + * we would want to check that + * + * size_to_sizeclass(length) == entry.get_sizeclass() + * + * I believe a relaxed CRRL test of + * + * length > 0 || (length == sizeclass_to_size(entry.get_sizeclass())) + * + * would also suffice and may be slightly less expensive than the test + * above, at the cost of not catching as many misbehaving clients. + * + * In either case, having bounded by CRRL bounds, we would need to be + * *reconstructing* the capabilities headed to our free lists to be given + * out to clients again; there are many more CRRL classes than snmalloc + * sizeclasses (this is the same reason that we can always get away with + * CSetBoundsExact in capptr_bound). Switching to CRRL bounds, if that's + * ever a thing we want to do, will be easier after we've done the + * plumbing for CHERI+MTE. + */ + + /* + * TODO: Unsurprisingly, the CHERI+MTE case once again has something to + * say here. In that world, again, we are certain to be reconstructing + * the capability for the free queue anyway, and so exactly what we wish + * to enforce, length-wise, of the provided capability, is somewhat more + * flexible. Using the provided capability bounds when recoloring memory + * could be a natural way to enforce that it covers the entire object, at + * the cost of a more elaborate recovery story (as we risk aborting with a + * partially recolored object). On non-SNMALLOC_CHECK_CLIENT builds, it + * likely makes sense to just enforce that length > 0 (*not* enforced by + * the CAmoCDecVersion instruction) and say that any authority-bearing + * interior pointer suffices to free the object. I believe that to be an + * acceptable security posture for the allocator and between clients; + * misbehavior is confined to the misbehaving client. + */ +#else + UNUSED(p); +#endif + } +} // namespace snmalloc \ No newline at end of file diff --git a/src/snmalloc/ds_core/ds_core.h b/src/snmalloc/ds_core/ds_core.h index 01121f5cb..38e99dce2 100644 --- a/src/snmalloc/ds_core/ds_core.h +++ b/src/snmalloc/ds_core/ds_core.h @@ -8,6 +8,7 @@ */ #include "bits.h" +#include "cheri.h" #include "concept.h" #include "defines.h" #include "helpers.h" diff --git a/src/snmalloc/ds_core/helpers.h b/src/snmalloc/ds_core/helpers.h index 3008da4f5..e5a0ca00e 100644 --- a/src/snmalloc/ds_core/helpers.h +++ b/src/snmalloc/ds_core/helpers.h @@ -85,55 +85,6 @@ namespace snmalloc } }; - /** - * Non-owning version of std::function. Wraps a reference to a callable object - * (eg. a lambda) and allows calling it through dynamic dispatch, with no - * allocation. This is useful in the allocator code paths, where we can't - * safely use std::function. - * - * Inspired by the C++ proposal: - * http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2018/p0792r2.html - */ - template - struct function_ref; - - template - struct function_ref - { - // The enable_if is used to stop this constructor from shadowing the default - // copy / move constructors. - template< - typename Fn, - typename = - stl::enable_if_t, function_ref>>> - function_ref(Fn&& fn) - { - data_ = static_cast(&fn); - fn_ = execute; - } - - R operator()(Args... args) const - { - return fn_(data_, args...); - } - - private: - void* data_; - R (*fn_)(void*, Args...); - - template - static R execute(void* p, Args... args) - { - return (*static_cast>(p))(args...); - }; - }; - - template typename Ptr> - void ignore(Ptr t) - { - UNUSED(t); - } - /** * Helper class for building fatal errors. Used by `report_fatal_error` to * build an on-stack buffer containing the formatted string. diff --git a/src/snmalloc/ds_core/ptrwrap.h b/src/snmalloc/ds_core/ptrwrap.h index 853e5fb4c..843eb8d73 100644 --- a/src/snmalloc/ds_core/ptrwrap.h +++ b/src/snmalloc/ds_core/ptrwrap.h @@ -517,21 +517,6 @@ namespace snmalloc return CapPtr::unsafe_from( this->unsafe_capptr.exchange(desired.unsafe_ptr(), order)); } - - SNMALLOC_FAST_PATH bool operator==(const AtomicCapPtr& rhs) const - { - return this->unsafe_capptr == rhs.unsafe_capptr; - } - - SNMALLOC_FAST_PATH bool operator!=(const AtomicCapPtr& rhs) const - { - return this->unsafe_capptr != rhs.unsafe_capptr; - } - - SNMALLOC_FAST_PATH bool operator<(const AtomicCapPtr& rhs) const - { - return this->unsafe_capptr < rhs.unsafe_capptr; - } }; namespace capptr diff --git a/src/snmalloc/global/bounds_checks.h b/src/snmalloc/global/bounds_checks.h index 8388256f3..676ac4641 100644 --- a/src/snmalloc/global/bounds_checks.h +++ b/src/snmalloc/global/bounds_checks.h @@ -1,4 +1,5 @@ #pragma once +#include "globalalloc.h" #include "threadalloc.h" namespace snmalloc @@ -61,18 +62,17 @@ namespace snmalloc } else { - auto& alloc = ThreadAlloc::get(); void* p = const_cast(ptr); auto range_end = pointer_offset(p, len); - auto object_end = alloc.template external_pointer(p); + auto object_end = external_pointer(p); report_fatal_error( "Fatal Error!\n{}: \n\trange [{}, {})\n\tallocation [{}, " "{})\nrange goes beyond allocation by {} bytes \n", msg, p, range_end, - alloc.template external_pointer(p), + external_pointer(p), object_end, pointer_diff(object_end, range_end)); } @@ -86,13 +86,16 @@ namespace snmalloc * The template parameter indicates whether the check should be performed. It * defaults to true. If it is false, the check will always succeed. */ - template + template SNMALLOC_FAST_PATH_INLINE bool check_bounds(const void* ptr, size_t len) { if constexpr (PerformCheck) { - auto& alloc = ThreadAlloc::get(); - return alloc.check_bounds(ptr, len); + if (SNMALLOC_LIKELY(Config::is_initialised())) + { + return remaining_bytes(address_cast(ptr)) >= len; + } + return true; } else { diff --git a/src/snmalloc/global/global.h b/src/snmalloc/global/global.h index 514d69b7c..ffa149567 100644 --- a/src/snmalloc/global/global.h +++ b/src/snmalloc/global/global.h @@ -1,4 +1,5 @@ #include "bounds_checks.h" +#include "globalalloc.h" #include "libc.h" #include "memcpy.h" #include "scopedalloc.h" diff --git a/src/snmalloc/global/globalalloc.h b/src/snmalloc/global/globalalloc.h new file mode 100644 index 000000000..49e531e26 --- /dev/null +++ b/src/snmalloc/global/globalalloc.h @@ -0,0 +1,378 @@ +#pragma once + +#include "../mem/mem.h" +#include "threadalloc.h" + +namespace snmalloc +{ + template + inline static void cleanup_unused() + { + static_assert( + Config_::Options.CoreAllocIsPoolAllocated, + "Global cleanup is available only for pool-allocated configurations"); + // Call this periodically to free and coalesce memory allocated by + // allocators that are not currently in use by any thread. + // One atomic operation to extract the stack, another to restore it. + // Handling the message queue for each stack is non-atomic. + auto* first = AllocPool::extract(); + auto* alloc = first; + decltype(alloc) last; + + if (alloc != nullptr) + { + while (alloc != nullptr) + { + alloc->flush(); + last = alloc; + alloc = AllocPool::extract(alloc); + } + + AllocPool::restore(first, last); + } + } + + /** + If you pass a pointer to a bool, then it returns whether all the + allocators are empty. If you don't pass a pointer to a bool, then will + raise an error all the allocators are not empty. + */ + template + inline static void debug_check_empty(bool* result = nullptr) + { + static_assert( + Config_::Options.CoreAllocIsPoolAllocated, + "Global status is available only for pool-allocated configurations"); + // This is a debugging function. It checks that all memory from all + // allocators has been freed. + auto* alloc = AllocPool::iterate(); + +#ifdef SNMALLOC_TRACING + message<1024>("debug check empty: first {}", alloc); +#endif + bool done = false; + bool okay = true; + + while (!done) + { +#ifdef SNMALLOC_TRACING + message<1024>("debug_check_empty: Check all allocators!"); +#endif + done = true; + alloc = AllocPool::iterate(); + okay = true; + + while (alloc != nullptr) + { +#ifdef SNMALLOC_TRACING + message<1024>("debug check empty: {}", alloc); +#endif + // Check that the allocator has freed all memory. + // repeat the loop if empty caused message sends. + if (alloc->debug_is_empty(&okay)) + { + done = false; +#ifdef SNMALLOC_TRACING + message<1024>("debug check empty: sent messages {}", alloc); +#endif + } + +#ifdef SNMALLOC_TRACING + message<1024>("debug check empty: okay = {}", okay); +#endif + alloc = AllocPool::iterate(alloc); + } + } + + if (result != nullptr) + { + *result = okay; + return; + } + + // Redo check so abort is on allocator with allocation left. + if (!okay) + { + alloc = AllocPool::iterate(); + while (alloc != nullptr) + { + alloc->debug_is_empty(nullptr); + alloc = AllocPool::iterate(alloc); + } + } + } + + template + inline static void debug_in_use(size_t count) + { + static_assert( + Config_::Options.CoreAllocIsPoolAllocated, + "Global status is available only for pool-allocated configurations"); + auto alloc = AllocPool::iterate(); + while (alloc != nullptr) + { + if (alloc->debug_is_in_use()) + { + if (count == 0) + { + error("ERROR: allocator in use."); + } + count--; + } + alloc = AllocPool::iterate(alloc); + + if (count != 0) + { + error("Error: two few allocators in use."); + } + } + } + + /** + * Returns the number of remaining bytes in an object. + * + * auto p = (char*)malloc(size) + * remaining_bytes(p + n) == size - n provided n < size + */ + template + size_t remaining_bytes(address_t p) + { + const auto& entry = Config_::Backend::template get_metaentry(p); + + auto sizeclass = entry.get_sizeclass(); + return snmalloc::remaining_bytes(sizeclass, p); + } + + /** + * Returns the byte offset into an object. + * + * auto p = (char*)malloc(size) + * index_in_object(p + n) == n provided n < size + */ + template + static inline size_t index_in_object(address_t p) + { + const auto& entry = Config_::Backend::template get_metaentry(p); + + auto sizeclass = entry.get_sizeclass(); + return snmalloc::index_in_object(sizeclass, p); + } + + enum Boundary + { + /** + * The location of the first byte of this allocation. + */ + Start, + /** + * The location of the last byte of the allocation. + */ + End, + /** + * The location one past the end of the allocation. This is mostly useful + * for bounds checking, where anything less than this value is safe. + */ + OnePastEnd + }; + + /** + * Returns the Start/End of an object allocated by this allocator + * + * It is valid to pass any pointer, if the object was not allocated + * by this allocator, then it give the start and end as the whole of + * the potential pointer space. + */ + template< + Boundary location = Start, + SNMALLOC_CONCEPT(IsConfig) Config_ = Config> + inline static void* external_pointer(void* p) + { + /* + * Note that: + * * each case uses `pointer_offset`, so that on CHERI, our behaviour is + * monotone with respect to the capability `p`. + * + * * the returned pointer could be outside the CHERI bounds of `p`, and + * thus not something that can be followed. + * + * * we don't use capptr_from_client()/capptr_reveal(), to avoid the + * syntactic clutter. By inspection, `p` flows only to address_cast + * and pointer_offset, and so there's no risk that we follow or act + * to amplify the rights carried by `p`. + */ + if constexpr (location == Start) + { + size_t index = index_in_object(address_cast(p)); + return pointer_offset(p, 0 - index); + } + else if constexpr (location == End) + { + return pointer_offset(p, remaining_bytes(address_cast(p)) - 1); + } + else + { + return pointer_offset(p, remaining_bytes(address_cast(p))); + } + } + + /** + * @brief Get the client meta data for the snmalloc allocation covering this + * pointer. + */ + template + typename Config_::ClientMeta::DataRef get_client_meta_data(void* p) + { + const auto& entry = Config_::Backend::get_metaentry(address_cast(p)); + + size_t index = slab_index(entry.get_sizeclass(), address_cast(p)); + + auto* meta_slab = entry.get_slab_metadata(); + + if (SNMALLOC_UNLIKELY(entry.is_backend_owned())) + { + error("Cannot access meta-data for write for freed memory!"); + } + + if (SNMALLOC_UNLIKELY(meta_slab == nullptr)) + { + error( + "Cannot access meta-data for non-snmalloc object in writable form!"); + } + + return meta_slab->get_meta_for_object(index); + } + + /** + * @brief Get the client meta data for the snmalloc allocation covering this + * pointer. + */ + template + stl::add_const_t + get_client_meta_data_const(void* p) + { + const auto& entry = + Config_::Backend::template get_metaentry(address_cast(p)); + + size_t index = slab_index(entry.get_sizeclass(), address_cast(p)); + + auto* meta_slab = entry.get_slab_metadata(); + + if (SNMALLOC_UNLIKELY((meta_slab == nullptr) || (entry.is_backend_owned()))) + { + static typename Config_::ClientMeta::StorageType null_meta_store{}; + return Config_::ClientMeta::get(&null_meta_store, 0); + } + + return meta_slab->get_meta_for_object(index); + } + + /** + * @brief Checks that the supplied size of the allocation matches the size + * snmalloc believes the allocation is. Only performs the check if + * mitigations(sanity_checks) + * is enabled. + */ + template + SNMALLOC_FAST_PATH_INLINE void check_size(void* p, size_t size) + { + if constexpr (mitigations(sanity_checks)) + { + const auto& entry = Config_::Backend::get_metaentry(address_cast(p)); + if (!entry.is_owned()) + return; + size = size == 0 ? 1 : size; + auto sc = size_to_sizeclass_full(size); + auto pm_sc = entry.get_sizeclass(); + auto rsize = sizeclass_full_to_size(sc); + auto pm_size = sizeclass_full_to_size(pm_sc); + snmalloc_check_client( + mitigations(sanity_checks), + (sc == pm_sc) || (p == nullptr), + "Dealloc rounded size mismatch: {} != {}", + rsize, + pm_size); + } + else + UNUSED(p, size); + } + + template + SNMALLOC_FAST_PATH_INLINE size_t alloc_size(const void* p_raw) + { + const auto& entry = Config_::Backend::get_metaentry(address_cast(p_raw)); + + if (SNMALLOC_UNLIKELY( + !SecondaryAllocator::pass_through && !entry.is_owned() && + p_raw != nullptr)) + return SecondaryAllocator::alloc_size(p_raw); + // TODO What's the domestication policy here? At the moment we just + // probe the pagemap with the raw address, without checks. There could + // be implicit domestication through the `Config::Pagemap` or + // we could just leave well enough alone. + + // Note that alloc_size should return 0 for nullptr. + // Other than nullptr, we know the system will be initialised as it must + // be called with something we have already allocated. + // + // To handle this case we require the uninitialised pagemap contain an + // entry for the first chunk of memory, that states it represents a + // large object, so we can pull the check for null off the fast path. + + return sizeclass_full_to_size(entry.get_sizeclass()); + } + + template + SNMALLOC_FAST_PATH_INLINE void* alloc() + { + return ThreadAlloc::get().alloc(aligned_size(align, size)); + } + + template + SNMALLOC_FAST_PATH_INLINE void* alloc(size_t size) + { + return ThreadAlloc::get().alloc(aligned_size(align, size)); + } + + template + SNMALLOC_FAST_PATH_INLINE void* alloc_aligned(size_t align, size_t size) + { + return ThreadAlloc::get().alloc(aligned_size(align, size)); + } + + SNMALLOC_FAST_PATH_INLINE void dealloc(void* p) + { + ThreadAlloc::get().dealloc(p); + } + + SNMALLOC_FAST_PATH_INLINE void dealloc(void* p, size_t size) + { + check_size(p, size); + ThreadAlloc::get().dealloc(p); + } + + template + SNMALLOC_FAST_PATH_INLINE void dealloc(void* p) + { + check_size(p, size); + ThreadAlloc::get().dealloc(p); + } + + SNMALLOC_FAST_PATH_INLINE void dealloc(void* p, size_t size, size_t align) + { + auto rsize = aligned_size(align, size); + check_size(p, rsize); + ThreadAlloc::get().dealloc(p); + } + + SNMALLOC_FAST_PATH_INLINE void debug_teardown() + { + return ThreadAlloc::get().teardown(); + } + + template + SNMALLOC_FAST_PATH_INLINE bool is_owned(void* p) + { + const auto& entry = Config_::Backend::get_metaentry(address_cast(p)); + return entry.is_owned(); + } +} // namespace snmalloc diff --git a/src/snmalloc/global/libc.h b/src/snmalloc/global/libc.h index 2ce178a96..73e178335 100644 --- a/src/snmalloc/global/libc.h +++ b/src/snmalloc/global/libc.h @@ -1,6 +1,6 @@ #pragma once -#include "threadalloc.h" +#include "globalalloc.h" #include #include @@ -21,22 +21,22 @@ namespace snmalloc::libc inline void* __malloc_end_pointer(void* ptr) { - return ThreadAlloc::get().external_pointer(ptr); + return snmalloc::external_pointer(ptr); } SNMALLOC_FAST_PATH_INLINE void* malloc(size_t size) { - return ThreadAlloc::get().alloc(size); + return snmalloc::alloc(size); } SNMALLOC_FAST_PATH_INLINE void free(void* ptr) { - ThreadAlloc::get().dealloc(ptr); + dealloc(ptr); } SNMALLOC_FAST_PATH_INLINE void free_sized(void* ptr, size_t size) { - ThreadAlloc::get().dealloc(ptr, size); + dealloc(ptr, size); } SNMALLOC_FAST_PATH_INLINE void* calloc(size_t nmemb, size_t size) @@ -47,27 +47,19 @@ namespace snmalloc::libc { return set_error(); } - return ThreadAlloc::get().alloc(sz); + return alloc(sz); } SNMALLOC_FAST_PATH_INLINE void* realloc(void* ptr, size_t size) { - auto& a = ThreadAlloc::get(); - size_t sz = a.alloc_size(ptr); + size_t sz = alloc_size(ptr); // Keep the current allocation if the given size is in the same sizeclass. if (sz == round_size(size)) { -#ifdef SNMALLOC_PASS_THROUGH - // snmallocs alignment guarantees can be broken by realloc in pass-through - // this is not exercised, by existing clients, but is tested. - if (pointer_align_up(ptr, natural_alignment(size)) == ptr) - return ptr; -#else return ptr; -#endif } - void* p = a.alloc(size); + void* p = alloc(size); if (SNMALLOC_LIKELY(p != nullptr)) { sz = bits::min(size, sz); @@ -78,11 +70,11 @@ namespace snmalloc::libc SNMALLOC_ASSUME(ptr != nullptr); ::memcpy(p, ptr, sz); } - a.dealloc(ptr); + dealloc(ptr); } else if (SNMALLOC_LIKELY(size == 0)) { - a.dealloc(ptr); + dealloc(ptr); } else { @@ -93,7 +85,7 @@ namespace snmalloc::libc inline size_t malloc_usable_size(const void* ptr) { - return ThreadAlloc::get().alloc_size(ptr); + return alloc_size(ptr); } inline void* reallocarray(void* ptr, size_t nmemb, size_t size) @@ -110,7 +102,6 @@ namespace snmalloc::libc inline int reallocarr(void* ptr_, size_t nmemb, size_t size) { int err = errno; - auto& a = ThreadAlloc::get(); bool overflow = false; size_t sz = bits::umul(size, nmemb, overflow); if (SNMALLOC_UNLIKELY(sz == 0)) @@ -124,13 +115,13 @@ namespace snmalloc::libc } void** ptr = reinterpret_cast(ptr_); - void* p = a.alloc(sz); + void* p = alloc(sz); if (SNMALLOC_UNLIKELY(p == nullptr)) { return set_error_and_return(ENOMEM); } - sz = bits::min(sz, a.alloc_size(*ptr)); + sz = bits::min(sz, alloc_size(*ptr)); SNMALLOC_ASSUME(*ptr != nullptr || sz == 0); // Guard memcpy as GCC is assuming not nullptr for ptr after the memcpy @@ -138,7 +129,7 @@ namespace snmalloc::libc if (SNMALLOC_UNLIKELY(sz != 0)) ::memcpy(p, *ptr, sz); errno = err; - a.dealloc(*ptr); + dealloc(*ptr); *ptr = p; return 0; } @@ -150,7 +141,7 @@ namespace snmalloc::libc return set_error(EINVAL); } - return malloc(aligned_size(alignment, size)); + return alloc_aligned(alignment, size); } inline void* aligned_alloc(size_t alignment, size_t size) @@ -175,17 +166,4 @@ namespace snmalloc::libc *memptr = p; return 0; } - - inline typename snmalloc::Alloc::Config::ClientMeta::DataRef - get_client_meta_data(void* p) - { - return ThreadAlloc::get().get_client_meta_data(p); - } - - inline stl::add_const_t - get_client_meta_data_const(void* p) - { - return ThreadAlloc::get().get_client_meta_data_const(p); - } - } // namespace snmalloc::libc diff --git a/src/snmalloc/global/scopedalloc.h b/src/snmalloc/global/scopedalloc.h index 345635a70..5f8c427fa 100644 --- a/src/snmalloc/global/scopedalloc.h +++ b/src/snmalloc/global/scopedalloc.h @@ -15,12 +15,13 @@ namespace snmalloc * This does not depend on thread-local storage working, so can be used for * bootstrapping. */ + template struct ScopedAllocator { /** * The allocator that this wrapper will use. */ - Alloc alloc; + SAlloc alloc; /** * Constructor. Claims an allocator from the global pool @@ -66,7 +67,7 @@ namespace snmalloc * Arrow operator, allows methods exposed by `Alloc` to be called on the * wrapper. */ - Alloc* operator->() + SAlloc* operator->() { return &alloc; } @@ -76,7 +77,8 @@ namespace snmalloc * Returns a new scoped allocator. When the `ScopedAllocator` goes out of * scope, the underlying `Alloc` will be returned to the pool. */ - inline ScopedAllocator get_scoped_allocator() + template + inline ScopedAllocator get_scoped_allocator() { return {}; } diff --git a/src/snmalloc/mem/external_alloc.h b/src/snmalloc/mem/external_alloc.h deleted file mode 100644 index 56d9f9ac6..000000000 --- a/src/snmalloc/mem/external_alloc.h +++ /dev/null @@ -1,89 +0,0 @@ -#pragma once - -#ifdef SNMALLOC_PASS_THROUGH -# if defined(__HAIKU__) -# define _GNU_SOURCE -# endif -# include -# if defined(_WIN32) //|| defined(__APPLE__) -# error "Pass through not supported on this platform" -// The Windows aligned allocation API is not capable of supporting the -// snmalloc API Apple was not providing aligned memory in some tests. -# else -// Defines malloc_size for the platform. -# if defined(_WIN32) -namespace snmalloc::external_alloc -{ - inline size_t malloc_usable_size(void* ptr) - { - return _msize(ptr); - } -} -# elif defined(__APPLE__) -# include - -namespace snmalloc::external_alloc -{ - inline size_t malloc_usable_size(void* ptr) - { - return malloc_size(ptr); - } -} -# elif defined(__linux__) || defined(__HAIKU__) -# include - -namespace snmalloc::external_alloc -{ - using ::malloc_usable_size; -} -# elif defined(__sun) || defined(__NetBSD__) || defined(__OpenBSD__) -namespace snmalloc::external_alloc -{ - using ::malloc_usable_size; -} -# elif defined(__FreeBSD__) -# include - -namespace snmalloc::external_alloc -{ - using ::malloc_usable_size; -} -# elif defined(__DragonFly__) -namespace snmalloc::external_alloc -{ - using ::malloc_usable_size; -} -# else -# error Define malloc size macro for this platform. -# endif -namespace snmalloc::external_alloc -{ - inline void* aligned_alloc(size_t alignment, size_t size) - { - // TSAN complains if allocation is large than this. - if constexpr (bits::BITS == 64) - { - if (size >= 0x10000000000) - { - errno = ENOMEM; - return nullptr; - } - } - - if (alignment < sizeof(void*)) - alignment = sizeof(void*); - - void* result; - int err = posix_memalign(&result, alignment, size); - if (err != 0) - { - errno = err; - result = nullptr; - } - return result; - } - - using ::free; -} -# endif -#endif diff --git a/src/snmalloc/mem/freelist_queue.h b/src/snmalloc/mem/freelist_queue.h index 0688a75a5..53fb69821 100644 --- a/src/snmalloc/mem/freelist_queue.h +++ b/src/snmalloc/mem/freelist_queue.h @@ -52,7 +52,7 @@ namespace snmalloc { SNMALLOC_ASSERT( (address_cast(front.load()) == address_cast(&stub)) || - (back != nullptr)); + (back.load() != nullptr)); } void init() diff --git a/src/snmalloc/mem/globalalloc.h b/src/snmalloc/mem/globalalloc.h deleted file mode 100644 index dc9528f66..000000000 --- a/src/snmalloc/mem/globalalloc.h +++ /dev/null @@ -1,137 +0,0 @@ -#pragma once - -#include "../ds_core/ds_core.h" -#include "localalloc.h" - -namespace snmalloc -{ - template - inline static void cleanup_unused() - { -#ifndef SNMALLOC_PASS_THROUGH - static_assert( - Config::Options.CoreAllocIsPoolAllocated, - "Global cleanup is available only for pool-allocated configurations"); - // Call this periodically to free and coalesce memory allocated by - // allocators that are not currently in use by any thread. - // One atomic operation to extract the stack, another to restore it. - // Handling the message queue for each stack is non-atomic. - auto* first = AllocPool::extract(); - auto* alloc = first; - decltype(alloc) last; - - if (alloc != nullptr) - { - while (alloc != nullptr) - { - alloc->flush(); - last = alloc; - alloc = AllocPool::extract(alloc); - } - - AllocPool::restore(first, last); - } -#endif - } - - /** - If you pass a pointer to a bool, then it returns whether all the - allocators are empty. If you don't pass a pointer to a bool, then will - raise an error all the allocators are not empty. - */ - template - inline static void debug_check_empty(bool* result = nullptr) - { -#ifndef SNMALLOC_PASS_THROUGH - static_assert( - Config::Options.CoreAllocIsPoolAllocated, - "Global status is available only for pool-allocated configurations"); - // This is a debugging function. It checks that all memory from all - // allocators has been freed. - auto* alloc = AllocPool::iterate(); - -# ifdef SNMALLOC_TRACING - message<1024>("debug check empty: first {}", alloc); -# endif - bool done = false; - bool okay = true; - - while (!done) - { -# ifdef SNMALLOC_TRACING - message<1024>("debug_check_empty: Check all allocators!"); -# endif - done = true; - alloc = AllocPool::iterate(); - okay = true; - - while (alloc != nullptr) - { -# ifdef SNMALLOC_TRACING - message<1024>("debug check empty: {}", alloc); -# endif - // Check that the allocator has freed all memory. - // repeat the loop if empty caused message sends. - if (alloc->debug_is_empty(&okay)) - { - done = false; -# ifdef SNMALLOC_TRACING - message<1024>("debug check empty: sent messages {}", alloc); -# endif - } - -# ifdef SNMALLOC_TRACING - message<1024>("debug check empty: okay = {}", okay); -# endif - alloc = AllocPool::iterate(alloc); - } - } - - if (result != nullptr) - { - *result = okay; - return; - } - - // Redo check so abort is on allocator with allocation left. - if (!okay) - { - alloc = AllocPool::iterate(); - while (alloc != nullptr) - { - alloc->debug_is_empty(nullptr); - alloc = AllocPool::iterate(alloc); - } - } -#else - UNUSED(result); -#endif - } - - template - inline static void debug_in_use(size_t count) - { - static_assert( - Config::Options.CoreAllocIsPoolAllocated, - "Global status is available only for pool-allocated configurations"); - auto alloc = AllocPool::iterate(); - while (alloc != nullptr) - { - if (alloc->debug_is_in_use()) - { - if (count == 0) - { - error("ERROR: allocator in use."); - } - count--; - } - alloc = AllocPool::iterate(alloc); - - if (count != 0) - { - error("Error: two few allocators in use."); - } - } - } - -} // namespace snmalloc diff --git a/src/snmalloc/mem/localalloc.h b/src/snmalloc/mem/localalloc.h index 09256a063..4e30fb5fc 100644 --- a/src/snmalloc/mem/localalloc.h +++ b/src/snmalloc/mem/localalloc.h @@ -18,34 +18,12 @@ #include "pool.h" #include "remotecache.h" #include "sizeclasstable.h" - -#ifdef SNMALLOC_PASS_THROUGH -# include "external_alloc.h" -#endif - #include "snmalloc/stl/utility.h" #include namespace snmalloc { - enum Boundary - { - /** - * The location of the first byte of this allocation. - */ - Start, - /** - * The location of the last byte of the allocation. - */ - End, - /** - * The location one past the end of the allocation. This is mostly useful - * for bounds checking, where anything less than this value is safe. - */ - OnePastEnd - }; - /** * A local allocator contains the fast-path allocation routines and * encapsulates all of the behaviour of an allocator that is local to some @@ -270,14 +248,6 @@ namespace snmalloc return local_cache.template alloc(domesticate, size, slowpath); } - /** - * Send all remote deallocation to other threads. - */ - void post_remote_cache() - { - core_alloc->post(); - } - /** * Slow path for deallocation we do not have space for this remote * deallocation. This could be because, @@ -296,12 +266,13 @@ namespace snmalloc message<1024>( "Remote dealloc post {} ({}, {})", p.unsafe_ptr(), - alloc_size(p.unsafe_ptr()), + sizeclass_full_to_size(entry.get_sizeclass()), address_cast(entry.get_slab_metadata())); #endif local_cache.remote_dealloc_cache.template dealloc( entry.get_slab_metadata(), p, &local_cache.entropy); - post_remote_cache(); + + core_alloc->post(); return; } @@ -317,15 +288,6 @@ namespace snmalloc p); } - /** - * Abstracts access to the message queue to handle different - * layout configurations of the allocator. - */ - auto& message_queue() - { - return local_cache.remote_allocator; - } - /** * Call `Config::is_initialised()` if it is implemented, * unconditionally returns true otherwise. @@ -449,16 +411,6 @@ namespace snmalloc template SNMALLOC_FAST_PATH ALLOCATOR void* alloc(size_t size) { -#ifdef SNMALLOC_PASS_THROUGH - // snmalloc guarantees a lot of alignment, so we can depend on this - // make pass through call aligned_alloc with the alignment snmalloc - // would guarantee. - void* result = external_alloc::aligned_alloc( - natural_alignment(size), round_size(size)); - if (zero_mem == YesZero && result != nullptr) - memset(result, 0, size); - return result; -#else // Perform the - 1 on size, so that zero wraps around and ends up on // slow path. if (SNMALLOC_LIKELY( @@ -470,158 +422,9 @@ namespace snmalloc } return capptr_reveal(alloc_not_small(size)); -#endif - } - - /** - * Allocate memory of a statically known size. - */ - template - SNMALLOC_FAST_PATH ALLOCATOR void* alloc() - { - return alloc(size); - } - - /* - * Many of these tests come with an "or is null" branch that they'd need to - * add if we did them up front. Instead, defer them until we're past the - * point where we know, from the pagemap, or by explicitly testing, that the - * pointer under test is not nullptr. - */ - SNMALLOC_FAST_PATH void dealloc_cheri_checks(void* p) - { -#if defined(__CHERI_PURE_CAPABILITY__) - /* - * Enforce the use of an unsealed capability. - * - * TODO In CHERI+MTE, this, is part of the CAmoCDecVersion instruction; - * elide this test in that world. - */ - snmalloc_check_client( - mitigations(cheri_checks), - !__builtin_cheri_sealed_get(p), - "Sealed capability in deallocation"); - - /* - * Enforce permissions on the returned pointer. These pointers end up in - * free queues and will be cycled out to clients again, so try to catch - * erroneous behavior now, rather than later. - * - * TODO In the CHERI+MTE case, we must reconstruct the pointer for the - * free queues as part of the discovery of the start of the object (so - * that it has the correct version), and the CAmoCDecVersion call imposes - * its own requirements on the permissions (to ensure that it's at least - * not zero). They are somewhat more lax than we might wish, so this test - * may remain, guarded by SNMALLOC_CHECK_CLIENT, but no explicit - * permissions checks are required in the non-SNMALLOC_CHECK_CLIENT case - * to defend ourselves or other clients against a misbehaving client. - */ - static const size_t reqperm = CHERI_PERM_LOAD | CHERI_PERM_STORE | - CHERI_PERM_LOAD_CAP | CHERI_PERM_STORE_CAP; - snmalloc_check_client( - mitigations(cheri_checks), - (__builtin_cheri_perms_get(p) & reqperm) == reqperm, - "Insufficient permissions on capability in deallocation"); - - /* - * We check for a valid tag here, rather than in domestication, because - * domestication might be answering a slightly different question, about - * the plausibility of addresses rather than of exact pointers. - * - * TODO Further, in the CHERI+MTE case, the tag check will be implicit in - * a future CAmoCDecVersion instruction, and there should be no harm in - * the lookups we perform along the way to get there. In that world, - * elide this test. - */ - snmalloc_check_client( - mitigations(cheri_checks), - __builtin_cheri_tag_get(p), - "Untagged capability in deallocation"); - - /* - * Verify that the capability is not zero-length, ruling out the other - * edge case around monotonicity. - */ - snmalloc_check_client( - mitigations(cheri_checks), - __builtin_cheri_length_get(p) > 0, - "Zero-length capability in deallocation"); - - /* - * At present we check for the pointer also being the start of an - * allocation closer to dealloc; for small objects, that happens in - * dealloc_local_object_fast, either below or *on the far end of message - * receipt*. For large objects, it happens below by directly rounding to - * power of two rather than using the is_start_of_object helper. - * (XXX This does mean that we might end up threading our remote queue - * state somewhere slightly unexpected rather than at the head of an - * object. That is perhaps fine for now?) - */ - - /* - * TODO - * - * We could enforce other policies here, including that the length exactly - * match the sizeclass. At present, we bound caps we give for allocations - * to the underlying sizeclass, so even malloc(0) will have a non-zero - * length. Monotonicity would then imply that the pointer must be the - * head of an object (modulo, perhaps, temporal aliasing if we somehow - * introduced phase shifts in heap layout like some allocators do). - * - * If we switched to bounding with upwards-rounded representable bounds - * (c.f., CRRL) rather than underlying object size, then we should, - * instead, in general require plausibility of p_raw by checking that its - * length is nonzero and the snmalloc size class associated with its - * length is the one for the slab in question... except for the added - * challenge of malloc(0). Since 0 rounds up to 0, we might end up - * constructing zero-length caps to hand out, which we would then reject - * upon receipt. Instead, as part of introducing CRRL bounds, we should - * introduce a sizeclass for slabs holding zero-size objects. All told, - * we would want to check that - * - * size_to_sizeclass(length) == entry.get_sizeclass() - * - * I believe a relaxed CRRL test of - * - * length > 0 || (length == sizeclass_to_size(entry.get_sizeclass())) - * - * would also suffice and may be slightly less expensive than the test - * above, at the cost of not catching as many misbehaving clients. - * - * In either case, having bounded by CRRL bounds, we would need to be - * *reconstructing* the capabilities headed to our free lists to be given - * out to clients again; there are many more CRRL classes than snmalloc - * sizeclasses (this is the same reason that we can always get away with - * CSetBoundsExact in capptr_bound). Switching to CRRL bounds, if that's - * ever a thing we want to do, will be easier after we've done the - * plumbing for CHERI+MTE. - */ - - /* - * TODO: Unsurprisingly, the CHERI+MTE case once again has something to - * say here. In that world, again, we are certain to be reconstructing - * the capability for the free queue anyway, and so exactly what we wish - * to enforce, length-wise, of the provided capability, is somewhat more - * flexible. Using the provided capability bounds when recoloring memory - * could be a natural way to enforce that it covers the entire object, at - * the cost of a more elaborate recovery story (as we risk aborting with a - * partially recolored object). On non-SNMALLOC_CHECK_CLIENT builds, it - * likely makes sense to just enforce that length > 0 (*not* enforced by - * the CAmoCDecVersion instruction) and say that any authority-bearing - * interior pointer suffices to free the object. I believe that to be an - * acceptable security posture for the allocator and between clients; - * misbehavior is confined to the misbehaving client. - */ -#else - UNUSED(p); -#endif } - // The domestic pointer with its origin allocator - using DomesticInfo = stl::Pair, const PagemapEntry&>; - - // Check whether the raw pointer is owned by snmalloc - SNMALLOC_FAST_PATH_INLINE DomesticInfo get_domestic_info(const void* p_raw) + SNMALLOC_FAST_PATH void dealloc(void* p_raw) { #ifdef __CHERI_PURE_CAPABILITY__ /* @@ -646,22 +449,7 @@ namespace snmalloc capptr_domesticate(core_alloc->backend_state_ptr(), p_wild); const PagemapEntry& entry = Config::Backend::get_metaentry(address_cast(p_tame)); - return {p_tame, entry}; - } - - // Check if a pointer is domestic to SnMalloc - SNMALLOC_FAST_PATH bool is_snmalloc_owned(const void* p_raw) - { - auto [_, entry] = get_domestic_info(p_raw); - RemoteAllocator* remote = entry.get_remote(); - return remote != nullptr; - } - SNMALLOC_FAST_PATH void dealloc(void* p_raw) - { -#ifdef SNMALLOC_PASS_THROUGH - external_alloc::free(p_raw); -#else /* * p_tame may be nullptr, even if p_raw/p_wild are not, in the case * where domestication fails. We exclusively use p_tame below so that @@ -674,8 +462,6 @@ namespace snmalloc * well-formedness) of this pointer. The remainder of the logic will * deal with the object's extent. */ - auto [p_tame, entry] = get_domestic_info(p_raw); - if (SNMALLOC_LIKELY(local_cache.remote_allocator == entry.get_remote())) { dealloc_cheri_checks(p_tame.unsafe_ptr()); @@ -689,8 +475,7 @@ namespace snmalloc SNMALLOC_SLOW_PATH void dealloc_remote(const PagemapEntry& entry, capptr::Alloc p_tame) { - RemoteAllocator* remote = entry.get_remote(); - if (SNMALLOC_LIKELY(remote != nullptr)) + if (SNMALLOC_LIKELY(entry.is_owned())) { dealloc_cheri_checks(p_tame.unsafe_ptr()); @@ -705,13 +490,13 @@ namespace snmalloc { local_cache.remote_dealloc_cache.template dealloc( entry.get_slab_metadata(), p_tame, &local_cache.entropy); -# ifdef SNMALLOC_TRACING +#ifdef SNMALLOC_TRACING message<1024>( "Remote dealloc fast {} ({}, {})", address_cast(p_tame), - alloc_size(p_tame.unsafe_ptr()), + sizeclass_full_to_size(entry.get_sizeclass()), address_cast(entry.get_slab_metadata())); -# endif +#endif return; } @@ -721,54 +506,14 @@ namespace snmalloc if (SNMALLOC_LIKELY(p_tame == nullptr)) { -# ifdef SNMALLOC_TRACING +#ifdef SNMALLOC_TRACING message<1024>("nullptr deallocation"); -# endif +#endif return; } + dealloc_cheri_checks(p_tame.unsafe_ptr()); SecondaryAllocator::deallocate(p_tame.unsafe_ptr()); -#endif - } - - void check_size(void* p, size_t size) - { -#ifdef SNMALLOC_PASS_THROUGH - UNUSED(p, size); -#else - if constexpr (mitigations(sanity_checks)) - { - if (!is_snmalloc_owned(p)) - return; - size = size == 0 ? 1 : size; - auto sc = size_to_sizeclass_full(size); - auto pm_sc = - Config::Backend::get_metaentry(address_cast(p)).get_sizeclass(); - auto rsize = sizeclass_full_to_size(sc); - auto pm_size = sizeclass_full_to_size(pm_sc); - snmalloc_check_client( - mitigations(sanity_checks), - (sc == pm_sc) || (p == nullptr), - "Dealloc rounded size mismatch: {} != {}", - rsize, - pm_size); - } - else - UNUSED(p, size); -#endif - } - - SNMALLOC_FAST_PATH void dealloc(void* p, size_t s) - { - check_size(p, s); - dealloc(p); - } - - template - SNMALLOC_FAST_PATH void dealloc(void* p) - { - check_size(p, size); - dealloc(p); } void teardown() @@ -783,182 +528,6 @@ namespace snmalloc } } - SNMALLOC_FAST_PATH size_t alloc_size(const void* p_raw) - { -#ifdef SNMALLOC_PASS_THROUGH - return external_alloc::malloc_usable_size(const_cast(p_raw)); -#else - - if ( - !SecondaryAllocator::pass_through && !is_snmalloc_owned(p_raw) && - p_raw != nullptr) - return SecondaryAllocator::alloc_size(p_raw); - // TODO What's the domestication policy here? At the moment we just - // probe the pagemap with the raw address, without checks. There could - // be implicit domestication through the `Config::Pagemap` or - // we could just leave well enough alone. - - // Note that alloc_size should return 0 for nullptr. - // Other than nullptr, we know the system will be initialised as it must - // be called with something we have already allocated. - // - // To handle this case we require the uninitialised pagemap contain an - // entry for the first chunk of memory, that states it represents a - // large object, so we can pull the check for null off the fast path. - const PagemapEntry& entry = - Config::Backend::get_metaentry(address_cast(p_raw)); - - return sizeclass_full_to_size(entry.get_sizeclass()); -#endif - } - - /** - * Returns the Start/End of an object allocated by this allocator - * - * It is valid to pass any pointer, if the object was not allocated - * by this allocator, then it give the start and end as the whole of - * the potential pointer space. - */ - template - void* external_pointer(void* p) - { - /* - * Note that: - * * each case uses `pointer_offset`, so that on CHERI, our behaviour is - * monotone with respect to the capability `p`. - * - * * the returned pointer could be outside the CHERI bounds of `p`, and - * thus not something that can be followed. - * - * * we don't use capptr_from_client()/capptr_reveal(), to avoid the - * syntactic clutter. By inspection, `p` flows only to address_cast - * and pointer_offset, and so there's no risk that we follow or act - * to amplify the rights carried by `p`. - */ - if constexpr (location == Start) - { - size_t index = index_in_object(address_cast(p)); - return pointer_offset(p, 0 - index); - } - else if constexpr (location == End) - { - return pointer_offset(p, remaining_bytes(address_cast(p)) - 1); - } - else - { - return pointer_offset(p, remaining_bytes(address_cast(p))); - } - } - - /** - * @brief Get the client meta data for the snmalloc allocation covering this - * pointer. - */ - typename Config::ClientMeta::DataRef get_client_meta_data(void* p) - { - const PagemapEntry& entry = - Config::Backend::get_metaentry(address_cast(p)); - - size_t index = slab_index(entry.get_sizeclass(), address_cast(p)); - - auto* meta_slab = entry.get_slab_metadata(); - - if (SNMALLOC_UNLIKELY(entry.is_backend_owned())) - { - error("Cannot access meta-data for write for freed memory!"); - } - - if (SNMALLOC_UNLIKELY(meta_slab == nullptr)) - { - error( - "Cannot access meta-data for non-snmalloc object in writable form!"); - } - - return meta_slab->get_meta_for_object(index); - } - - /** - * @brief Get the client meta data for the snmalloc allocation covering this - * pointer. - */ - stl::add_const_t - get_client_meta_data_const(void* p) - { - const PagemapEntry& entry = - Config::Backend::template get_metaentry(address_cast(p)); - - size_t index = slab_index(entry.get_sizeclass(), address_cast(p)); - - auto* meta_slab = entry.get_slab_metadata(); - - if (SNMALLOC_UNLIKELY( - (meta_slab == nullptr) || (entry.is_backend_owned()))) - { - static typename Config::ClientMeta::StorageType null_meta_store{}; - return Config::ClientMeta::get(&null_meta_store, 0); - } - - return meta_slab->get_meta_for_object(index); - } - - /** - * Returns the number of remaining bytes in an object. - * - * auto p = (char*)malloc(size) - * remaining_bytes(p + n) == size - n provided n < size - */ - size_t remaining_bytes(address_t p) - { -#ifndef SNMALLOC_PASS_THROUGH - const PagemapEntry& entry = - Config::Backend::template get_metaentry(p); - - auto sizeclass = entry.get_sizeclass(); - return snmalloc::remaining_bytes(sizeclass, p); -#else - constexpr address_t mask = static_cast(-1); - constexpr bool is_signed = mask < 0; - constexpr address_t sign_bit = - bits::one_at_bit(CHAR_BIT * sizeof(address_t) - 1); - if constexpr (is_signed) - { - return (mask ^ sign_bit) - p; - } - else - { - return mask - p; - } -#endif - } - - bool check_bounds(const void* p, size_t s) - { - if (SNMALLOC_LIKELY(Config::is_initialised())) - { - return remaining_bytes(address_cast(p)) >= s; - } - return true; - } - - /** - * Returns the byte offset into an object. - * - * auto p = (char*)malloc(size) - * index_in_object(p + n) == n provided n < size - */ - size_t index_in_object(address_t p) - { -#ifndef SNMALLOC_PASS_THROUGH - const PagemapEntry& entry = - Config::Backend::template get_metaentry(p); - - auto sizeclass = entry.get_sizeclass(); - return snmalloc::index_in_object(sizeclass, p); -#else - return reinterpret_cast(p); -#endif - } - /** * Accessor, returns the local cache. If embedding code is allocating the * core allocator for use by this local allocator then it needs to access diff --git a/src/snmalloc/mem/mem.h b/src/snmalloc/mem/mem.h index 9fb29a985..e0be22082 100644 --- a/src/snmalloc/mem/mem.h +++ b/src/snmalloc/mem/mem.h @@ -2,9 +2,7 @@ #include "backend_wrappers.h" #include "corealloc.h" #include "entropy.h" -#include "external_alloc.h" #include "freelist.h" -#include "globalalloc.h" #include "localalloc.h" #include "localcache.h" #include "metadata.h" diff --git a/src/snmalloc/mem/metadata.h b/src/snmalloc/mem/metadata.h index 2cbe630bd..eb7ebf442 100644 --- a/src/snmalloc/mem/metadata.h +++ b/src/snmalloc/mem/metadata.h @@ -207,6 +207,15 @@ namespace snmalloc get_remote_and_sizeclass())); } + /** + * Returns true if this memory is owned by snmalloc. Some backend memory + * may return false, but all frontend memory will return true. + */ + [[nodiscard]] SNMALLOC_FAST_PATH bool is_owned() const + { + return get_remote() != nullptr; + } + /** * Return the sizeclass. * @@ -708,9 +717,8 @@ namespace snmalloc * Ensure that the template parameter is valid. */ static_assert( - stl::is_convertible_v, - "The front end requires that the back end provides slab metadata that is " - "compatible with the front-end's structure"); + stl::is_base_of_v, + "Template should be a subclass of FrontendSlabMetadata"); public: using SlabMetadata = SlabMetadataType; diff --git a/src/snmalloc/override/jemalloc_compat.cc b/src/snmalloc/override/jemalloc_compat.cc index 0f478451a..fc4d86047 100644 --- a/src/snmalloc/override/jemalloc_compat.cc +++ b/src/snmalloc/override/jemalloc_compat.cc @@ -143,11 +143,11 @@ extern "C" } if (f.should_zero()) { - *ptr = ThreadAlloc::get().alloc(size); + *ptr = alloc(size); } else { - *ptr = ThreadAlloc::get().alloc(size); + *ptr = alloc(size); } return (*ptr != nullptr) ? allocm_success : allocm_err_oom; } @@ -164,12 +164,11 @@ extern "C" void** ptr, size_t* rsize, size_t size, size_t extra, int flags) { auto f = JEMallocFlags(flags); - auto alloc_size = f.aligned_size(size); + auto asize = f.aligned_size(size); - auto& a = ThreadAlloc::get(); - size_t sz = a.alloc_size(*ptr); + size_t sz = alloc_size(*ptr); // Keep the current allocation if the given size is in the same sizeclass. - if (sz == round_size(alloc_size)) + if (sz == round_size(asize)) { if (rsize != nullptr) { @@ -185,25 +184,24 @@ extern "C" if (SIZE_MAX - size > extra) { - alloc_size = f.aligned_size(size + extra); + asize = f.aligned_size(size + extra); } - void* p = - f.should_zero() ? a.alloc(alloc_size) : a.alloc(alloc_size); + void* p = f.should_zero() ? alloc(asize) : alloc(asize); if (SNMALLOC_LIKELY(p != nullptr)) { - sz = bits::min(alloc_size, sz); + sz = bits::min(asize, sz); // Guard memcpy as GCC is assuming not nullptr for ptr after the memcpy // otherwise. if (sz != 0) { memcpy(p, *ptr, sz); } - a.dealloc(*ptr); + dealloc(*ptr); *ptr = p; if (rsize != nullptr) { - *rsize = alloc_size; + *rsize = asize; } return allocm_success; } @@ -217,7 +215,7 @@ extern "C" */ int SNMALLOC_NAME_MANGLE(sallocm)(const void* ptr, size_t* rsize, int) { - *rsize = ThreadAlloc::get().alloc_size(ptr); + *rsize = alloc_size(ptr); return allocm_success; } @@ -228,7 +226,7 @@ extern "C" */ int SNMALLOC_NAME_MANGLE(dallocm)(void* ptr, int) { - ThreadAlloc::get().dealloc(ptr); + dealloc(ptr); return allocm_success; } @@ -257,9 +255,9 @@ extern "C" size = f.aligned_size(size); if (f.should_zero()) { - return ThreadAlloc::get().alloc(size); + return alloc(size); } - return ThreadAlloc::get().alloc(size); + return alloc(size); } /** @@ -274,8 +272,7 @@ extern "C" auto f = JEMallocFlags(flags); size = f.aligned_size(size); - auto& a = ThreadAlloc::get(); - size_t sz = round_size(a.alloc_size(ptr)); + size_t sz = round_size(alloc_size(ptr)); // Keep the current allocation if the given size is in the same sizeclass. if (sz == size) { @@ -292,7 +289,7 @@ extern "C" // allocations, because we get zeroed memory from the PAL and don't zero it // twice. This is not profiled and so should be considered for refactoring // if anyone cares about the performance of these APIs. - void* p = f.should_zero() ? a.alloc(size) : a.alloc(size); + void* p = f.should_zero() ? alloc(size) : alloc(size); if (SNMALLOC_LIKELY(p != nullptr)) { sz = bits::min(size, sz); @@ -300,7 +297,7 @@ extern "C" // otherwise. if (sz != 0) memcpy(p, ptr, sz); - a.dealloc(ptr); + dealloc(ptr); } return p; } @@ -313,8 +310,7 @@ extern "C" */ size_t SNMALLOC_NAME_MANGLE(xallocx)(void* ptr, size_t, size_t, int) { - auto& a = ThreadAlloc::get(); - return a.alloc_size(ptr); + return alloc_size(ptr); } /** @@ -323,8 +319,7 @@ extern "C" */ size_t SNMALLOC_NAME_MANGLE(sallocx)(const void* ptr, int) { - auto& a = ThreadAlloc::get(); - return a.alloc_size(ptr); + return alloc_size(ptr); } /** @@ -334,7 +329,7 @@ extern "C" */ void SNMALLOC_NAME_MANGLE(dallocx)(void* ptr, int) { - ThreadAlloc::get().dealloc(ptr); + dealloc(ptr); } /** @@ -347,7 +342,7 @@ extern "C" */ void SNMALLOC_NAME_MANGLE(sdallocx)(void* ptr, size_t, int) { - ThreadAlloc::get().dealloc(ptr); + dealloc(ptr); } /** diff --git a/src/snmalloc/override/malloc.cc b/src/snmalloc/override/malloc.cc index 4f87a130e..267546785 100644 --- a/src/snmalloc/override/malloc.cc +++ b/src/snmalloc/override/malloc.cc @@ -109,7 +109,7 @@ extern "C" #if __has_include() # include #endif -#if defined(__GLIBC__) && !defined(SNMALLOC_PASS_THROUGH) +#if defined(__GLIBC__) // glibc uses these hooks to replace malloc. // This is required when RTL_DEEPBIND is used and the library is // LD_PRELOADed. diff --git a/src/snmalloc/override/rust.cc b/src/snmalloc/override/rust.cc index 980ba8db5..571f1d3bf 100644 --- a/src/snmalloc/override/rust.cc +++ b/src/snmalloc/override/rust.cc @@ -12,19 +12,19 @@ using namespace snmalloc; extern "C" SNMALLOC_EXPORT void* SNMALLOC_NAME_MANGLE(rust_alloc)(size_t alignment, size_t size) { - return ThreadAlloc::get().alloc(aligned_size(alignment, size)); + return alloc(aligned_size(alignment, size)); } extern "C" SNMALLOC_EXPORT void* SNMALLOC_NAME_MANGLE(rust_alloc_zeroed)(size_t alignment, size_t size) { - return ThreadAlloc::get().alloc(aligned_size(alignment, size)); + return alloc(aligned_size(alignment, size)); } extern "C" SNMALLOC_EXPORT void SNMALLOC_NAME_MANGLE(rust_dealloc)(void* ptr, size_t alignment, size_t size) { - ThreadAlloc::get().dealloc(ptr, aligned_size(alignment, size)); + dealloc(ptr, aligned_size(alignment, size)); } extern "C" SNMALLOC_EXPORT void* SNMALLOC_NAME_MANGLE(rust_realloc)( @@ -36,11 +36,11 @@ extern "C" SNMALLOC_EXPORT void* SNMALLOC_NAME_MANGLE(rust_realloc)( size_to_sizeclass_full(aligned_old_size).raw() == size_to_sizeclass_full(aligned_new_size).raw()) return ptr; - void* p = ThreadAlloc::get().alloc(aligned_new_size); + void* p = alloc(aligned_new_size); if (p) { memcpy(p, ptr, old_size < new_size ? old_size : new_size); - ThreadAlloc::get().dealloc(ptr, aligned_old_size); + dealloc(ptr, aligned_old_size); } return p; } @@ -55,5 +55,5 @@ extern "C" SNMALLOC_EXPORT void SNMALLOC_NAME_MANGLE(rust_statistics)( extern "C" SNMALLOC_EXPORT size_t SNMALLOC_NAME_MANGLE(rust_usable_size)(const void* ptr) { - return ThreadAlloc::get().alloc_size(ptr); + return alloc_size(ptr); } diff --git a/src/snmalloc/pal/pal_ds.h b/src/snmalloc/pal/pal_ds.h index ff1741237..59976b881 100644 --- a/src/snmalloc/pal/pal_ds.h +++ b/src/snmalloc/pal/pal_ds.h @@ -38,7 +38,8 @@ namespace snmalloc /** * Applies function to all the elements of the list */ - void apply_all(function_ref func) + template + void apply_all(F func) { T* curr = elements; while (curr != nullptr) diff --git a/src/snmalloc/snmalloc.h b/src/snmalloc/snmalloc.h index b05b1a330..f55197142 100644 --- a/src/snmalloc/snmalloc.h +++ b/src/snmalloc/snmalloc.h @@ -6,19 +6,19 @@ // Provides the global configuration for the snmalloc implementation. #include "backend/globalconfig.h" +namespace snmalloc +{ // If you define SNMALLOC_PROVIDE_OWN_CONFIG then you must provide your own // definition of `snmalloc::Alloc` before including any files that include // `snmalloc.h` or consume the global allocation APIs. #ifndef SNMALLOC_PROVIDE_OWN_CONFIG -namespace snmalloc -{ + using Config = snmalloc::StandardConfigClientMeta; +#endif /** * Create allocator type for this configuration. */ - using Alloc = snmalloc::LocalAllocator< - snmalloc::StandardConfigClientMeta>; + using Alloc = snmalloc::LocalAllocator; } // namespace snmalloc -#endif // User facing API surface, needs to know what `Alloc` is. #include "snmalloc_front.h" diff --git a/src/snmalloc/stl/cxx/type_traits.h b/src/snmalloc/stl/cxx/type_traits.h index a6b25821b..4e2fbb4dd 100644 --- a/src/snmalloc/stl/cxx/type_traits.h +++ b/src/snmalloc/stl/cxx/type_traits.h @@ -13,19 +13,14 @@ namespace snmalloc using std::bool_constant; using std::conditional; using std::conditional_t; - using std::decay; - using std::decay_t; using std::enable_if; using std::enable_if_t; using std::false_type; using std::has_unique_object_representations_v; using std::integral_constant; - using std::is_array_v; using std::is_base_of_v; - using std::is_convertible_v; using std::is_copy_assignable_v; using std::is_copy_constructible_v; - using std::is_function_v; using std::is_integral; using std::is_integral_v; using std::is_move_assignable_v; @@ -37,7 +32,6 @@ namespace snmalloc using std::remove_const_t; using std::remove_cv; using std::remove_cv_t; - using std::remove_extent_t; using std::remove_reference; using std::remove_reference_t; using std::true_type; diff --git a/src/snmalloc/stl/gnu/type_traits.h b/src/snmalloc/stl/gnu/type_traits.h index 5ae96685f..3840cdbb8 100644 --- a/src/snmalloc/stl/gnu/type_traits.h +++ b/src/snmalloc/stl/gnu/type_traits.h @@ -231,90 +231,6 @@ namespace snmalloc template using remove_reference_t = typename remove_reference::type; - /** - * add_pointer - */ -#if __has_builtin(__add_pointer) - template - using add_pointer_t = __add_pointer(T); -#else - template - auto __add_pointer_impl(int) -> type_identity*>; - template - auto __add_pointer_impl(...) -> type_identity; - - template - struct add_pointer : decltype(__add_pointer_impl(0)) - {}; - - template - using add_pointer_t = typename add_pointer::type; -#endif - /** - * is_array - */ - template - inline constexpr bool is_array_v = __is_array(T); - - /** - * is_function - */ - template - inline constexpr bool is_function_v = __is_function(T); - - /** - * remove_extent - */ - -#if __has_builtin(__remove_extent) - template - using remove_extent_t = __remove_extent(T); -#else - template - struct remove_extent - { - using type = T; - }; - - template - struct remove_extent - { - using type = T; - }; - - template - struct remove_extent - { - using type = T; - }; - - template - using remove_extent_t = typename remove_extent::type; -#endif - - /** - * decay - */ -#if __has_builtin(__decay) - template - using decay_t = __decay(T); -#else - template - class decay - { - using U = remove_reference_t; - - public: - using type = conditional_t< - is_array_v, - add_pointer_t>, - conditional_t, add_pointer_t, remove_cv_t>>; - }; - - template - using decay_t = typename decay::type; -#endif - /** * is_copy_assignable */ @@ -343,12 +259,6 @@ namespace snmalloc inline constexpr bool is_move_constructible_v = __is_constructible(T, add_rvalue_reference_t); - /** - * is_convertible - */ - template - inline constexpr bool is_convertible_v = __is_convertible(From, To); - /** * is_base_of */ diff --git a/src/test/func/cheri/cheri.cc b/src/test/func/cheri/cheri.cc index cde8be071..46e9a3f0a 100644 --- a/src/test/func/cheri/cheri.cc +++ b/src/test/func/cheri/cheri.cc @@ -1,6 +1,6 @@ #include -#if defined(SNMALLOC_PASS_THROUGH) || !defined(__CHERI_PURE_CAPABILITY__) +#if !defined(__CHERI_PURE_CAPABILITY__) // This test does not make sense in pass-through or w/o CHERI int main() { diff --git a/src/test/func/client_meta/client_meta.cc b/src/test/func/client_meta/client_meta.cc index d1d99a341..3e0ae9117 100644 --- a/src/test/func/client_meta/client_meta.cc +++ b/src/test/func/client_meta/client_meta.cc @@ -14,8 +14,8 @@ namespace snmalloc { // Create an allocator that stores an std::atomic> per allocation. - using Alloc = snmalloc::LocalAllocator>>>; + using Config = snmalloc::StandardConfigClientMeta< + ArrayClientMetaDataProvider>>; } #define SNMALLOC_PROVIDE_OWN_CONFIG @@ -23,9 +23,8 @@ namespace snmalloc int main() { -#if defined(SNMALLOC_PASS_THROUGH) || \ - defined(SNMALLOC_ENABLE_GWP_ASAN_INTEGRATION) - // This test does not make sense in pass-through +#if defined(SNMALLOC_ENABLE_GWP_ASAN_INTEGRATION) + // This test does not make sense in GWP-ASan mode. return 0; #else // Allocate a bunch of objects, and store the index into the meta-data. @@ -33,7 +32,7 @@ int main() for (size_t i = 0; i < 10000; i++) { auto p = snmalloc::libc::malloc(1024); - auto& meta = snmalloc::libc::get_client_meta_data(p); + auto& meta = snmalloc::get_client_meta_data(p); meta = i; ptrs.push_back(p); memset(p, (uint8_t)i, 1024); @@ -44,7 +43,7 @@ int main() for (size_t i = 0; i < 10000; i++) { auto p = ptrs[i]; - auto& meta = snmalloc::libc::get_client_meta_data(p); + auto& meta = snmalloc::get_client_meta_data(p); if (meta != i) { std::cout << "Failed at index " << i << std::endl; @@ -63,7 +62,7 @@ int main() // Access in a read-only way meta-data associated with the stack. // This would fail if it was accessed for write. - auto& meta = snmalloc::libc::get_client_meta_data_const(&ptrs); + auto& meta = snmalloc::get_client_meta_data_const(&ptrs); std::cout << "meta for stack" << meta << std::endl; return 0; diff --git a/src/test/func/domestication/domestication.cc b/src/test/func/domestication/domestication.cc index 390bba741..11067d667 100644 --- a/src/test/func/domestication/domestication.cc +++ b/src/test/func/domestication/domestication.cc @@ -1,22 +1,14 @@ #include -#ifdef SNMALLOC_PASS_THROUGH -// This test does not make sense in pass-through -int main() -{ - return 0; -} -#else - // # define SNMALLOC_TRACING -# include -# include -# include -# include +#include +#include +#include +#include // Specify type of allocator -# define SNMALLOC_PROVIDE_OWN_CONFIG +#define SNMALLOC_PROVIDE_OWN_CONFIG namespace snmalloc { @@ -98,9 +90,9 @@ namespace snmalloc if (domesticate_trace) { std::cout << "Domesticating " << p.unsafe_ptr() -# if __has_builtin(__builtin_return_address) +#if __has_builtin(__builtin_return_address) << " from " << __builtin_return_address(0) -# endif +#endif << std::endl; } @@ -121,11 +113,11 @@ namespace snmalloc } }; - using Alloc = LocalAllocator; + using Config = CustomConfig; } -# define SNMALLOC_NAME_MANGLE(a) test_##a -# include +#define SNMALLOC_NAME_MANGLE(a) test_##a +#include int main() { @@ -141,7 +133,7 @@ int main() entropy.make_free_list_key(RemoteAllocator::key_global); entropy.make_free_list_key(freelist::Object::key_root); - auto alloc1 = new Alloc(); + ScopedAllocator alloc1; // Allocate from alloc1; the size doesn't matter a whole lot, it just needs to // be a small object and so definitely owned by this allocator rather. @@ -149,7 +141,7 @@ int main() std::cout << "Allocated p " << p << std::endl; // Put that free object on alloc1's remote queue - auto alloc2 = new Alloc(); + ScopedAllocator alloc2; alloc2->dealloc(p); alloc2->flush(); @@ -184,12 +176,5 @@ int main() static constexpr size_t expected_count = snmalloc::CustomConfig::Options.QueueHeadsAreTame ? 2 : 3; SNMALLOC_CHECK(snmalloc::CustomConfig::domesticate_count == expected_count); - - // Prevent the allocators from going out of scope during the above test - alloc1->flush(); - alloc2->flush(); - return 0; -} - -#endif +} \ No newline at end of file diff --git a/src/test/func/external_pagemap/external_pagemap.cc b/src/test/func/external_pagemap/external_pagemap.cc index 5e79c6a1a..1d9b093dd 100644 --- a/src/test/func/external_pagemap/external_pagemap.cc +++ b/src/test/func/external_pagemap/external_pagemap.cc @@ -1,5 +1,4 @@ -#if defined(SNMALLOC_PASS_THROUGH) || defined(_WIN32) || \ - !defined(TODO_REINSTATE_POSSIBLY) +#if defined(_WIN32) || !defined(TODO_REINSTATE_POSSIBLY) // This test does not make sense with malloc pass-through, skip it. // The malloc definitions are also currently incompatible with Windows headers // so skip this test on Windows as well. diff --git a/src/test/func/first_operation/first_operation.cc b/src/test/func/first_operation/first_operation.cc index 629027fc9..c548d4b0c 100644 --- a/src/test/func/first_operation/first_operation.cc +++ b/src/test/func/first_operation/first_operation.cc @@ -13,29 +13,14 @@ void alloc1(size_t size) { - void* r = snmalloc::ThreadAlloc::get().alloc(size); - snmalloc::ThreadAlloc::get().dealloc(r); + void* r = snmalloc::alloc(size); + snmalloc::dealloc(r); } void alloc2(size_t size) { - auto& a = snmalloc::ThreadAlloc::get(); - void* r = a.alloc(size); - a.dealloc(r); -} - -void alloc3(size_t size) -{ - auto& a = snmalloc::ThreadAlloc::get(); - void* r = a.alloc(size); - a.dealloc(r, size); -} - -void alloc4(size_t size) -{ - auto& a = snmalloc::ThreadAlloc::get(); - void* r = a.alloc(size); - a.dealloc(r); + void* r = snmalloc::alloc(size); + snmalloc::dealloc(r, size); } void check_calloc(void* p, size_t size) @@ -62,79 +47,43 @@ void check_calloc(void* p, size_t size) void calloc1(size_t size) { - void* r = - snmalloc::ThreadAlloc::get().alloc(size); + void* r = snmalloc::alloc(size); check_calloc(r, size); - snmalloc::ThreadAlloc::get().dealloc(r); + snmalloc::dealloc(r); } void calloc2(size_t size) { - auto& a = snmalloc::ThreadAlloc::get(); - void* r = a.alloc(size); - check_calloc(r, size); - a.dealloc(r); -} - -void calloc3(size_t size) -{ - auto& a = snmalloc::ThreadAlloc::get(); - void* r = a.alloc(size); + void* r = snmalloc::alloc(size); check_calloc(r, size); - a.dealloc(r, size); -} - -void calloc4(size_t size) -{ - auto& a = snmalloc::ThreadAlloc::get(); - void* r = a.alloc(size); - check_calloc(r, size); - a.dealloc(r); + snmalloc::dealloc(r, size); } void dealloc1(void* p, size_t) { - snmalloc::ThreadAlloc::get().dealloc(p); + snmalloc::dealloc(p); } void dealloc2(void* p, size_t size) { - snmalloc::ThreadAlloc::get().dealloc(p, size); -} - -void dealloc3(void* p, size_t) -{ - snmalloc::ThreadAlloc::get().dealloc(p); -} - -void dealloc4(void* p, size_t size) -{ - snmalloc::ThreadAlloc::get().dealloc(p, size); + snmalloc::dealloc(p, size); } void f(size_t size) { auto t1 = std::thread(alloc1, size); auto t2 = std::thread(alloc2, size); - auto t3 = std::thread(alloc3, size); - auto t4 = std::thread(alloc4, size); - auto t5 = std::thread(calloc1, size); - auto t6 = std::thread(calloc2, size); - auto t7 = std::thread(calloc3, size); - auto t8 = std::thread(calloc4, size); + auto t3 = std::thread(calloc1, size); + auto t4 = std::thread(calloc2, size); { auto a = snmalloc::get_scoped_allocator(); auto p1 = a->alloc(size); auto p2 = a->alloc(size); - auto p3 = a->alloc(size); - auto p4 = a->alloc(size); - auto t9 = std::thread(dealloc1, p1, size); - auto t10 = std::thread(dealloc2, p2, size); - auto t11 = std::thread(dealloc3, p3, size); - auto t12 = std::thread(dealloc4, p4, size); + auto t5 = std::thread(dealloc1, p1, size); + auto t6 = std::thread(dealloc2, p2, size); t1.join(); t2.join(); @@ -142,14 +91,8 @@ void f(size_t size) t4.join(); t5.join(); t6.join(); - t7.join(); - t8.join(); - t9.join(); - t10.join(); - t11.join(); - t12.join(); } // Drops a. - // snmalloc::current_alloc_pool()->debug_in_use(0); + snmalloc::debug_in_use(0); printf("."); fflush(stdout); } diff --git a/src/test/func/fixed_region/fixed_region.cc b/src/test/func/fixed_region/fixed_region.cc index f5e513b0c..8b11fdcaa 100644 --- a/src/test/func/fixed_region/fixed_region.cc +++ b/src/test/func/fixed_region/fixed_region.cc @@ -17,7 +17,6 @@ using FixedAlloc = LocalAllocator; int main() { -#ifndef SNMALLOC_PASS_THROUGH // Depends on snmalloc specific features setup(); // 28 is large enough to produce a nested allocator. @@ -31,14 +30,14 @@ int main() << pointer_offset(oe_base, size) << std::endl; CustomGlobals::init(nullptr, oe_base, size); - FixedAlloc a; + auto a = get_scoped_allocator(); size_t object_size = 128; size_t count = 0; size_t i = 0; while (true) { - auto r1 = a.alloc(object_size); + auto r1 = a->alloc(object_size); count += object_size; i++; @@ -48,9 +47,9 @@ int main() if (r1 == nullptr) break; - if (!a.is_snmalloc_owned(r1)) + if (!snmalloc::is_owned(r1)) { - a.dealloc(r1); + a->dealloc(r1); continue; } @@ -75,7 +74,4 @@ int main() std::cout << "Total allocated: " << count << " out of " << size << std::endl; std::cout << "Overhead: 1/" << (double)size / (double)(size - count) << std::endl; - - a.teardown(); -#endif } diff --git a/src/test/func/jemalloc/jemalloc.cc b/src/test/func/jemalloc/jemalloc.cc index 0e34e1fc3..ee51421cb 100644 --- a/src/test/func/jemalloc/jemalloc.cc +++ b/src/test/func/jemalloc/jemalloc.cc @@ -318,9 +318,6 @@ extern "C" int main() { -#ifdef SNMALLOC_PASS_THROUGH - return 0; -#endif check_lg_align_macro<63>(); static_assert( OUR_MALLOCX_ZERO == MALLOCX_ZERO, "Our MALLOCX_ZERO macro is wrong"); diff --git a/src/test/func/malloc/malloc.cc b/src/test/func/malloc/malloc.cc index 14440185d..a2b5bffbf 100644 --- a/src/test/func/malloc/malloc.cc +++ b/src/test/func/malloc/malloc.cc @@ -33,21 +33,7 @@ void check_result(size_t size, size_t align, void* p, int err, bool null) } const auto alloc_size = our_malloc_usable_size(p); auto expected_size = our_malloc_good_size(size); -#ifdef SNMALLOC_PASS_THROUGH - // Calling system allocator may allocate a larger block than - // snmalloc. Note, we have called the system allocator with - // the size snmalloc would allocate, so it won't be smaller. - const auto exact_size = false; - // We allocate MIN_ALLOC_SIZE byte for 0-sized allocations (and so round_size - // will tell us that the minimum size is MIN_ALLOC_SIZE), but the system - // allocator may return a 0-sized allocation. - if (size == 0) - { - expected_size = 0; - } -#else const auto exact_size = align == 1; -#endif #ifdef __CHERI_PURE_CAPABILITY__ const auto cheri_size = __builtin_cheri_length_get(p); if (cheri_size != alloc_size && (size != 0)) @@ -376,6 +362,6 @@ int main(int argc, char** argv) our_malloc_usable_size(nullptr) == 0, "malloc_usable_size(nullptr) should be zero"); - snmalloc::debug_check_empty(); + snmalloc::debug_check_empty(); return 0; } diff --git a/src/test/func/memcpy/func-memcpy.cc b/src/test/func/memcpy/func-memcpy.cc index 16efde3ef..eda741c83 100644 --- a/src/test/func/memcpy/func-memcpy.cc +++ b/src/test/func/memcpy/func-memcpy.cc @@ -19,12 +19,7 @@ int main() # endif # define SNMALLOC_FAIL_FAST false # define SNMALLOC_STATIC_LIBRARY_PREFIX my_ -# ifndef SNMALLOC_PASS_THROUGH -# include "snmalloc/override/malloc.cc" -# else -# define my_malloc(x) malloc(x) -# define my_free(x) free(x) -# endif +# include "snmalloc/override/malloc.cc" # include "snmalloc/override/memcpy.cc" # include "test/helpers.h" @@ -150,9 +145,6 @@ void check_bounds(size_t size, size_t out_of_bounds) int main() { - // Skip the checks that expect bounds checks to fail when we are not the - // malloc implementation. -# if !defined(SNMALLOC_PASS_THROUGH) // Some sizes to check for out-of-bounds access. As we are only able to // catch overflows past the end of the sizeclass-padded allocation, make // sure we don't try to test on smaller allocations. @@ -173,10 +165,9 @@ int main() // Check one object out of bounds check_bounds(sz, sz); } -# endif for (size_t x = 0; x < 2048; x++) { check_size(x); } } -#endif +#endif \ No newline at end of file diff --git a/src/test/func/memory/memory.cc b/src/test/func/memory/memory.cc index 6b52e89a8..f80b5a5c7 100644 --- a/src/test/func/memory/memory.cc +++ b/src/test/func/memory/memory.cc @@ -69,10 +69,9 @@ void test_limited(rlim64_t as_limit, size_t& count) upper_bound, static_cast(info.freeram >> 3u)); std::cout << "trying to alloc " << upper_bound / KiB << " KiB" << std::endl; # endif - auto& alloc = ThreadAlloc::get(); std::cout << "allocator initialised" << std::endl; - auto chunk = alloc.alloc(upper_bound); - alloc.dealloc(chunk); + auto chunk = snmalloc::alloc(upper_bound); + snmalloc::dealloc(chunk); std::cout << "success" << std::endl; std::exit(0); } @@ -91,8 +90,6 @@ void test_limited(rlim64_t as_limit, size_t& count) void test_alloc_dealloc_64k() { - auto& alloc = ThreadAlloc::get(); - constexpr size_t count = 1 << 12; constexpr size_t outer_count = 12; void* garbage[count]; @@ -104,26 +101,25 @@ void test_alloc_dealloc_64k() // This will fill the short slab, and then start a new slab. for (size_t i = 0; i < count; i++) { - garbage[i] = alloc.alloc(16); + garbage[i] = snmalloc::alloc(16); } // Allocate one object on the second slab - keep_alive[j] = alloc.alloc(16); + keep_alive[j] = snmalloc::alloc(16); for (size_t i = 0; i < count; i++) { - alloc.dealloc(garbage[i]); + snmalloc::dealloc(garbage[i]); } } for (size_t j = 0; j < outer_count; j++) { - alloc.dealloc(keep_alive[j]); + snmalloc::dealloc(keep_alive[j]); } } void test_random_allocation() { - auto& alloc = ThreadAlloc::get(); std::unordered_set allocated; constexpr size_t count = 10000; @@ -146,13 +142,13 @@ void test_random_allocation() if (cell != nullptr) { allocated.erase(cell); - alloc.dealloc(cell); + snmalloc::dealloc(cell); cell = nullptr; alloc_count--; } if (!just_dealloc) { - cell = alloc.alloc(16); + cell = snmalloc::alloc(16); auto pair = allocated.insert(cell); // Check not already allocated SNMALLOC_CHECK(pair.second); @@ -170,20 +166,18 @@ void test_random_allocation() // Deallocate all the remaining objects for (size_t i = 0; i < count; i++) if (objects[i] != nullptr) - alloc.dealloc(objects[i]); + snmalloc::dealloc(objects[i]); } void test_calloc() { - auto& alloc = ThreadAlloc::get(); - for (size_t size = 16; size <= (1 << 24); size <<= 1) { - void* p = alloc.alloc(size); + void* p = snmalloc::alloc(size); memset(p, 0xFF, size); - alloc.dealloc(p, size); + snmalloc::dealloc(p, size); - p = alloc.alloc(size); + p = snmalloc::alloc(size); for (size_t i = 0; i < size; i++) { @@ -191,10 +185,10 @@ void test_calloc() abort(); } - alloc.dealloc(p, size); + snmalloc::dealloc(p, size); } - snmalloc::debug_check_empty(); + snmalloc::debug_check_empty(); } void test_double_alloc() @@ -227,48 +221,46 @@ void test_double_alloc() while (!set1.empty()) { auto it = set1.begin(); - a2->dealloc(*it, 20); + a2->dealloc(*it); set1.erase(it); } while (!set2.empty()) { auto it = set2.begin(); - a1->dealloc(*it, 20); + a1->dealloc(*it); set2.erase(it); } } } - snmalloc::debug_check_empty(); + snmalloc::debug_check_empty(); } void test_external_pointer() { - // Malloc does not have an external pointer querying mechanism. - auto& alloc = ThreadAlloc::get(); - for (snmalloc::smallsizeclass_t sc = size_to_sizeclass(MIN_ALLOC_SIZE); sc < NUM_SMALL_SIZECLASSES; sc++) { size_t size = sizeclass_to_size(sc); - void* p1 = alloc.alloc(size); + void* p1 = snmalloc::alloc(size); - if (size != alloc.alloc_size(p1)) + if (size != snmalloc::alloc_size(p1)) { std::cout << "Requested size: " << size - << " alloc_size: " << alloc.alloc_size(p1) << std::endl; + << " alloc_size: " << snmalloc::alloc_size(p1) << std::endl; abort(); } for (size_t offset = 0; offset < size; offset += 17) { void* p2 = pointer_offset(p1, offset); - void* p3 = alloc.external_pointer(p2); - void* p4 = alloc.external_pointer(p2); + void* p3 = snmalloc::external_pointer(p2); + void* p4 = snmalloc::external_pointer(p2); if (p1 != p3) { - std::cout << "size: " << size << " alloc_size: " << alloc.alloc_size(p1) + std::cout << "size: " << size + << " alloc_size: " << snmalloc::alloc_size(p1) << " offset: " << offset << " p1: " << p1 << " p3: " << p3 << std::endl; } @@ -282,16 +274,15 @@ void test_external_pointer() SNMALLOC_CHECK((size_t)p4 == (size_t)p1 + size - 1); } - alloc.dealloc(p1, size); + snmalloc::dealloc(p1, size); } - snmalloc::debug_check_empty(); + snmalloc::debug_check_empty(); }; void check_offset(void* base, void* interior) { - auto& alloc = ThreadAlloc::get(); - void* calced_base = alloc.external_pointer((void*)interior); + void* calced_base = snmalloc::external_pointer((void*)interior); if (calced_base != (void*)base) { std::cout << "Calced base: " << calced_base << " actual base: " << base @@ -315,8 +306,6 @@ void test_external_pointer_large() { xoroshiro::p128r64 r; - auto& alloc = ThreadAlloc::get(); - constexpr size_t count_log = DefaultPal::address_bits > 32 ? 5 : 3; constexpr size_t count = 1 << count_log; // Pre allocate all the objects @@ -331,9 +320,9 @@ void test_external_pointer_large() size_t size = (1 << 24) + rand; total_size += size; // store object - objects[i] = (size_t*)alloc.alloc(size); + objects[i] = (size_t*)snmalloc::alloc(size); // Store allocators size for this object - *objects[i] = alloc.alloc_size(objects[i]); + *objects[i] = snmalloc::alloc_size(objects[i]); check_external_pointer_large(objects[i]); if (i > 0) @@ -351,33 +340,32 @@ void test_external_pointer_large() // Deallocate everything for (size_t i = 0; i < count; i++) { - alloc.dealloc(objects[i]); + snmalloc::dealloc(objects[i]); } } void test_external_pointer_dealloc_bug() { std::cout << "Testing external pointer dealloc bug" << std::endl; - auto& alloc = ThreadAlloc::get(); constexpr size_t count = MIN_CHUNK_SIZE; void* allocs[count]; for (size_t i = 0; i < count; i++) { - allocs[i] = alloc.alloc(MIN_CHUNK_BITS / 2); + allocs[i] = snmalloc::alloc(MIN_CHUNK_BITS / 2); } for (size_t i = 1; i < count; i++) { - alloc.dealloc(allocs[i]); + snmalloc::dealloc(allocs[i]); } for (size_t i = 0; i < count; i++) { - alloc.external_pointer(allocs[i]); + snmalloc::external_pointer(allocs[i]); } - alloc.dealloc(allocs[0]); + snmalloc::dealloc(allocs[0]); std::cout << "Testing external pointer dealloc bug - done" << std::endl; } @@ -387,15 +375,12 @@ void test_external_pointer_stack() std::array stack; - auto& alloc = ThreadAlloc::get(); - for (size_t i = 0; i < stack.size(); i++) { - if (alloc.external_pointer(&stack[i]) > &stack[i]) + if (snmalloc::external_pointer(&stack[i]) > &stack[i]) { - std::cout << "Stack pointer: " << &stack[i] - << " external pointer: " << alloc.external_pointer(&stack[i]) - << std::endl; + std::cout << "Stack pointer: " << &stack[i] << " external pointer: " + << snmalloc::external_pointer(&stack[i]) << std::endl; abort(); } } @@ -405,92 +390,97 @@ void test_external_pointer_stack() void test_alloc_16M() { - auto& alloc = ThreadAlloc::get(); // sizes >= 16M use large_alloc const size_t size = 16'000'000; - void* p1 = alloc.alloc(size); - SNMALLOC_CHECK(alloc.alloc_size(alloc.external_pointer(p1)) >= size); - alloc.dealloc(p1); + void* p1 = snmalloc::alloc(size); + SNMALLOC_CHECK(snmalloc::alloc_size(snmalloc::external_pointer(p1)) >= size); + snmalloc::dealloc(p1); } void test_calloc_16M() { - auto& alloc = ThreadAlloc::get(); // sizes >= 16M use large_alloc const size_t size = 16'000'000; - void* p1 = alloc.alloc(size); - SNMALLOC_CHECK(alloc.alloc_size(alloc.external_pointer(p1)) >= size); - alloc.dealloc(p1); + void* p1 = snmalloc::alloc(size); + SNMALLOC_CHECK(snmalloc::alloc_size(snmalloc::external_pointer(p1)) >= size); + snmalloc::dealloc(p1); } void test_calloc_large_bug() { - auto& alloc = ThreadAlloc::get(); // Perform large calloc, to check for correct zeroing from PAL. // Some PALS have special paths for PAGE aligned zeroing of large // allocations. This is a large allocation that is intentionally // not a multiple of page size. const size_t size = (MAX_SMALL_SIZECLASS_SIZE << 3) - 7; - void* p1 = alloc.alloc(size); - SNMALLOC_CHECK(alloc.alloc_size(alloc.external_pointer(p1)) >= size); - alloc.dealloc(p1); + void* p1 = snmalloc::alloc(size); + SNMALLOC_CHECK(snmalloc::alloc_size(snmalloc::external_pointer(p1)) >= size); + snmalloc::dealloc(p1); } -template +template void test_static_sized_alloc() { - auto& alloc = ThreadAlloc::get(); - auto p = alloc.alloc(); + auto p = snmalloc::alloc(); static_assert((dealloc >= 0) && (dealloc <= 2), "bad dealloc flavor"); switch (dealloc) { case 0: - alloc.dealloc(p); + snmalloc::dealloc(p); break; case 1: - alloc.dealloc(p, asz); + snmalloc::dealloc(p, asz); break; case 2: - alloc.dealloc(p); + snmalloc::dealloc(p); break; } + + if constexpr (dealloc != 0) + test_static_sized_alloc(); } +template void test_static_sized_allocs() { - // For each small, medium, and large class, do each kind dealloc. This is - // mostly to ensure that all of these forms compile. - for (size_t sc = 0; sc < NUM_SMALL_SIZECLASSES; sc++) - { - // test_static_sized_alloc(); - // test_static_sized_alloc(); - // test_static_sized_alloc(); - } - // test_static_sized_alloc(); - // test_static_sized_alloc(); - // test_static_sized_alloc(); - - // test_static_sized_alloc(); - // test_static_sized_alloc(); - // test_static_sized_alloc(); + if (max_size < 16) + return; + + constexpr size_t next_size = max_size >> 1; + test_static_sized_allocs(); + + test_static_sized_alloc(); + test_static_sized_alloc(); + test_static_sized_alloc(); + test_static_sized_alloc(); + + test_static_sized_alloc(); + test_static_sized_alloc(); + test_static_sized_alloc(); + test_static_sized_alloc(); + + test_static_sized_alloc(); + test_static_sized_alloc(); + test_static_sized_alloc(); + test_static_sized_alloc(); } void test_remaining_bytes() { - auto& alloc = ThreadAlloc::get(); for (snmalloc::smallsizeclass_t sc = size_to_sizeclass(MIN_ALLOC_SIZE); sc < NUM_SMALL_SIZECLASSES; sc++) { auto size = sizeclass_to_size(sc); - char* p = (char*)alloc.alloc(size); + char* p = (char*)snmalloc::alloc(size); for (size_t offset = 0; offset < size; offset++) { - auto rem = alloc.remaining_bytes(address_cast(pointer_offset(p, offset))); + auto rem = + snmalloc::remaining_bytes(address_cast(pointer_offset(p, offset))); if (rem != (size - offset)) { printf( @@ -503,7 +493,7 @@ void test_remaining_bytes() abort(); } } - alloc.dealloc(p); + snmalloc::dealloc(p); } } @@ -513,18 +503,16 @@ void test_consolidaton_bug() * Check for consolidation of various sizes, but allocating and deallocating, * then requesting larger sizes. See issue #506 */ - auto& alloc = ThreadAlloc::get(); - for (size_t i = 0; i < 27; i++) { std::vector allocs; for (size_t j = 0; j < 4; j++) { - allocs.push_back(alloc.alloc(bits::one_at_bit(i))); + allocs.push_back(snmalloc::alloc(bits::one_at_bit(i))); } for (auto a : allocs) { - alloc.dealloc(a); + snmalloc::dealloc(a); } } } @@ -557,7 +545,6 @@ int main(int argc, char** argv) test_random_allocation(); test_calloc(); test_double_alloc(); -#ifndef SNMALLOC_PASS_THROUGH // Depends on snmalloc specific features test_remaining_bytes(); test_static_sized_allocs(); test_calloc_large_bug(); @@ -567,7 +554,6 @@ int main(int argc, char** argv) test_external_pointer(); test_alloc_16M(); test_calloc_16M(); -#endif test_consolidaton_bug(); return 0; } diff --git a/src/test/func/memory_usage/memory_usage.cc b/src/test/func/memory_usage/memory_usage.cc index 6d7dc40f6..68fa179f7 100644 --- a/src/test/func/memory_usage/memory_usage.cc +++ b/src/test/func/memory_usage/memory_usage.cc @@ -79,7 +79,6 @@ int main(int argc, char** argv) { UNUSED(argc); UNUSED(argv); -#ifndef SNMALLOC_PASS_THROUGH // Depends on snmalloc specific features setup(); add_n_allocs(5); @@ -103,5 +102,4 @@ int main(int argc, char** argv) remove_n_allocs(3); std::cout << "Teardown complete!" << std::endl; -#endif } diff --git a/src/test/func/miracle_ptr/miracle_ptr.cc b/src/test/func/miracle_ptr/miracle_ptr.cc index 8577dfb5e..f0504597b 100644 --- a/src/test/func/miracle_ptr/miracle_ptr.cc +++ b/src/test/func/miracle_ptr/miracle_ptr.cc @@ -25,8 +25,8 @@ namespace snmalloc { // Instantiate the allocator with a client meta data provider that uses an // atomic size_t to store the reference count. - using Alloc = snmalloc::LocalAllocator>>>; + using Config = snmalloc::StandardConfigClientMeta< + ArrayClientMetaDataProvider>>; } # define SNMALLOC_PROVIDE_OWN_CONFIG @@ -58,7 +58,7 @@ namespace snmalloc::miracle if (SNMALLOC_UNLIKELY(p == nullptr)) return nullptr; - snmalloc::libc::get_client_meta_data(p) = 1; + snmalloc::get_client_meta_data(p) = 1; return p; } @@ -68,8 +68,7 @@ namespace snmalloc::miracle return; // TODO could build a check into this that it is the start of the object? - auto previous = - snmalloc::libc::get_client_meta_data(ptr).fetch_add((size_t)-1); + auto previous = snmalloc::get_client_meta_data(ptr).fetch_add((size_t)-1); if (SNMALLOC_LIKELY(previous == 1)) { @@ -88,8 +87,7 @@ namespace snmalloc::miracle inline void acquire(void* p) { - auto previous = - snmalloc::libc::get_client_meta_data(p).fetch_add((size_t)2); + auto previous = snmalloc::get_client_meta_data(p).fetch_add((size_t)2); // Can we take new pointers to a deallocated object? check((previous & 1) == 1, "Acquiring a deallocated object"); @@ -97,8 +95,7 @@ namespace snmalloc::miracle inline void release(void* p) { - auto previous = - snmalloc::libc::get_client_meta_data(p).fetch_add((size_t)-2); + auto previous = snmalloc::get_client_meta_data(p).fetch_add((size_t)-2); if (previous > 2) return; @@ -185,7 +182,6 @@ void operator delete(void* p, size_t) int main() { -# ifndef SNMALLOC_PASS_THROUGH snmalloc::miracle::raw_ptr p; { auto up1 = std::make_unique(41); @@ -199,7 +195,6 @@ int main() // raw_ptr has kept the memory live. // Current implementation zeros the memory when the unique_ptr is destroyed. check(*p == 0, "Failed to keep memory live"); -# endif return 0; } #endif diff --git a/src/test/func/sandbox/sandbox.cc b/src/test/func/sandbox/sandbox.cc index 69ce99cfa..03c288789 100644 --- a/src/test/func/sandbox/sandbox.cc +++ b/src/test/func/sandbox/sandbox.cc @@ -1,4 +1,4 @@ -#if defined(SNMALLOC_PASS_THROUGH) || true +#if true /* * This test does not make sense with malloc pass-through, skip it. */ @@ -244,7 +244,7 @@ namespace // Use the outside-sandbox snmalloc to allocate memory, rather than using // the PAL directly, so that our out-of-sandbox can amplify sandbox // pointers - return ThreadAlloc::get().alloc(sb_size); + return snmalloc::alloc(sb_size); } }; } @@ -260,7 +260,7 @@ int main() auto check = [](Sandbox& sb, auto& alloc, size_t sz) { void* ptr = alloc.alloc(sz); SNMALLOC_CHECK(sb.is_in_sandbox_heap(ptr, sz)); - ThreadAlloc::get().dealloc(ptr); + snmalloc::dealloc(ptr); }; auto check_with_sb = [&](Sandbox& sb) { // Check with a range of sizes diff --git a/src/test/func/statistics/stats.cc b/src/test/func/statistics/stats.cc index 214a0bcf3..d66f060a1 100644 --- a/src/test/func/statistics/stats.cc +++ b/src/test/func/statistics/stats.cc @@ -1,23 +1,16 @@ -#ifdef SNMALLOC_PASS_THROUGH // This test depends on snmalloc internals -int main() -{ - return 0; -} -#else -# include -# include -# include +#include +#include +#include template void debug_check_empty_1() { std::cout << "debug_check_empty_1 " << size << std::endl; - snmalloc::Alloc& a = snmalloc::ThreadAlloc::get(); bool result; - auto r = a.alloc(size); + auto r = snmalloc::alloc(size); - snmalloc::debug_check_empty(&result); + snmalloc::debug_check_empty(&result); if (result != false) { std::cout << "debug_check_empty failed to detect leaked memory:" << size @@ -25,18 +18,18 @@ void debug_check_empty_1() abort(); } - a.dealloc(r); + snmalloc::dealloc(r); - snmalloc::debug_check_empty(&result); + snmalloc::debug_check_empty(&result); if (result != true) { std::cout << "debug_check_empty failed to say empty:" << size << std::endl; abort(); } - r = a.alloc(size); + r = snmalloc::alloc(size); - snmalloc::debug_check_empty(&result); + snmalloc::debug_check_empty(&result); if (result != false) { std::cout << "debug_check_empty failed to detect leaked memory:" << size @@ -44,9 +37,9 @@ void debug_check_empty_1() abort(); } - a.dealloc(r); + snmalloc::dealloc(r); - snmalloc::debug_check_empty(&result); + snmalloc::debug_check_empty(&result); if (result != true) { std::cout << "debug_check_empty failed to say empty:" << size << std::endl; @@ -58,7 +51,6 @@ template void debug_check_empty_2() { std::cout << "debug_check_empty_2 " << size << std::endl; - snmalloc::Alloc& a = snmalloc::ThreadAlloc::get(); bool result; std::vector allocs; // 1GB of allocations @@ -70,9 +62,9 @@ void debug_check_empty_2() { std::cout << "." << std::flush; } - auto r = a.alloc(size); + auto r = snmalloc::alloc(size); allocs.push_back(r); - snmalloc::debug_check_empty(&result); + snmalloc::debug_check_empty(&result); if (result != false) { std::cout << "False empty after " << i << " allocations of " << size @@ -88,17 +80,17 @@ void debug_check_empty_2() { std::cout << "." << std::flush; } - snmalloc::debug_check_empty(&result); + snmalloc::debug_check_empty(&result); if (result != false) { std::cout << "False empty after " << i << " deallocations of " << size << std::endl; abort(); } - a.dealloc(allocs[i]); + snmalloc::dealloc(allocs[i]); } std::cout << std::endl; - snmalloc::debug_check_empty(); + snmalloc::debug_check_empty(); } int main() @@ -115,4 +107,3 @@ int main() return 0; } -#endif \ No newline at end of file diff --git a/src/test/func/teardown/teardown.cc b/src/test/func/teardown/teardown.cc index f68ed4d03..ec5bce4b4 100644 --- a/src/test/func/teardown/teardown.cc +++ b/src/test/func/teardown/teardown.cc @@ -13,43 +13,25 @@ void trigger_teardown() { - auto& a = snmalloc::ThreadAlloc::get(); // Trigger init - void* r = a.alloc(16); - a.dealloc(r); + void* r = snmalloc::alloc(16); + snmalloc::dealloc(r); // Force teardown - a.teardown(); + snmalloc::debug_teardown(); } void alloc1(size_t size) { trigger_teardown(); - void* r = snmalloc::ThreadAlloc::get().alloc(size); - snmalloc::ThreadAlloc::get().dealloc(r); + void* r = snmalloc::alloc(size); + snmalloc::dealloc(r); } void alloc2(size_t size) { trigger_teardown(); - auto& a = snmalloc::ThreadAlloc::get(); - void* r = a.alloc(size); - a.dealloc(r); -} - -void alloc3(size_t size) -{ - trigger_teardown(); - auto& a = snmalloc::ThreadAlloc::get(); - void* r = a.alloc(size); - a.dealloc(r, size); -} - -void alloc4(size_t size) -{ - trigger_teardown(); - auto& a = snmalloc::ThreadAlloc::get(); - void* r = a.alloc(size); - a.dealloc(r); + void* r = snmalloc::alloc(size); + snmalloc::dealloc(r, size); } void check_calloc(void* p, size_t size) @@ -77,86 +59,46 @@ void check_calloc(void* p, size_t size) void calloc1(size_t size) { trigger_teardown(); - void* r = - snmalloc::ThreadAlloc::get().alloc(size); + void* r = snmalloc::alloc(size); check_calloc(r, size); - snmalloc::ThreadAlloc::get().dealloc(r); + snmalloc::dealloc(r); } void calloc2(size_t size) { trigger_teardown(); - auto& a = snmalloc::ThreadAlloc::get(); - void* r = a.alloc(size); - check_calloc(r, size); - a.dealloc(r); -} - -void calloc3(size_t size) -{ - trigger_teardown(); - auto& a = snmalloc::ThreadAlloc::get(); - void* r = a.alloc(size); + void* r = snmalloc::alloc(size); check_calloc(r, size); - a.dealloc(r, size); -} - -void calloc4(size_t size) -{ - trigger_teardown(); - auto& a = snmalloc::ThreadAlloc::get(); - void* r = a.alloc(size); - check_calloc(r, size); - a.dealloc(r); + snmalloc::dealloc(r, size); } void dealloc1(void* p, size_t) { trigger_teardown(); - snmalloc::ThreadAlloc::get().dealloc(p); + snmalloc::dealloc(p); } void dealloc2(void* p, size_t size) { trigger_teardown(); - snmalloc::ThreadAlloc::get().dealloc(p, size); -} - -void dealloc3(void* p, size_t) -{ - trigger_teardown(); - snmalloc::ThreadAlloc::get().dealloc(p); -} - -void dealloc4(void* p, size_t size) -{ - trigger_teardown(); - snmalloc::ThreadAlloc::get().dealloc(p, size); + snmalloc::dealloc(p, size); } void f(size_t size) { auto t1 = std::thread(alloc1, size); auto t2 = std::thread(alloc2, size); - auto t3 = std::thread(alloc3, size); - auto t4 = std::thread(alloc4, size); - auto t5 = std::thread(calloc1, size); - auto t6 = std::thread(calloc2, size); - auto t7 = std::thread(calloc3, size); - auto t8 = std::thread(calloc4, size); + auto t3 = std::thread(calloc1, size); + auto t4 = std::thread(calloc2, size); { auto a = snmalloc::get_scoped_allocator(); auto p1 = a->alloc(size); auto p2 = a->alloc(size); - auto p3 = a->alloc(size); - auto p4 = a->alloc(size); - auto t9 = std::thread(dealloc1, p1, size); - auto t10 = std::thread(dealloc2, p2, size); - auto t11 = std::thread(dealloc3, p3, size); - auto t12 = std::thread(dealloc4, p4, size); + auto t5 = std::thread(dealloc1, p1, size); + auto t6 = std::thread(dealloc2, p2, size); t1.join(); t2.join(); @@ -164,14 +106,8 @@ void f(size_t size) t4.join(); t5.join(); t6.join(); - t7.join(); - t8.join(); - t9.join(); - t10.join(); - t11.join(); - t12.join(); } // Drops a. - // snmalloc::current_alloc_pool()->debug_in_use(0); + snmalloc::debug_in_use(0); printf("."); fflush(stdout); } diff --git a/src/test/func/thread_alloc_external/thread_alloc_external.cc b/src/test/func/thread_alloc_external/thread_alloc_external.cc index ceb464c29..ac5a04c63 100644 --- a/src/test/func/thread_alloc_external/thread_alloc_external.cc +++ b/src/test/func/thread_alloc_external/thread_alloc_external.cc @@ -13,8 +13,8 @@ namespace snmalloc { - using Alloc = snmalloc::LocalAllocator< - snmalloc::StandardConfigClientMeta>; + using Config = snmalloc::StandardConfigClientMeta; + using Alloc = snmalloc::LocalAllocator; } using namespace snmalloc; @@ -65,16 +65,14 @@ int main() setup(); allocator_thread_init(); - auto& a = ThreadAlloc::get(); - for (size_t i = 0; i < 1000; i++) { - auto r1 = a.alloc(i); + auto r1 = snmalloc::alloc(i); - a.dealloc(r1); + snmalloc::dealloc(r1); } - ThreadAlloc::get().teardown(); + snmalloc::debug_teardown(); // This checks that the scoped allocator does not call // register clean up, as this configuration will fault diff --git a/src/test/func/two_alloc_types/alloc1.cc b/src/test/func/two_alloc_types/alloc1.cc index b4e0ae32a..59831ec06 100644 --- a/src/test/func/two_alloc_types/alloc1.cc +++ b/src/test/func/two_alloc_types/alloc1.cc @@ -13,8 +13,7 @@ namespace snmalloc { - using CustomGlobals = FixedRangeConfig>; - using Alloc = LocalAllocator; + using Config = FixedRangeConfig>; } #define SNMALLOC_NAME_MANGLE(a) enclave_##a @@ -22,6 +21,5 @@ namespace snmalloc extern "C" void oe_allocator_init(void* base, void* end) { - snmalloc::CustomGlobals::init( - nullptr, base, address_cast(end) - address_cast(base)); + snmalloc::Config::init(nullptr, base, address_cast(end) - address_cast(base)); } diff --git a/src/test/perf/contention/contention.cc b/src/test/perf/contention/contention.cc index c2cfd8f85..06a433d43 100644 --- a/src/test/perf/contention/contention.cc +++ b/src/test/perf/contention/contention.cc @@ -75,13 +75,12 @@ size_t swapcount; void test_tasks_f(size_t id) { - auto& a = ThreadAlloc::get(); xoroshiro::p128r32 r(id + 5000); for (size_t n = 0; n < swapcount; n++) { size_t size = 16 + (r.next() % 1024); - size_t* res = (size_t*)(use_malloc ? malloc(size) : a.alloc(size)); + size_t* res = (size_t*)(use_malloc ? malloc(size) : snmalloc::alloc(size)); if (res != nullptr) { @@ -102,7 +101,7 @@ void test_tasks_f(size_t id) if (use_malloc) free(out); else - a.dealloc(out, size); + snmalloc::dealloc(out, size); } } }; @@ -111,8 +110,6 @@ void test_tasks(size_t num_tasks, size_t count, size_t size) { std::cout << "Sequential setup" << std::endl; - auto& a = ThreadAlloc::get(); - contention = new std::atomic[size]; xoroshiro::p128r32 r; @@ -120,7 +117,7 @@ void test_tasks(size_t num_tasks, size_t count, size_t size) { size_t alloc_size = 16 + (r.next() % 1024); size_t* res = - (size_t*)(use_malloc ? malloc(alloc_size) : a.alloc(alloc_size)); + (size_t*)(use_malloc ? malloc(alloc_size) : snmalloc::alloc(alloc_size)); *res = alloc_size; contention[n] = res; } @@ -146,7 +143,7 @@ void test_tasks(size_t num_tasks, size_t count, size_t size) if (use_malloc) free(contention[n]); else - a.dealloc(contention[n], *contention[n]); + snmalloc::dealloc(contention[n], *contention[n]); } } @@ -154,7 +151,7 @@ void test_tasks(size_t num_tasks, size_t count, size_t size) } #ifndef NDEBUG - snmalloc::debug_check_empty(); + snmalloc::debug_check_empty(); #endif }; diff --git a/src/test/perf/external_pointer/externalpointer.cc b/src/test/perf/external_pointer/externalpointer.cc index b87b8800a..76be4c708 100644 --- a/src/test/perf/external_pointer/externalpointer.cc +++ b/src/test/perf/external_pointer/externalpointer.cc @@ -13,7 +13,7 @@ namespace test // Pre allocate all the objects size_t* objects[count]; - NOINLINE void setup(xoroshiro::p128r64& r, Alloc& alloc) + NOINLINE void setup(xoroshiro::p128r64& r) { for (size_t i = 0; i < count; i++) { @@ -31,28 +31,27 @@ namespace test if (size < 16) size = 16; // store object - objects[i] = (size_t*)alloc.alloc(size); + objects[i] = (size_t*)snmalloc::alloc(size); if (objects[i] == nullptr) abort(); // Store allocators size for this object - *objects[i] = alloc.alloc_size(objects[i]); + *objects[i] = snmalloc::alloc_size(objects[i]); } } - NOINLINE void teardown(Alloc& alloc) + NOINLINE void teardown() { // Deallocate everything for (size_t i = 0; i < count; i++) { - alloc.dealloc(objects[i]); + snmalloc::dealloc(objects[i]); } - snmalloc::debug_check_empty(); + snmalloc::debug_check_empty(); } void test_external_pointer(xoroshiro::p128r64& r) { - auto& alloc = ThreadAlloc::get(); // This is very slow on Windows at the moment. Until this is fixed, help // CI terminate. #if defined(NDEBUG) && !defined(_MSC_VER) @@ -66,7 +65,7 @@ namespace test static constexpr size_t iterations = 100000; # endif #endif - setup(r, alloc); + setup(r); { MeasureTime m; @@ -76,12 +75,12 @@ namespace test size_t rand = (size_t)r.next(); size_t oid = rand & (((size_t)1 << count_log) - 1); size_t* external_ptr = objects[oid]; - if (!alloc.is_snmalloc_owned(external_ptr)) + if (!snmalloc::is_owned(external_ptr)) continue; size_t size = *external_ptr; size_t offset = (size >> 4) * (rand & 15); void* interior_ptr = pointer_offset(external_ptr, offset); - void* calced_external = alloc.external_pointer(interior_ptr); + void* calced_external = snmalloc::external_pointer(interior_ptr); if (calced_external != external_ptr) { abort(); @@ -89,13 +88,12 @@ namespace test } } - teardown(alloc); + teardown(); } } int main(int, char**) { -#ifndef SNMALLOC_PASS_THROUGH // Depends on snmalloc specific features setup(); xoroshiro::p128r64 r; @@ -105,5 +103,4 @@ int main(int, char**) for (size_t n = 0; n < nn; n++) test::test_external_pointer(r); return 0; -#endif } diff --git a/src/test/perf/low_memory/low-memory.cc b/src/test/perf/low_memory/low-memory.cc index 703dad4c6..c9e973cc0 100644 --- a/src/test/perf/low_memory/low-memory.cc +++ b/src/test/perf/low_memory/low-memory.cc @@ -20,7 +20,7 @@ class Queue Node* new_node(size_t size) { - auto result = (Node*)ThreadAlloc::get().alloc(size); + auto result = (Node*)snmalloc::alloc(size); result->next = nullptr; return result; } @@ -44,7 +44,7 @@ class Queue return false; Node* next = head->next; - ThreadAlloc::get().dealloc(head); + snmalloc::dealloc(head); head = next; return true; } diff --git a/src/test/perf/memcpy/memcpy.cc b/src/test/perf/memcpy/memcpy.cc index d726ea429..c6c59b5a5 100644 --- a/src/test/perf/memcpy/memcpy.cc +++ b/src/test/perf/memcpy/memcpy.cc @@ -35,7 +35,7 @@ void shape(size_t size) // the memcpys. constexpr size_t alignment = 16; offset = (my_random() % // size / alignment) * alignment; Shape s; - s.object = ThreadAlloc::get().alloc(rsize); + s.object = snmalloc::alloc(rsize); s.dst = static_cast(s.object) + offset; // Bring into cache the destination of the copy. memset(s.dst, 0xFF, size); @@ -47,7 +47,7 @@ void unshape() { for (auto& s : allocs) { - ThreadAlloc::get().dealloc(s.object); + snmalloc::dealloc(s.object); } allocs.clear(); } @@ -68,7 +68,7 @@ void test( Memcpy mc, std::vector>& stats) { - auto src = ThreadAlloc::get().alloc(size); + auto src = snmalloc::alloc(size); shape(size); for (size_t i = 0; i < 10; i++) { @@ -77,7 +77,7 @@ void test( auto time = m.get_time(); stats.push_back({size, time}); } - ThreadAlloc::get().dealloc(src); + snmalloc::dealloc(src); unshape(); } @@ -108,7 +108,6 @@ void memcpy_platform_checked(void* dst, const void* src, size_t size) int main(int argc, char** argv) { opt::Opt opt(argc, argv); -#ifndef SNMALLOC_PASS_THROUGH bool full_test = opt.has("--full_test"); // size_t size = 0; @@ -182,8 +181,5 @@ int main(int argc, char** argv) stats_platform.clear(); stats_platform_checked.clear(); } -#else - snmalloc::UNUSED(opt); -#endif return 0; } diff --git a/src/test/perf/msgpass/msgpass.cc b/src/test/perf/msgpass/msgpass.cc index 3f43e13dd..12ef101d6 100644 --- a/src/test/perf/msgpass/msgpass.cc +++ b/src/test/perf/msgpass/msgpass.cc @@ -36,46 +36,6 @@ void chatty(const char* p, ...) } } -/* - * Interpret SNMALLOC_PASS_THROUGH ourselves to make this a bit more fair of a - * comparison, since relying of snmalloc itself to do the passing through - * results in it imposing its own idea of alignment onto the underlying - * allocator, which might result in it taking less optimized paths. - */ -#ifdef SNMALLOC_PASS_THROUGH -struct MyAlloc -{ - MyAlloc() {} - - void* alloc(size_t sz) - { - return malloc(sz); - } - - void dealloc(void* p) - { - free(p); - } -}; -#else -struct MyAlloc -{ - snmalloc::Alloc& a; - - MyAlloc() : a(ThreadAlloc::get()) {} - - void* alloc(size_t sz) - { - return a.alloc(sz); - } - - void dealloc(void* p) - { - a.dealloc(p); - } -}; -#endif - /* * FreeListMPSCQ make for convenient MPSC queues, so we use those for sending * "messages". Each consumer or proxy has its own (source) queue. @@ -106,7 +66,6 @@ freelist::HeadPtr domesticate_nop(freelist::QueuePtr p) void consumer(const struct params* param, size_t qix) { - MyAlloc a{}; auto& myq = param->msgqueue[qix]; chatty("Cl %zu q is %p\n", qix, &myq); @@ -118,13 +77,11 @@ void consumer(const struct params* param, size_t qix) if (myq.can_dequeue(domesticate_nop, domesticate_nop)) { myq.dequeue( - domesticate_nop, - domesticate_nop, - [qix, &a, &reap](freelist::HeadPtr o) { + domesticate_nop, domesticate_nop, [qix, &reap](freelist::HeadPtr o) { UNUSED(qix); auto p = o.as_void().unsafe_ptr(); chatty("Cl %zu free %p\n", qix, p); - a.dealloc(p); + snmalloc::dealloc(p); reap++; return true; }); @@ -145,7 +102,7 @@ void consumer(const struct params* param, size_t qix) producers_live || (queue_gate > param->N_CONSUMER)); chatty("Cl %zu fini\n", qix); - a.dealloc(myq.destroy().unsafe_ptr()); + snmalloc::dealloc(myq.destroy().unsafe_ptr()); } void proxy(const struct params* param, size_t qix) @@ -178,13 +135,12 @@ void proxy(const struct params* param, size_t qix) chatty("Px %zu fini\n", qix); - MyAlloc().dealloc(myq.destroy().unsafe_ptr()); + snmalloc::dealloc(myq.destroy().unsafe_ptr()); queue_gate--; } void producer(const struct params* param, size_t pix) { - MyAlloc a{}; static constexpr size_t msgsizes[] = {48, 64, 96, 128}; static constexpr size_t nmsgsizes = sizeof(msgsizes) / sizeof(msgsizes[0]); @@ -206,7 +162,7 @@ void producer(const struct params* param, size_t pix) /* Allocate batch and form list */ for (size_t msgix = 0; msgix < nmsg; msgix++) { - auto msg = a.alloc(msgsize); + auto msg = snmalloc::alloc(msgsize); chatty("Pd %zu make %p\n", pix, msg); auto msgc = capptr::Alloc::unsafe_from(msg) diff --git a/src/test/perf/singlethread/singlethread.cc b/src/test/perf/singlethread/singlethread.cc index 431d40d24..4755ea0ff 100644 --- a/src/test/perf/singlethread/singlethread.cc +++ b/src/test/perf/singlethread/singlethread.cc @@ -8,8 +8,6 @@ using namespace snmalloc; template void test_alloc_dealloc(size_t count, size_t size, bool write) { - auto& alloc = ThreadAlloc::get(); - { MeasureTime m; m << "Count: " << std::setw(6) << count << ", Size: " << std::setw(6) @@ -20,7 +18,7 @@ void test_alloc_dealloc(size_t count, size_t size, bool write) // alloc 1.5x objects for (size_t i = 0; i < ((count * 3) / 2); i++) { - void* p = alloc.alloc(size); + void* p = snmalloc::alloc(size); SNMALLOC_CHECK(set.find(p) == set.end()); if (write) @@ -36,13 +34,13 @@ void test_alloc_dealloc(size_t count, size_t size, bool write) void* p = *it; set.erase(it); SNMALLOC_CHECK(set.find(p) == set.end()); - alloc.dealloc(p, size); + snmalloc::dealloc(p, size); } // alloc 1x objects for (size_t i = 0; i < count; i++) { - void* p = alloc.alloc(size); + void* p = snmalloc::alloc(size); SNMALLOC_CHECK(set.find(p) == set.end()); if (write) @@ -55,12 +53,12 @@ void test_alloc_dealloc(size_t count, size_t size, bool write) while (!set.empty()) { auto it = set.begin(); - alloc.dealloc(*it, size); + snmalloc::dealloc(*it, size); set.erase(it); } } - snmalloc::debug_check_empty(); + snmalloc::debug_check_empty(); } int main(int, char**) diff --git a/src/test/perf/startup/startup.cc b/src/test/perf/startup/startup.cc index 46e18f90d..bcfd455f1 100644 --- a/src/test/perf/startup/startup.cc +++ b/src/test/perf/startup/startup.cc @@ -77,8 +77,7 @@ int main() ParallelTest test( [](size_t id) { auto start = Aal::tick(); - auto& alloc = snmalloc::ThreadAlloc::get(); - alloc.dealloc(alloc.alloc(1)); + snmalloc::dealloc(snmalloc::alloc(1)); auto end = Aal::tick(); counters[id] = end - start; },