diff --git a/CMakeLists.txt b/CMakeLists.txt index d07a7a7e8..dbed0c908 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.7) project(vsg - VERSION 1.1.5 + VERSION 1.1.6 DESCRIPTION "VulkanSceneGraph library" LANGUAGES CXX ) diff --git a/cmake/cppcheck-suppression-list.txt b/cmake/cppcheck-suppression-list.txt index 154ebc34d..ccf3afc76 100644 --- a/cmake/cppcheck-suppression-list.txt +++ b/cmake/cppcheck-suppression-list.txt @@ -48,6 +48,7 @@ useStlAlgorithm:*/src/vsg/vk/Instance.cpp useStlAlgorithm:*/src/vsg/vk/RenderPass.cpp useStlAlgorithm:*/src/vsg/core/Allocator.cpp useStlAlgorithm:*/src/vsg/core/MemorySlots.cpp +useStlAlgorithm:*/src/vsg/core/IntrusiveAllocator.cpp useStlAlgorithm:*/src/vsg/state/ArrayState.cpp useStlAlgorithm:*/src/vsg/app/CompileTraversal.cpp useStlAlgorithm:*/src/vsg/utils/ShaderSet.cpp @@ -74,6 +75,7 @@ syntaxError:*include/vsg/core/Data.h:51 unusedStructMember:*include/vsg/core/Data.h unusedStructMember:*include/vsg/core/Exception.h unusedStructMember:*include/vsg/core/Version.h +unusedStructMember:*include/vsg/core/IntrusiveAllocator.h unusedStructMember:*include/vsg/io/ObjectCache.h unusedStructMember:*include/vsg/nodes/Bin.h unusedStructMember:*include/vsg/nodes/LOD.h @@ -145,6 +147,8 @@ returnTempReference:*/include/vsg/core/Inherit.h variableScope:*/include/vsg/utils/SharedObjects.h variableScope:*/src/vsg/utils/SharedObjects.cpp variableScope:*/src/vsg/app/CompileManager.cpp +variableScope:*/src/vsg/core/IntrusiveAllocator.cpp // suppress really stupid warning of pointerLessThanZero pointerLessThanZero:*/src/vsg/app/Viewer.cpp + diff --git a/include/vsg/all.h b/include/vsg/all.h index cae892c94..d88ba9be3 100644 --- a/include/vsg/all.h +++ b/include/vsg/all.h @@ -24,6 +24,7 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI #include #include #include +#include #include #include #include diff --git a/include/vsg/core/Allocator.h b/include/vsg/core/Allocator.h index 69e067522..568506aa3 100644 --- a/include/vsg/core/Allocator.h +++ b/include/vsg/core/Allocator.h @@ -12,11 +12,11 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI */ -#include +#include +#include #include #include -#include namespace vsg { @@ -41,94 +41,46 @@ namespace vsg class VSG_DECLSPEC Allocator { public: - explicit Allocator(size_t in_default_alignment = 4); - explicit Allocator(std::unique_ptr in_nestedAllocator, size_t in_default_alignment = 4); - - virtual ~Allocator(); + explicit Allocator(size_t in_defaultAlignment = 4) : + defaultAlignment(in_defaultAlignment) {} + explicit Allocator(std::unique_ptr in_nestedAllocator, size_t in_defaultAlignment = 4) : + defaultAlignment(in_defaultAlignment), nestedAllocator(std::move(in_nestedAllocator)) {} + virtual ~Allocator() {} /// Allocator singleton static std::unique_ptr& instance(); /// allocate from the pool of memory blocks, or allocate from a new memory block - virtual void* allocate(std::size_t size, AllocatorAffinity allocatorAffinity = ALLOCATOR_AFFINITY_OBJECTS); + virtual void* allocate(std::size_t size, AllocatorAffinity allocatorAffinity = ALLOCATOR_AFFINITY_OBJECTS) = 0; /// deallocate, returning data to pool. - virtual bool deallocate(void* ptr, std::size_t size); + virtual bool deallocate(void* ptr, std::size_t size) = 0; /// delete any MemoryBlock that are empty - virtual size_t deleteEmptyMemoryBlocks(); + virtual size_t deleteEmptyMemoryBlocks() = 0; /// return the total available size of allocated MemoryBlocks - virtual size_t totalAvailableSize() const; + virtual size_t totalAvailableSize() const = 0; /// return the total reserved size of allocated MemoryBlocks - virtual size_t totalReservedSize() const; + virtual size_t totalReservedSize() const = 0; /// return the total memory size of allocated MemoryBlocks - virtual size_t totalMemorySize() const; - - /// report stats about blocks of memory allocated. - virtual void report(std::ostream& out) const; + virtual size_t totalMemorySize() const = 0; AllocatorType allocatorType = ALLOCATOR_TYPE_VSG_ALLOCATOR; // use MemoryBlocks by default - int memoryTracking = MEMORY_TRACKING_DEFAULT; - /// set the MemoryTracking member of the vsg::Allocator and all the MemoryBlocks that it manages. - void setMemoryTracking(int mt); - - struct MemoryBlock - { - MemoryBlock(size_t blockSize, int memoryTracking, size_t in_alignment); - virtual ~MemoryBlock(); + virtual void setBlockSize(AllocatorAffinity allocatorAffinity, size_t blockSize) = 0; - void* allocate(std::size_t size); - bool deallocate(void* ptr, std::size_t size); - - vsg::MemorySlots memorySlots; - size_t alignment = 4; - size_t block_alignment = 16; - uint8_t* memory = nullptr; - }; - - struct MemoryBlocks - { - Allocator* parent = nullptr; - std::string name; - size_t blockSize = 0; - size_t alignment = 4; - std::map> memoryBlocks; - std::shared_ptr latestMemoryBlock; - - MemoryBlocks(Allocator* in_parent, const std::string& in_name, size_t in_blockSize, size_t in_alignment); - virtual ~MemoryBlocks(); - - void* allocate(std::size_t size); - bool deallocate(void* ptr, std::size_t size); - - size_t deleteEmptyMemoryBlocks(); - size_t totalAvailableSize() const; - size_t totalReservedSize() const; - size_t totalMemorySize() const; - }; - - MemoryBlocks* getMemoryBlocks(AllocatorAffinity allocatorAffinity); - - MemoryBlocks* getOrCreateMemoryBlocks(AllocatorAffinity allocatorAffinity, const std::string& name, size_t blockSize, size_t in_alignment = 4); - - void setBlockSize(AllocatorAffinity allocatorAffinity, size_t blockSize); + /// report stats about blocks of memory allocated. + virtual void report(std::ostream& out) const = 0; mutable std::mutex mutex; - - size_t default_alignment = 4; - - double allocationTime = 0.0; - double deallocationTime = 0.0; + size_t defaultAlignment = 4; protected: // if you are assigning a custom allocator you must retain the old allocator to manage the memory it allocated and needs to delete std::unique_ptr nestedAllocator; - - std::vector> allocatorMemoryBlocks; }; /// allocate memory using vsg::Allocator::instance() if available, otherwise use std::malloc(size) diff --git a/include/vsg/core/IntrusiveAllocator.h b/include/vsg/core/IntrusiveAllocator.h new file mode 100644 index 000000000..11ffc5dd1 --- /dev/null +++ b/include/vsg/core/IntrusiveAllocator.h @@ -0,0 +1,199 @@ +#pragma once + +/* + +Copyright(c) 2024 Robert Osfield + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + */ + +#include + +#include +#include + +namespace vsg +{ + //////////////////////////////////////////////////////////////////////////////////////////////////// + // + // InstrusiveAllocator is the default Allocator implenentation + // + // Memory is allocated for fixed sized blocks, with indexing of allocated and available slots of memory + // are stored within the same memory block that user memory allocation are made from. The memory block + // is created a contiguous block of 4 bytes Elements, where the Element is a union of bitfield linked list + // market the beginning of the previous slot or the begging of the next, the status of whether the slot is + // allocated or available, or an index when used as part of doubling linked list of free slots. + // + // The block allocation is done based on the type of object so all nodes, data or general objects are + // allocated within the blocks containing objects of similar type. This form of block allocation helps + // scene graph traversal speeds by improving cache coherency/reducing cache missing as it ensures that + // nodes etc. are packed in adjacent memory. + // + // The instrusive indexing means there is only a 4 byte panalty for each memory allocation, and a minimum + // memory use per allocation of 12 bytes (3 Elements - 1 for the slot{previous, next, status} and 2 for the + // previous and next free list indices. + // + // The maximum size of allocations within the block allocation is (2^15-2) * 4, allocations larger than this + // are allocated using aligned versions of std::new and std::delete. + // + class VSG_DECLSPEC IntrusiveAllocator : public Allocator + { + public: + explicit IntrusiveAllocator(size_t in_defaultAlignment = 4); + explicit IntrusiveAllocator(std::unique_ptr in_nestedAllocator, size_t in_defaultAlignment = 4); + + ~IntrusiveAllocator(); + + void report(std::ostream& out) const override; + + void* allocate(std::size_t size, AllocatorAffinity allocatorAffinity = ALLOCATOR_AFFINITY_OBJECTS) override; + + bool deallocate(void* ptr, std::size_t size) override; + + bool validate() const; + + size_t deleteEmptyMemoryBlocks() override; + size_t totalAvailableSize() const override; + size_t totalReservedSize() const override; + size_t totalMemorySize() const override; + void setBlockSize(AllocatorAffinity allocatorAffinity, size_t blockSize) override; + + protected: + struct VSG_DECLSPEC MemoryBlock + { + MemoryBlock(const std::string& in_name, size_t in_blockSize, size_t in_alignment); + virtual ~MemoryBlock(); + + std::string name; + + void* allocate(std::size_t size); + bool deallocate(void* ptr, std::size_t size); + + void report(std::ostream& out) const; + + // bitfield packing of doubly-linked with status field into a 4 byte word + struct Element + { + union + { + uint32_t index; + + struct + { + unsigned int previous : 15; + unsigned int next : 15; + unsigned int status : 2; + }; + }; + + using Offset = decltype(previous); + using Status = decltype(status); + using Index = decltype(index); + + explicit Element(Index in_index) : + index(static_cast(in_index)) {} + + Element(Offset in_previous, Offset in_next, Status in_status) : + previous(static_cast(in_previous)), + next(static_cast(in_next)), + status(in_status) {} + + Element() = default; + Element(const Element&) = default; + }; + + struct FreeList + { + Element::Index count = 0; + Element::Index head = 0; + }; + + Element* memory = nullptr; + Element* memoryEnd = nullptr; + + size_t alignment = 4; // min aligment is 4 { sizeof(Element) } + size_t blockAlignment = 16; + size_t blockSize = 0; + size_t maximumAllocationSize = 0; + Element::Index elementAlignment = 1; + Element::Index firstSlot = 1; + Element::Index capacity = 0; + + std::vector freeLists; + + bool validate() const; + + bool freeSlotsAvaible(size_t size) const; + + inline bool within(const void* ptr) const { return memory <= ptr && ptr < memoryEnd; } + + size_t totalAvailableSize() const; + size_t totalReservedSize() const; + size_t totalMemorySize() const; + + // used for debugging only. + struct VSG_DECLSPEC SlotTester + { + SlotTester(Element* in_mem, size_t in_head) : + mem(in_mem), head(in_head){}; + + const Element* mem = nullptr; + size_t head = 0; + + struct Entry + { + std::string name; + size_t position; + Element slot; + size_t previousFree; + size_t nextFree; + }; + + std::list elements; + + void slot(size_t position, const std::string& name); + + void report(std::ostream& out); + }; + + static inline size_t computeMaxiumAllocationSize(size_t blockSize, size_t alignment) + { + return std::min(blockSize - alignment, size_t((1 << 15) - 2) * sizeof(Element)); + } + }; + + class VSG_DECLSPEC MemoryBlocks + { + public: + MemoryBlocks(IntrusiveAllocator* in_parent, const std::string& in_name, size_t in_blockSize, size_t in_alignment); + virtual ~MemoryBlocks(); + + IntrusiveAllocator* parent = nullptr; + std::string name; + size_t alignment = 4; + size_t blockSize = 0; + size_t maximumAllocationSize = 0; + std::vector> memoryBlocks; + std::shared_ptr memoryBlockWithSpace; + + void* allocate(std::size_t size); + void report(std::ostream& out) const; + bool validate() const; + + size_t deleteEmptyMemoryBlocks(); + size_t totalAvailableSize() const; + size_t totalReservedSize() const; + size_t totalMemorySize() const; + }; + + std::vector> allocatorMemoryBlocks; + std::map> memoryBlocks; + std::map> largeAllocations; + }; + +} // namespace vsg diff --git a/include/vsg/utils/FindDynamicObjects.h b/include/vsg/utils/FindDynamicObjects.h index 2e1427256..2ac97e0ec 100644 --- a/include/vsg/utils/FindDynamicObjects.h +++ b/include/vsg/utils/FindDynamicObjects.h @@ -24,7 +24,6 @@ namespace vsg class VSG_DECLSPEC FindDynamicObjects : public Inherit { public: - std::mutex mutex; std::set dynamicObjects; diff --git a/include/vsg/vk/Device.h b/include/vsg/vk/Device.h index be78f4b98..e2f7772c1 100644 --- a/include/vsg/vk/Device.h +++ b/include/vsg/vk/Device.h @@ -23,6 +23,7 @@ namespace vsg // forward declare class WindowTraits; + class MemoryBufferPools; struct QueueSetting { @@ -82,6 +83,10 @@ namespace vsg /// return true if Device was created with specified extension bool supportsDeviceExtension(const char* extensionName) const; + // provide observer_ptr to memory buffer pools so that these can be accessed when required + observer_ptr deviceMemoryBufferPools; + observer_ptr stagingMemoryBufferPools; + protected: virtual ~Device(); diff --git a/include/vsg/vk/DeviceMemory.h b/include/vsg/vk/DeviceMemory.h index 0cf45afed..ebd86590c 100644 --- a/include/vsg/vk/DeviceMemory.h +++ b/include/vsg/vk/DeviceMemory.h @@ -13,6 +13,7 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI */ #include +#include #include #include @@ -49,6 +50,7 @@ namespace vsg VkDeviceSize maximumAvailableSpace() const; size_t totalAvailableSize() const; size_t totalReservedSize() const; + size_t totalMemorySize() const; Device* getDevice() { return _device; } const Device* getDevice() const { return _device; } @@ -66,6 +68,9 @@ namespace vsg }; VSG_type_name(vsg::DeviceMemory); + using DeviceMemoryList = std::list>; + extern VSG_DECLSPEC DeviceMemoryList getActiveDeviceMemoryList(VkMemoryPropertyFlagBits propertyFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); + template class MappedData : public T { diff --git a/include/vsg/vk/MemoryBufferPools.h b/include/vsg/vk/MemoryBufferPools.h index 329672a4f..7f5911869 100644 --- a/include/vsg/vk/MemoryBufferPools.h +++ b/include/vsg/vk/MemoryBufferPools.h @@ -54,5 +54,6 @@ namespace vsg using BufferPools = std::vector>; BufferPools bufferPools; }; + VSG_type_name(vsg::MemoryBufferPools); } // namespace vsg diff --git a/src/vsg/CMakeLists.txt b/src/vsg/CMakeLists.txt index edb217ee9..287b8b445 100644 --- a/src/vsg/CMakeLists.txt +++ b/src/vsg/CMakeLists.txt @@ -11,6 +11,7 @@ endif() set(SOURCES core/Allocator.cpp + core/IntrusiveAllocator.cpp core/Auxiliary.cpp core/ConstVisitor.cpp core/Data.cpp diff --git a/src/vsg/core/Allocator.cpp b/src/vsg/core/Allocator.cpp index 0e74a3498..6999c5596 100644 --- a/src/vsg/core/Allocator.cpp +++ b/src/vsg/core/Allocator.cpp @@ -10,476 +10,27 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI */ -#include #include +#include +#include #include #include #include #include +#include using namespace vsg; -//////////////////////////////////////////////////////////////////////////////////////////////////// -// -// vsg::Allocator -// -Allocator::Allocator(size_t in_default_alignment) : - default_alignment(in_default_alignment) -{ - allocatorMemoryBlocks.resize(vsg::ALLOCATOR_AFFINITY_LAST); - - size_t Megabyte = 1024 * 1024; - allocatorMemoryBlocks[vsg::ALLOCATOR_AFFINITY_OBJECTS].reset(new MemoryBlocks(this, "MemoryBlocks_OBJECTS", size_t(Megabyte), default_alignment)); - allocatorMemoryBlocks[vsg::ALLOCATOR_AFFINITY_DATA].reset(new MemoryBlocks(this, "MemoryBlocks_DATA", size_t(16 * Megabyte), default_alignment)); - allocatorMemoryBlocks[vsg::ALLOCATOR_AFFINITY_NODES].reset(new MemoryBlocks(this, "MemoryBlocks_NODES", size_t(Megabyte), default_alignment)); - allocatorMemoryBlocks[vsg::ALLOCATOR_AFFINITY_PHYSICS].reset(new MemoryBlocks(this, "MemoryBlocks_PHYSICS", size_t(Megabyte), 16)); -} - -Allocator::Allocator(std::unique_ptr in_nestedAllocator, size_t in_default_alignment) : - Allocator(in_default_alignment) -{ - nestedAllocator = std::move(in_nestedAllocator); -} - -Allocator::~Allocator() -{ -} - +static std::unique_ptr s_allocator(new IntrusiveAllocator()); std::unique_ptr& Allocator::instance() { - static std::unique_ptr s_allocator(new Allocator()); return s_allocator; } -void Allocator::report(std::ostream& out) const -{ - out << "Allocator::report() " << allocatorMemoryBlocks.size() << std::endl; - out << "totalAvailableSize = " << totalAvailableSize() << ", totalReservedSize = " << totalReservedSize() << ", totalMemorySize = " << totalMemorySize() << std::endl; - double totalReserved = static_cast(totalReservedSize()); - - std::scoped_lock lock(mutex); - for (const auto& memoryBlocks : allocatorMemoryBlocks) - { - if (memoryBlocks) - { - size_t totalForBlock = memoryBlocks->totalReservedSize(); - out << memoryBlocks->name << " used = " << totalForBlock; - if (totalReserved > 0.0) - { - out << ", " << (double(totalForBlock) / totalReserved) * 100.0 << "% of total used."; - } - out << std::endl; - } - } - - for (const auto& memoryBlocks : allocatorMemoryBlocks) - { - if (memoryBlocks) - { - out << memoryBlocks->name << " " << memoryBlocks->memoryBlocks.size() << " blocks"; - for (const auto& value : memoryBlocks->memoryBlocks) - { - const auto& memorySlots = value.second->memorySlots; - out << " [used = " << memorySlots.totalReservedSize() << ", avail = " << memorySlots.maximumAvailableSpace() << "]"; - } - out << std::endl; - } - } -} - -void* Allocator::allocate(std::size_t size, AllocatorAffinity allocatorAffinity) -{ - std::scoped_lock lock(mutex); - - // create a MemoryBlocks entry if one doesn't already exist - if (allocatorAffinity > allocatorMemoryBlocks.size()) - { - if (memoryTracking & MEMORY_TRACKING_REPORT_ACTIONS) - { - info("Allocator::allocate(", size, ", ", allocatorAffinity, ") out of bounds allocating new MemoryBlock"); - } - - auto name = make_string("MemoryBlocks_", allocatorAffinity); - size_t blockSize = 1024 * 1024; // Megabyte - - allocatorMemoryBlocks.resize(allocatorAffinity + 1); - allocatorMemoryBlocks[allocatorAffinity].reset(new MemoryBlocks(this, name, blockSize, default_alignment)); - } - - auto& memoryBlocks = allocatorMemoryBlocks[allocatorAffinity]; - if (memoryBlocks) - { - auto mem_ptr = memoryBlocks->allocate(size); - if (mem_ptr) - { - if (memoryTracking & MEMORY_TRACKING_REPORT_ACTIONS) - { - info("Allocated from MemoryBlock mem_ptr = ", mem_ptr, ", size = ", size, ", allocatorAffinity = ", int(allocatorAffinity)); - } - return mem_ptr; - } - } - - void* ptr = Allocator::allocate(size, allocatorAffinity); - if (memoryTracking & MEMORY_TRACKING_REPORT_ACTIONS) - { - info("Allocator::allocate(", size, ", ", int(allocatorAffinity), ") ptr = ", ptr); - } - return ptr; -} - -bool Allocator::deallocate(void* ptr, std::size_t size) -{ - std::scoped_lock lock(mutex); - - for (auto& memoryBlocks : allocatorMemoryBlocks) - { - if (memoryBlocks) - { - if (memoryBlocks->deallocate(ptr, size)) - { - if (memoryTracking & MEMORY_TRACKING_REPORT_ACTIONS) - { - info("Deallocated from MemoryBlock ", ptr); - } - return true; - } - } - } - - if (nestedAllocator && nestedAllocator->deallocate(ptr, size)) return true; - - if (allocatorType == ALLOCATOR_TYPE_NEW_DELETE) - { - operator delete(ptr); - return true; - } - else if (allocatorType == ALLOCATOR_TYPE_MALLOC_FREE) - { - std::free(ptr); - return true; - } - - return false; -} - -size_t Allocator::deleteEmptyMemoryBlocks() -{ - std::scoped_lock lock(mutex); - - size_t memoryDeleted = 0; - for (auto& memoryBlocks : allocatorMemoryBlocks) - { - if (memoryBlocks) memoryDeleted += memoryBlocks->deleteEmptyMemoryBlocks(); - } - return memoryDeleted; -} - -size_t Allocator::totalAvailableSize() const -{ - std::scoped_lock lock(mutex); - - size_t size = 0; - for (auto& memoryBlocks : allocatorMemoryBlocks) - { - if (memoryBlocks) size += memoryBlocks->totalAvailableSize(); - } - return size; -} - -size_t Allocator::totalReservedSize() const -{ - std::scoped_lock lock(mutex); - - size_t size = 0; - for (auto& memoryBlocks : allocatorMemoryBlocks) - { - if (memoryBlocks) size += memoryBlocks->totalReservedSize(); - } - return size; -} - -size_t Allocator::totalMemorySize() const -{ - std::scoped_lock lock(mutex); - - size_t size = 0; - for (auto& memoryBlocks : allocatorMemoryBlocks) - { - if (memoryBlocks) size += memoryBlocks->totalMemorySize(); - } - return size; -} - -Allocator::MemoryBlocks* Allocator::getMemoryBlocks(AllocatorAffinity allocatorAffinity) -{ - std::scoped_lock lock(mutex); - - if (size_t(allocatorAffinity) < allocatorMemoryBlocks.size()) return allocatorMemoryBlocks[allocatorAffinity].get(); - return {}; -} - -Allocator::MemoryBlocks* Allocator::getOrCreateMemoryBlocks(AllocatorAffinity allocatorAffinity, const std::string& name, size_t blockSize, size_t alignment) -{ - std::scoped_lock lock(mutex); - - if (size_t(allocatorAffinity) < allocatorMemoryBlocks.size()) - { - allocatorMemoryBlocks[allocatorAffinity]->name = name; - allocatorMemoryBlocks[allocatorAffinity]->blockSize = blockSize; - allocatorMemoryBlocks[allocatorAffinity]->alignment = alignment; - } - else - { - allocatorMemoryBlocks.resize(allocatorAffinity + 1); - allocatorMemoryBlocks[allocatorAffinity].reset(new MemoryBlocks(this, name, blockSize, alignment)); - } - return allocatorMemoryBlocks[allocatorAffinity].get(); -} - -void Allocator::setBlockSize(AllocatorAffinity allocatorAffinity, size_t blockSize) -{ - std::scoped_lock lock(mutex); - - if (size_t(allocatorAffinity) < allocatorMemoryBlocks.size()) - { - allocatorMemoryBlocks[allocatorAffinity]->blockSize = blockSize; - } - else - { - auto name = make_string("MemoryBlocks_", allocatorAffinity); - - allocatorMemoryBlocks.resize(allocatorAffinity + 1); - allocatorMemoryBlocks[allocatorAffinity].reset(new MemoryBlocks(this, name, blockSize, default_alignment)); - } -} - -void Allocator::setMemoryTracking(int mt) -{ - memoryTracking = mt; - for (auto& amb : allocatorMemoryBlocks) - { - if (amb) - { - for (auto& value : amb->memoryBlocks) - { - value.second->memorySlots.memoryTracking = mt; - } - } - } -} - -//////////////////////////////////////////////////////////////////////////////////////////////////// -// -// vsg::Allocator::MemoryBlock -// -Allocator::MemoryBlock::MemoryBlock(size_t blockSize, int memoryTracking, size_t in_alignment) : - memorySlots(blockSize, memoryTracking), - alignment(in_alignment) -{ - block_alignment = std::max(alignment, alignof(std::max_align_t)); - block_alignment = std::max(block_alignment, size_t{16}); - - memory = static_cast(operator new (blockSize, std::align_val_t{block_alignment})); - - if (memorySlots.memoryTracking & MEMORY_TRACKING_REPORT_ACTIONS) - { - info("MemoryBlock(", blockSize, ") allocated memory"); - } -} - -Allocator::MemoryBlock::~MemoryBlock() -{ - if (memorySlots.memoryTracking & MEMORY_TRACKING_REPORT_ACTIONS) - { - info("MemoryBlock::~MemoryBlock(", memorySlots.totalMemorySize(), ") freed memory"); - } - - operator delete (memory, std::align_val_t{block_alignment}); -} - -void* Allocator::MemoryBlock::allocate(std::size_t size) -{ - auto [allocated, offset] = memorySlots.reserve(size, alignment); - if (allocated) - return memory + offset; - else - return nullptr; -} - -bool Allocator::MemoryBlock::deallocate(void* ptr, std::size_t size) -{ - if (ptr >= memory) - { - size_t offset = static_cast(ptr) - memory; - if (offset < memorySlots.totalMemorySize()) - { - if (!memorySlots.release(offset, size)) - { - warn("Allocator::MemoryBlock::deallocate(", ptr, ") problem - couldn't release"); - } - return true; - } - } - return false; -} - -//////////////////////////////////////////////////////////////////////////////////////////////////// -// -// vsg::Allocator::MemoryBlocks -// -Allocator::MemoryBlocks::MemoryBlocks(Allocator* in_parent, const std::string& in_name, size_t in_blockSize, size_t in_alignment) : - parent(in_parent), - name(in_name), - blockSize(in_blockSize), - alignment(in_alignment) -{ - if (parent->memoryTracking & MEMORY_TRACKING_REPORT_ACTIONS) - { - info("Allocator::MemoryBlocks::MemoryBlocks(", parent, ", ", name, ", ", blockSize, ")"); - } -} - -Allocator::MemoryBlocks::~MemoryBlocks() -{ - if (parent->memoryTracking & MEMORY_TRACKING_REPORT_ACTIONS) - { - info("MemoryBlocks::~MemoryBlocks() name = ", name, ", ", memoryBlocks.size()); - } -} - -void* Allocator::MemoryBlocks::allocate(std::size_t size) -{ - if (latestMemoryBlock) - { - auto ptr = latestMemoryBlock->allocate(size); - if (ptr) return ptr; - } - - // search existing blocks from last to first for space for the required memory allocation. - for (auto itr = memoryBlocks.rbegin(); itr != memoryBlocks.rend(); ++itr) - { - auto& block = itr->second; - if (block != latestMemoryBlock) - { - auto ptr = block->allocate(size); - if (ptr) return ptr; - } - } - - size_t new_blockSize = std::max(size, blockSize); - - auto block = std::make_shared(new_blockSize, parent->memoryTracking, alignment); - latestMemoryBlock = block; - - auto ptr = block->allocate(size); - - memoryBlocks[block->memory] = std::move(block); - - if (parent->memoryTracking & MEMORY_TRACKING_REPORT_ACTIONS) - { - info("Allocator::MemoryBlocks::allocate(", size, ") MemoryBlocks.name = ", name, ", allocated in new MemoryBlock ", parent->memoryTracking); - } - - return ptr; -} - -bool Allocator::MemoryBlocks::deallocate(void* ptr, std::size_t size) -{ - if (memoryBlocks.empty()) return false; - - auto itr = memoryBlocks.upper_bound(ptr); - if (itr != memoryBlocks.end()) - { - if (itr != memoryBlocks.begin()) - { - --itr; - auto& block = itr->second; - if (block->deallocate(ptr, size)) return true; - } - else - { - auto& block = itr->second; - if (block->deallocate(ptr, size)) return true; - } - } - else - { - auto& block = memoryBlocks.rbegin()->second; - if (block->deallocate(ptr, size)) return true; - } - - if (parent->memoryTracking & MEMORY_TRACKING_REPORT_ACTIONS) - { - info("MemoryBlocks:deallocate() MemoryBlocks.name = ", name, ", couldn't locate pointer to deallocate ", ptr); - } - return false; -} - -size_t Allocator::MemoryBlocks::deleteEmptyMemoryBlocks() -{ - size_t memoryDeleted = 0; - if (parent->memoryTracking & MEMORY_TRACKING_REPORT_ACTIONS) - { - info("MemoryBlocks:deleteEmptyMemoryBlocks() MemoryBlocks.name = ", name); - } - - auto itr = memoryBlocks.begin(); - while (itr != memoryBlocks.end()) - { - auto& block = itr->second; - if (block->memorySlots.empty()) - { - if (parent->memoryTracking & MEMORY_TRACKING_REPORT_ACTIONS) - { - info(" MemoryBlocks:deleteEmptyMemoryBlocks() MemoryBlocks.name = ", name, ", removing MemoryBlock", block.get()); - } - if (block == latestMemoryBlock) latestMemoryBlock = nullptr; - memoryDeleted += block->memorySlots.totalMemorySize(); - itr = memoryBlocks.erase(itr); - } - else - { - ++itr; - } - } - return memoryDeleted; -} - -size_t Allocator::MemoryBlocks::totalAvailableSize() const -{ - size_t size = 0; - for (auto& value : memoryBlocks) - { - size += value.second->memorySlots.totalAvailableSize(); - } - return size; -} - -size_t Allocator::MemoryBlocks::totalReservedSize() const -{ - size_t size = 0; - for (auto& value : memoryBlocks) - { - size += value.second->memorySlots.totalReservedSize(); - } - return size; -} - -size_t Allocator::MemoryBlocks::totalMemorySize() const -{ - size_t size = 0; - for (auto& value : memoryBlocks) - { - size += value.second->memorySlots.totalMemorySize(); - } - return size; -} - //////////////////////////////////////////////////////////////////////////////////////////////////// // -// vsg::allocate and vsg::deallocate convenience functions that map to using the Allocator singleton. +// vsg::allocate and vsg::deallocate convenience functions that map to using the OriginalBlockAllocator singleton. // void* vsg::allocate(std::size_t size, AllocatorAffinity allocatorAffinity) { diff --git a/src/vsg/core/IntrusiveAllocator.cpp b/src/vsg/core/IntrusiveAllocator.cpp new file mode 100644 index 000000000..17a80b891 --- /dev/null +++ b/src/vsg/core/IntrusiveAllocator.cpp @@ -0,0 +1,1041 @@ +/* + +Copyright(c) 2024 Robert Osfield + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + */ + +#include +#include +#include +#include +#include + +#include +#include +#include + +using namespace vsg; + +//////////////////////////////////////////////////////////////////////////////////////////////////// +// +// vsg::InstrusiveAllocator +// + +#define DEBUG_ALLOCATOR 0 + +////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// +// MemoryBlock +// +IntrusiveAllocator::MemoryBlock::MemoryBlock(const std::string& in_name, size_t in_blockSize, size_t in_alignment) : + name(in_name), + alignment(in_alignment), + blockSize(in_blockSize) +{ + alignment = std::max(alignment, sizeof(Element)); // we need to be a multiple of sizeof(value_type) + elementAlignment = static_cast(alignment / sizeof(Element)); + + blockAlignment = std::max(alignment, alignof(std::max_align_t)); + blockAlignment = std::max(blockAlignment, size_t{16}); + + // round blockSize up to nearest aligned size + blockSize = ((blockSize + alignment - 1) / alignment) * alignment; + + memory = static_cast(operator new (blockSize, std::align_val_t{blockAlignment})); + memoryEnd = memory + blockSize / sizeof(Element); + capacity = static_cast(blockSize / alignment); + firstSlot = static_cast(((1 + elementAlignment) / elementAlignment) * elementAlignment - 1); + + Element::Index max_slot_size = (1 << 15); + + // // vsg::debug(" capacity = ", capacity, ", max_slot_size = ", max_slot_size); + + // set up the free tracking to encompass the whole buffer + // start at element before the first aligned element so that position 0 can be used to mark beginning or end of free lists + freeLists.emplace_back(); + FreeList& freeList = freeLists.front(); + freeList.count = 0; + freeList.head = firstSlot; + maximumAllocationSize = computeMaxiumAllocationSize(blockSize, alignment); + + // mark the first element as 0. + memory[0].index = 0; + + Element::Index previous_position = 0; // 0 marks the beginning of the free list + Element::Index position = firstSlot; + for (; position < capacity;) + { + Element::Index aligned_start = ((position + max_slot_size) / elementAlignment) * elementAlignment; + Element::Index next_position = std::min(aligned_start - 1, capacity); + + memory[position] = Element{(previous_position == 0) ? 0 : (position - previous_position), next_position - position, 1}; + memory[position + 1].index = static_cast(previous_position); + memory[position + 2].index = static_cast((next_position < capacity) ? next_position : 0); + previous_position = position; + position = next_position; + ++freeList.count; + } + +#if DEBUG_ALLOCATOR + std::cout << "IntrusiveAllocator::MemoryBlock::MemoryBlock(" << in_blockSize << ", " << in_alignment << ")" << std::endl; + + std::cout << "blockSize = " << blockSize << std::endl; + std::cout << "capacity = " << capacity << std::endl; + std::cout << "totalReservedSize = " << totalReservedSize() << std::endl; + std::cout << "totalAvailableSize = " << totalAvailableSize() << std::endl; + std::cout << "alignment = " << alignment << std::endl; + std::cout << "elementAlignment = " << elementAlignment << std::endl; + std::cout << "freeList.head = " << freeList.head << std::endl; + + report(std::cout); +#endif +} + +IntrusiveAllocator::MemoryBlock::~MemoryBlock() +{ + operator delete (memory, std::align_val_t{blockAlignment}); +} + +bool IntrusiveAllocator::MemoryBlock::freeSlotsAvaible(size_t size) const +{ + if (size > maximumAllocationSize) return false; + + for (auto& freeList : freeLists) + { + if (freeList.count > 0) return true; + } + return false; +} + +void* IntrusiveAllocator::MemoryBlock::allocate(std::size_t size) +{ +#if DEBUG_ALLOCATOR + if (!validate()) std::cout << "ERROR detected before IntrusiveAllocator::MemoryBlock::allocate(" << size << ") " << this << std::endl; +#endif + + // check if maximumAllocationSize is big enough + if (size > maximumAllocationSize) return nullptr; + + const size_t minimumNumElementsInSlot = 3; + + for (auto& freeList : freeLists) + { + // check if freeList has available slots + if (freeList.count == 0) continue; + + Element::Index freePosition = freeList.head; + while (freePosition != 0) + { + auto& slot = memory[freePosition]; + if (slot.status != 1) + { + throw "Warning: allocated slot found in freeList"; + } + + Element::Index previousFreePosition = memory[freePosition + 1].index; + Element::Index nextFreePosition = memory[freePosition + 2].index; + + size_t slotSpace = static_cast(slot.next); + if (slot.next == 0) + { + std::cerr << "Warn: IntrusiveAllocator::MemoryBlock::allocate(" << size << ") slot = { " << static_cast(slot.previous) << ", " << static_cast(slot.next) << ", " << static_cast(slot.status) << " }" << std::endl; + } + + Element::Index nextPosition = freePosition + static_cast(slotSpace); + size_t slotSize = sizeof(Element) * (slotSpace - 1); + + if (size <= slotSize) + { + // we can us slot for memory; + + size_t numElementsToBeUsed = std::max((size + sizeof(Element) - 1) / sizeof(Element), minimumNumElementsInSlot); + Element::Index nextAlignedStart = static_cast(((freePosition + 1 + numElementsToBeUsed + elementAlignment) / elementAlignment) * elementAlignment); + Element::Index minimumAlignedEnd = nextAlignedStart + static_cast(minimumNumElementsInSlot); +#if DEBUG_ALLOCATOR + std::cout << "allocating, size = " << size << ", numElementsToBeUsed = " << numElementsToBeUsed << ", freePosition = " << freePosition << ", nextPosition = " << nextPosition << ", nextAlignedStart = " << nextAlignedStart << ", minimumAlignedEnd = " << minimumAlignedEnd << std::endl; +#endif + if (minimumAlignedEnd < nextPosition) + { + + // enough space in slot to split, so adjust + Element::Index newSlotPosition = nextAlignedStart - 1; + slot.next = static_cast(newSlotPosition - freePosition); + +#if DEBUG_ALLOCATOR + std::cout << "splitting slot newSlotPosition = " << newSlotPosition << std::endl; +#endif + // set up the new slot as a free slot + auto& newSlot = memory[newSlotPosition] = Element(slot.next, static_cast(nextPosition - newSlotPosition), 1); + memory[newSlotPosition + 1].index = previousFreePosition; + memory[newSlotPosition + 2].index = nextFreePosition; + + if (previousFreePosition != 0) + { + // need to update the previous slot in the free list + memory[previousFreePosition + 2].index = newSlotPosition; // set previous free slots next index to the newly created slot + } + + if (nextFreePosition != 0) + { + // need to update the previous slot in the free list + memory[nextFreePosition + 1].index = newSlotPosition; // set next free slots previous index to the newly created slot + } + + if (nextPosition < capacity) + { + auto& nextSlot = memory[nextPosition]; + nextSlot.previous = newSlot.next; + } + + if (freePosition == freeList.head) + { + // slot was at head of freeList so move it to the new slot position + freeList.head = newSlotPosition; + } + } + else + { + + // std::cout<<"Removing slot as it's fully used freePosition = "<(slot.status) << std::endl; + else + std::cout << "ERROR detected after IntrusiveAllocator::MemoryBlock::allocate(" << size << ") " << this << " allocated = " << &memory[freePosition + 1] << std::endl; +#endif + + return &memory[freePosition + 1]; + } + + freePosition = nextFreePosition; + } + } + +#if DEBUG_ALLOCATOR + std::cout << "IntrusiveAllocator::MemoryBlock::allocator(" << size << ") " << this << " No space found" << std::endl; +#endif + + return nullptr; +} + +void IntrusiveAllocator::MemoryBlock::SlotTester::slot(size_t position, const std::string& name) +{ + if (mem[position].status == 0) + elements.push_back(Entry{name, position, mem[position], 0, 0}); + else + elements.push_back(Entry{name, position, mem[position], mem[position + 1].index, mem[position + 2].index}); +} + +void IntrusiveAllocator::MemoryBlock::SlotTester::report(std::ostream& out) +{ + out << "head = " << head << std::endl; + for (auto& entry : elements) + { + out << " " << entry.name << ", pos = " << entry.position << " slot { " << entry.slot.previous << ", " << entry.slot.next << ", " << static_cast(entry.slot.status) << " } "; + if (entry.slot.status != 0) + out << " previous free = " << entry.previousFree << ", next free = " << entry.nextFree << std::endl; + else + out << std::endl; + } +} + +bool IntrusiveAllocator::MemoryBlock::deallocate(void* ptr, std::size_t /*size*/) +{ + if (within(ptr)) + { + auto& freeList = freeLists.front(); + size_t maxSize = 1 + maximumAllocationSize / sizeof(Element); + + // + // sequential slots around the slot to be deallocated are named: + // PP (Previous' Previous), P (Previous), C (Current slot being deallocated), N (Next), NN (Next's Next) + // + // the FreeList linked list entries of interest are named: + // PPF (Previous' Previous Free), PNF (Previous's Next Free), NPF (Next's Previous Free), NNF (Next's Next Free) + // + + Element::Index C = static_cast(static_cast(ptr) - memory) - 1; + auto& slot = memory[C]; + +#if DEBUG_ALLOCATOR + if (validate()) + { + std::cout << "IntrusiveAllocator::MemoryBlock::deallocate(" << ptr << ", " << size << ") " << this << " C = " << C << ", slot = {" << slot.previous << ", " << slot.next << ", " << static_cast(slot.status) << std::endl; + } + else + { + std::cout << "ERROR detected befpre IntrusiveAllocator::MemoryBlock::deallocate(" << ptr << ", " << size << ") " << this << std::endl; + } +#endif + + if (slot.next == 0) + { + std::cerr << "Warn: IntrusiveAllocator::MemoryBlock::deallocate(" << ptr << ") C = " << C << ", slot = { " << slot.previous << ", " << slot.next << ", " << slot.status << " }" << std::endl; + throw "slot.ext == 0"; + } + + if (slot.status != 0) + { + std::cerr << "Warn: IntrusiveAllocator::MemoryBlock::deallocate(" << ptr << ") C = " << C << ", Attempt to deallocate already available slot : slot = { " << slot.previous << ", " << slot.next << ", " << slot.status << " }" << std::endl; + throw "Attempt to deallocate already available slot"; + } + + // set up the indices for the previous and next slots + Element::Index P = (slot.previous > 0) ? (C - static_cast(slot.previous)) : 0; + Element::Index N = C + static_cast(slot.next); + if (N >= capacity) N = 0; + + // set up the indices for the previous free entry + Element::Index PPF = 0; + Element::Index PNF = 0; + if (P != 0) + { + if (memory[P].status != 0) + { + PPF = memory[P + 1].index; + PNF = memory[P + 2].index; + } + } + + // set up the indices for the next free entries + Element::Index NN = 0; + Element::Index NPF = 0; + Element::Index NNF = 0; + if (N != 0) + { + NN = N + static_cast(memory[N].next); + if (NN >= capacity) NN = 0; + + if (memory[N].status != 0) + { + NPF = memory[N + 1].index; + NNF = memory[N + 2].index; + } + } + + // 3 way merge of P, C and C + auto mergePCN = [&]() -> void { +#if DEBUG_ALLOCATOR + SlotTester before(memory, freeList.head); + before.slot(P, "P"); + before.slot(C, "C"); + before.slot(N, "N"); + before.slot(PPF, "PPF"); + before.slot(PNF, "PNF"); + before.slot(NPF, "NPF"); + before.slot(NNF, "NNF"); +#endif + // update slots for the merge + memory[P].next += memory[C].next + memory[N].next; + if (NN != 0) memory[NN].previous = memory[P].next; + + // update freeList linked list entries + if (PNF == N) // also implies NPF == P + { + // case 1. in order sequential +#if DEBUG_ALLOCATOR + std::cout << " case 1. in order sequential" << std::endl; +#endif + memory[P + 2].index = NNF; + if (NNF != 0) memory[NNF + 1].index = P; + } + else if (PPF == N) // also implies NNF == P + { + // case 2. reverse sequential +#if DEBUG_ALLOCATOR + std::cout << " case 2. reverse sequential" << std::endl; +#endif + if (freeList.head == N) + { + freeList.head = P; + memory[P + 1].index = 0; + } + else + { + memory[P + 1].index = NPF; + if (NPF != 0) memory[NPF + 2].index = P; + } + } + else // P and N aren't directly connected within the freeList + { + // case 3. disconnected +#if DEBUG_ALLOCATOR + std::cout << " case 3. disconnected" << std::endl; +#endif + if (NPF != 0) memory[NPF + 2].index = NNF; + if (NNF != 0) memory[NNF + 1].index = NPF; + + if (freeList.head == N) + { + freeList.head = NNF; + } + } + + // N slot is nolonger a seperate free slot so decrement free count + --freeList.count; + +#if DEBUG_ALLOCATOR + if (!validate()) + { + SlotTester after(memory, freeList.head); + after.slot(P, "P"); + after.slot(PPF, "PPF"); + after.slot(PNF, "PNF"); + after.slot(NPF, "NPF"); + after.slot(NNF, "NNF"); + + std::cout << "ERROR detected after mergePCN() IntrusiveAllocator::MemoryBlock::deallocate(" << ptr << ", " << size << ") " << this << std::endl; + + std::cout << "Before: "; + before.report(std::cout); + std::cout << "After: "; + after.report(std::cout); + } +#endif + }; + + // 2 way merge of P and C + auto mergePC = [&]() -> void { + // update slots for the merge + memory[P].next += memory[C].next; + if (N != 0) memory[N].previous = memory[P].next; + + // freeList linked list entries will not need updating. + +#if DEBUG_ALLOCATOR + if (!validate()) + { + std::cout << "ERROR detected after mergePC() IntrusiveAllocator::MemoryBlock::deallocate(" << ptr << ", " << size << ") " << this << std::endl; + } +#endif + }; + + // 2 way merge of C and N + auto mergeCN = [&]() -> void { + // update slots for merge + memory[C].status = 1; + memory[C].next += memory[N].next; + if (NN != 0) memory[NN].previous = memory[C].next; + + // update freeList linked list entries + if (NPF != 0) memory[NPF + 2].index = C; + if (NNF != 0) memory[NNF + 1].index = C; + memory[C + 1].index = NPF; + memory[C + 2].index = NNF; + + // if N was the head then change head to C + if (freeList.head == N) freeList.head = C; + +#if DEBUG_ALLOCATOR + if (!validate()) + { + std::cout << "ERROR detected after mergeCN() IntrusiveAllocator::MemoryBlock::deallocate(" << ptr << ", " << size << ") " << this << std::endl; + } +#endif + }; + + // standalone insertion of C into head of freeList + auto standalone = [&]() -> void { + memory[C].status = 1; + memory[C + 1].index = 0; + memory[C + 2].index = freeList.head; + + if (freeList.head != 0) + { + memory[freeList.head + 1].index = C; // set previous heads previousFree to C. + } + + // set the head to C. + freeList.head = C; + + // Inserted new free slot so increment free count + ++freeList.count; + +#if DEBUG_ALLOCATOR + if (!validate()) + { + std::cout << "ERROR detected after standalone() IntrusiveAllocator::MemoryBlock::deallocate(" << ptr << ", " << size << ") " << this << " C = " << C << ", memory[C + 2].index = " << memory[C + 2].index << std::endl; + } +#endif + }; + + if (P != 0 && memory[P].status != 0) + { + if (N != 0 && memory[N].status != 0) + { + if ((static_cast(memory[P].next) + static_cast(memory[C].next) + static_cast(memory[N].next)) <= maxSize) + mergePCN(); + else if ((static_cast(memory[P].next) + static_cast(memory[C].next)) <= maxSize) + mergePC(); // merge P and C + else if ((static_cast(memory[C].next) + static_cast(memory[N].next)) <= maxSize) + mergeCN(); // merge C and N + else + standalone(); // C is standalone + } + else if ((static_cast(memory[P].next) + static_cast(memory[C].next)) <= maxSize) + mergePC(); // merge P and C + else + standalone(); // C is standalone + } + else if (N != 0 && memory[N].status != 0) + { + if (static_cast(memory[C].next) + static_cast(memory[N].next) <= maxSize) + mergeCN(); // merge C and N + else + standalone(); // standalone + } + else + { + // C is standalone + standalone(); + } + + return true; + } + +#if DEBUG_ALLOCATOR + std::cout << "IntrusiveAllocator::MemoryBlock::deallocate((" << ptr << ", " << size << ") OUTWITH block : " << this << std::endl; +#endif + + return false; +} + +void IntrusiveAllocator::MemoryBlock::report(std::ostream& out) const +{ + out << "MemoryBlock " << this << " " << name << std::endl; + out << " alignment = " << alignment << std::endl; + out << " blockAlignment = " << blockAlignment << std::endl; + out << " blockSize = " << blockSize << ", memory = " << static_cast(memory) << std::endl; + out << " maximumAllocationSize = " << maximumAllocationSize << std::endl; + out << " firstSlot = " << firstSlot << std::endl; + out << " totalAvailableSize = " << totalAvailableSize() << std::endl; + out << " totalReservedSize = " << totalReservedSize() << std::endl; + + size_t position = firstSlot; + while (position < capacity) + { + auto& slot = memory[position]; + if (slot.status == 1) + { + out << " memory[" << position << "] slot { " << slot.previous << ", " << slot.next << ", " << slot.status << "}, " << memory[position + 1].index << ", " << memory[position + 2].index << std::endl; + } + else + { + out << " memory[" << position << "] slot { " << slot.previous << ", " << slot.next << ", " << slot.status << "} " << std::endl; + } + + position += slot.next; + if (slot.next == 0) break; + } + + out << " freeList.size() = " << freeLists.size() << " { " << std::endl; + for (auto& freeList : freeLists) + { + out << " FreeList ( count = " << freeList.count << " , head = " << freeList.head << " ) {" << std::endl; + + size_t freePosition = freeList.head; + while (freePosition != 0 && freePosition < capacity) + { + auto& slot = memory[freePosition]; + out << " slot " << freePosition << " { " << slot.previous << ", " << slot.next << ", " << slot.status + << " } previous = " << memory[freePosition + 1].index << ", next = " << memory[freePosition + 2].index << std::endl; + freePosition = memory[freePosition + 2].index; + } + + out << " }" << std::endl; + } +} + +bool IntrusiveAllocator::MemoryBlock::validate() const +{ + size_t previous = 0; + size_t position = firstSlot; + + std::set allocated; + std::set available; + + while (position < capacity) + { + auto& slot = memory[position]; + if (slot.previous > capacity || slot.next > capacity) + { + std::cerr << "IntrusiveAllocator::MemoryBlock::validate() " << this << " slot.corrupted invalid position = " << position << ", slot = {" << slot.previous << ", " << slot.next << ", " << int(slot.status) << "}" << std::endl; + return false; + } + + if (slot.status == 0) + allocated.insert(position); + else + available.insert(position); + + if (slot.previous != 0) + { + if (slot.previous > position) + { + std::cerr << "IntrusiveAllocator::MemoryBlock::validate() " << this << " slot.previous invalid position = " << position << ", slot = {" << slot.previous << ", " << slot.next << ", " << int(slot.status) << "}" << std::endl; + return false; + } + size_t previous_position = position - slot.previous; + if (previous_position != previous) + { + std::cerr << "IntrusiveAllocator::MemoryBlock::validate() " << this << " validation failed : previous slot = " << previous << " doesn't match slot.previous, position = " << position << ", slot = {" << slot.previous << ", " << slot.next << ", " << int(slot.status) << "}" << std::endl; + return false; + } + + size_t previousFree = memory[position + 1].index; + size_t nextFree = memory[position + 2].index; + if (previousFree == position || nextFree == position) + { + std::cerr << "IntrusiveAllocator::MemoryBlock::validate() " << this << " validation failed : slot's previous/nextFree points back to itself, position = " << position << ", slot = {" << slot.previous << ", " << slot.next << ", " << int(slot.status) << "} previousFree = " << previousFree << ", nextFree = " << nextFree << std::endl; + return false; + } + } + + if (slot.next == 0) + { + std::cerr << "IntrusiveAllocator::MemoryBlock::validate() " << this << " validation failed: position = " << position << " slot = {" << slot.previous << ", " << slot.next << ", " << static_cast(slot.status) << "}" << std::endl; + return false; + } + + previous = position; + position += slot.next; + } + + std::set inFreeList; + + // std::cout<<"No invalid entries found"<(slot.status) << "}" << std::endl; + return false; + } + + if (memory[freePosition + 1].index != previousPosition || memory[freePosition + 1].index == freePosition) + { + std::cerr << "IntrusiveAllocator::MemoryBlock::validate() " << this << " validation failed, free list inconsisntent, head = " << freeList.head << ", previousPosition = " << previousPosition << ", freePosition = " << freePosition << ", slot = {" << slot.previous << ", " << slot.next << ", " << static_cast(slot.status) << "} previousFree = " << memory[freePosition + 1].index << ", nextFree = " << memory[freePosition + 2].index << std::endl; + return false; + } + + previousPosition = freePosition; + freePosition = memory[freePosition + 2].index; + } + } + + if (available.size() != inFreeList.size()) + { + std::cerr << "IntrusiveAllocator::MemoryBlock::validate() " << this << " validation failed, Different number of entries in available and in freeList: available.size() = " << available.size() << ", inFreeList.size() = " << inFreeList.size() << std::endl; + return false; + } + + return true; +} + +size_t IntrusiveAllocator::MemoryBlock::totalAvailableSize() const +{ + size_t count = 0; + size_t position = firstSlot; + while (position < capacity) + { + auto& slot = memory[position]; + position += slot.next; + if (slot.status != 0) count += slot.next - 1; + } + + return count * sizeof(Element); +} + +size_t IntrusiveAllocator::MemoryBlock::totalReservedSize() const +{ + size_t count = 0; + size_t position = firstSlot; + while (position < capacity) + { + auto& slot = memory[position]; + position += slot.next; + if (slot.status == 0) count += slot.next - 1; + } + + return count * sizeof(Element); +} + +size_t IntrusiveAllocator::MemoryBlock::totalMemorySize() const +{ + size_t count = 0; + size_t position = firstSlot; + while (position < capacity) + { + auto& slot = memory[position]; + position += slot.next; + count += slot.next - 1; + } + + return count * sizeof(Element); +} + +////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// +// MemoryBlocks +// +IntrusiveAllocator::MemoryBlocks::MemoryBlocks(IntrusiveAllocator* in_parent, const std::string& in_name, size_t in_blockSize, size_t in_alignment) : + parent(in_parent), + name(in_name), + alignment(in_alignment), + blockSize(in_blockSize), + maximumAllocationSize(IntrusiveAllocator::MemoryBlock::computeMaxiumAllocationSize(in_blockSize, in_alignment)) +{ +} + +IntrusiveAllocator::MemoryBlocks::~MemoryBlocks() +{ +} + +void* IntrusiveAllocator::MemoryBlocks::allocate(std::size_t size) +{ + if (memoryBlockWithSpace) + { + auto ptr = memoryBlockWithSpace->allocate(size); + if (ptr) return ptr; + } + + size_t new_blockSize = std::max(size, blockSize); + for (auto& block : memoryBlocks) + { + if (block != memoryBlockWithSpace) + { + auto ptr = block->allocate(size); + if (ptr) return ptr; + } + } + + auto new_block = std::make_shared(name, new_blockSize, alignment); + if (parent) + { + parent->memoryBlocks[new_block->memory] = new_block; + } + + if (memoryBlocks.empty()) + { + maximumAllocationSize = new_block->maximumAllocationSize; + } + + memoryBlockWithSpace = new_block; + memoryBlocks.push_back(new_block); + + auto ptr = new_block->allocate(size); + + return ptr; +} + +void IntrusiveAllocator::MemoryBlocks::report(std::ostream& out) const +{ + out << "IntrusiveAllocator::MemoryBlocks::report() memoryBlocks.size() = " << memoryBlocks.size() << std::endl; + for (auto& memoryBlock : memoryBlocks) + { + memoryBlock->report(out); + } +} + +bool IntrusiveAllocator::MemoryBlocks::validate() const +{ + bool valid = true; + for (auto& memoryBlock : memoryBlocks) + { + valid = memoryBlock->validate() && valid; + } + return valid; +} + +size_t IntrusiveAllocator::MemoryBlocks::deleteEmptyMemoryBlocks() +{ + size_t count = 0; + decltype(memoryBlocks) remainingBlocks; + for (auto& memoryBlock : memoryBlocks) + { + if (memoryBlock->totalReservedSize() == 0) + { + count += memoryBlock->totalAvailableSize(); + } + else + { + remainingBlocks.push_back(memoryBlock); + } + } + memoryBlocks.swap(remainingBlocks); + + return count; +} + +size_t IntrusiveAllocator::MemoryBlocks::totalAvailableSize() const +{ + size_t count = 0; + for (auto& memoryBlock : memoryBlocks) + { + count += memoryBlock->totalAvailableSize(); + } + return count; +} + +size_t IntrusiveAllocator::MemoryBlocks::totalReservedSize() const +{ + size_t count = 0; + for (auto& memoryBlock : memoryBlocks) + { + count += memoryBlock->totalReservedSize(); + } + return count; +} + +size_t IntrusiveAllocator::MemoryBlocks::totalMemorySize() const +{ + size_t count = 0; + for (auto& memoryBlock : memoryBlocks) + { + count += memoryBlock->totalMemorySize(); + } + return count; +} + +////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// +// IntrusiveAllocator +// +IntrusiveAllocator::IntrusiveAllocator(size_t in_defaultAlignment) : + Allocator(in_defaultAlignment) +{ + size_t Megabyte = size_t(1024) * size_t(1024); + size_t blockSize = size_t(1) * Megabyte; + + allocatorMemoryBlocks.resize(vsg::ALLOCATOR_AFFINITY_LAST); + allocatorMemoryBlocks[vsg::ALLOCATOR_AFFINITY_OBJECTS].reset(new MemoryBlocks(this, "ALLOCATOR_AFFINITY_OBJECTS", blockSize, defaultAlignment)); + allocatorMemoryBlocks[vsg::ALLOCATOR_AFFINITY_DATA].reset(new MemoryBlocks(this, "ALLOCATOR_AFFINITY_DATA", size_t(16) * blockSize, defaultAlignment)); + allocatorMemoryBlocks[vsg::ALLOCATOR_AFFINITY_NODES].reset(new MemoryBlocks(this, "ALLOCATOR_AFFINITY_NODES", blockSize, defaultAlignment)); + allocatorMemoryBlocks[vsg::ALLOCATOR_AFFINITY_PHYSICS].reset(new MemoryBlocks(this, "ALLOCATOR_AFFINITY_PHYSICS", blockSize, 16)); +} + +IntrusiveAllocator::IntrusiveAllocator(std::unique_ptr in_nestedAllocator, size_t in_defaultAlignment) : + Allocator(std::move(in_nestedAllocator), in_defaultAlignment) +{ + size_t Megabyte = size_t(1024) * size_t(1024); + size_t blockSize = size_t(1) * Megabyte; + + allocatorMemoryBlocks.resize(vsg::ALLOCATOR_AFFINITY_LAST); + allocatorMemoryBlocks[vsg::ALLOCATOR_AFFINITY_OBJECTS].reset(new MemoryBlocks(this, "ALLOCATOR_AFFINITY_OBJECTS", blockSize, defaultAlignment)); + allocatorMemoryBlocks[vsg::ALLOCATOR_AFFINITY_DATA].reset(new MemoryBlocks(this, "ALLOCATOR_AFFINITY_DATA", size_t(16) * blockSize, defaultAlignment)); + allocatorMemoryBlocks[vsg::ALLOCATOR_AFFINITY_NODES].reset(new MemoryBlocks(this, "ALLOCATOR_AFFINITY_NODES", blockSize, defaultAlignment)); + allocatorMemoryBlocks[vsg::ALLOCATOR_AFFINITY_PHYSICS].reset(new MemoryBlocks(this, "ALLOCATOR_AFFINITY_PHYSICS", blockSize, 16)); +} + +IntrusiveAllocator::~IntrusiveAllocator() +{ +} + +void IntrusiveAllocator::setBlockSize(AllocatorAffinity allocatorAffinity, size_t blockSize) +{ + std::scoped_lock lock(mutex); + + if (size_t(allocatorAffinity) < allocatorMemoryBlocks.size()) + { + allocatorMemoryBlocks[allocatorAffinity]->blockSize = blockSize; + } + else + { + auto name = vsg::make_string("MemoryBlocks_", allocatorAffinity); + + allocatorMemoryBlocks.resize(allocatorAffinity + 1); + allocatorMemoryBlocks[allocatorAffinity].reset(new MemoryBlocks(this, name, blockSize, defaultAlignment)); + } +} + +void IntrusiveAllocator::report(std::ostream& out) const +{ + out << "IntrusiveAllocator::report() " << allocatorMemoryBlocks.size() << std::endl; + + for (const auto& memoryBlock : allocatorMemoryBlocks) + { + if (memoryBlock) memoryBlock->report(out); + } + + validate(); +} + +void* IntrusiveAllocator::allocate(std::size_t size, AllocatorAffinity allocatorAffinity) +{ + std::scoped_lock lock(mutex); + + // create a MemoryBlocks entry if one doesn't already exist + if (allocatorAffinity > allocatorMemoryBlocks.size()) + { + size_t blockSize = 1024 * 1024; // Megabyte + allocatorMemoryBlocks.resize(allocatorAffinity + 1); + allocatorMemoryBlocks[allocatorAffinity].reset(new MemoryBlocks(this, "MemoryBlockAffinity", blockSize, defaultAlignment)); + } + + void* ptr = nullptr; + + auto& blocks = allocatorMemoryBlocks[allocatorAffinity]; + if (blocks) + { + if (size <= blocks->maximumAllocationSize) + { + ptr = blocks->allocate(size); + if (ptr) return ptr; + //std::cout<<"IntrusiveAllocator::allocate() Failed to allocator memory from memoryBlocks "<alignment}); + if (ptr) largeAllocations[ptr] = std::pair(blocks->alignment, size); + //std::cout<<"IntrusiveAllocator::allocate() MemoryBlocks aligned large allocation = "<maximumAllocationSize = "<maximumAllocationSize<(defaultAlignment, size); + //std::cout<<"IntrusiveAllocator::allocate() default aligned large allocation = "<second.first}); + largeAllocations.erase(la_itr); + return true; + } + + if (nestedAllocator && nestedAllocator->deallocate(ptr, size)) + { + return true; + } + + return false; +} + +bool IntrusiveAllocator::validate() const +{ + bool valid = true; + for (auto& memoryBlock : allocatorMemoryBlocks) + { + valid = memoryBlock->validate() && valid; + } + return valid; +} + +size_t IntrusiveAllocator::deleteEmptyMemoryBlocks() +{ + size_t count = 0; + for (auto& blocks : allocatorMemoryBlocks) + { + count += blocks->deleteEmptyMemoryBlocks(); + } + return count; +} + +size_t IntrusiveAllocator::totalAvailableSize() const +{ + size_t count = 0; + for (auto& blocks : allocatorMemoryBlocks) + { + count += blocks->totalAvailableSize(); + } + return count; +} + +size_t IntrusiveAllocator::totalReservedSize() const +{ + size_t count = 0; + for (auto& blocks : allocatorMemoryBlocks) + { + count += blocks->totalReservedSize(); + } + return count; +} + +size_t IntrusiveAllocator::totalMemorySize() const +{ + size_t count = 0; + for (auto& blocks : allocatorMemoryBlocks) + { + count += blocks->totalMemorySize(); + } + return count; +} diff --git a/src/vsg/core/MemorySlots.cpp b/src/vsg/core/MemorySlots.cpp index 05258ea3e..bdd761ce2 100644 --- a/src/vsg/core/MemorySlots.cpp +++ b/src/vsg/core/MemorySlots.cpp @@ -10,8 +10,8 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI */ -#include #include +#include #include #include diff --git a/src/vsg/io/Path.cpp b/src/vsg/io/Path.cpp index 958589a64..27b5cbdbd 100644 --- a/src/vsg/io/Path.cpp +++ b/src/vsg/io/Path.cpp @@ -14,6 +14,7 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI #include #include +#include using namespace vsg; diff --git a/src/vsg/utils/ShaderCompiler.cpp b/src/vsg/utils/ShaderCompiler.cpp index 2ca86f87a..1707057dd 100644 --- a/src/vsg/utils/ShaderCompiler.cpp +++ b/src/vsg/utils/ShaderCompiler.cpp @@ -20,9 +20,9 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI #include #if VSG_SUPPORTS_ShaderCompiler -# include # include # include +# include #endif #include diff --git a/src/vsg/vk/Context.cpp b/src/vsg/vk/Context.cpp index b07a517ba..ba4813280 100644 --- a/src/vsg/vk/Context.cpp +++ b/src/vsg/vk/Context.cpp @@ -89,8 +89,6 @@ Context::Context(Device* in_device, const ResourceRequirements& in_resourceRequi deviceID(in_device->deviceID), device(in_device), resourceRequirements(in_resourceRequirements), - deviceMemoryBufferPools(MemoryBufferPools::create("Device_MemoryBufferPool", device, in_resourceRequirements)), - stagingMemoryBufferPools(MemoryBufferPools::create("Staging_MemoryBufferPool", device, in_resourceRequirements)), scratchBufferSize(0) { //semaphore = vsg::Semaphore::create(device); @@ -98,6 +96,29 @@ Context::Context(Device* in_device, const ResourceRequirements& in_resourceRequi minimum_maxSets = in_resourceRequirements.computeNumDescriptorSets(); minimum_descriptorPoolSizes = in_resourceRequirements.computeDescriptorPoolSizes(); + + deviceMemoryBufferPools = device->deviceMemoryBufferPools.ref_ptr(); + stagingMemoryBufferPools = device->stagingMemoryBufferPools.ref_ptr(); + + if (!deviceMemoryBufferPools) + { + device->deviceMemoryBufferPools = deviceMemoryBufferPools = MemoryBufferPools::create("Device_MemoryBufferPool", device, in_resourceRequirements); + vsg::debug("Context::Context() creating new deviceMemoryBufferPools = ", deviceMemoryBufferPools); + } + else + { + vsg::debug("Context::Context() reusing deviceMemoryBufferPools = ", deviceMemoryBufferPools); + } + + if (!stagingMemoryBufferPools) + { + device->stagingMemoryBufferPools = stagingMemoryBufferPools = MemoryBufferPools::create("Staging_MemoryBufferPool", device, in_resourceRequirements); + vsg::debug("Context::Context() creating new stagingMemoryBufferPools = ", stagingMemoryBufferPools); + } + else + { + vsg::debug("Context::Context() reusing stagingMemoryBufferPools = ", stagingMemoryBufferPools); + } } Context::Context(const Context& context) : diff --git a/src/vsg/vk/Device.cpp b/src/vsg/vk/Device.cpp index 85ae23ab8..96f9fe31b 100644 --- a/src/vsg/vk/Device.cpp +++ b/src/vsg/vk/Device.cpp @@ -15,6 +15,7 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI #include #include #include +#include #include #include diff --git a/src/vsg/vk/DeviceMemory.cpp b/src/vsg/vk/DeviceMemory.cpp index d8438ff95..fc3cda084 100644 --- a/src/vsg/vk/DeviceMemory.cpp +++ b/src/vsg/vk/DeviceMemory.cpp @@ -21,6 +21,24 @@ using namespace vsg; #define DO_CHECK 0 +static std::mutex s_DeviceMemoryListMutex; +static std::list> s_DeviceMemoryList; + +DeviceMemoryList vsg::getActiveDeviceMemoryList(VkMemoryPropertyFlagBits propertyFlags) +{ + std::scoped_lock lock(s_DeviceMemoryListMutex); + DeviceMemoryList dml; + for (auto& dm : s_DeviceMemoryList) + { + auto dm_ref_ptr = dm.ref_ptr(); + if ((dm_ref_ptr->getMemoryPropertyFlags() & propertyFlags) != 0) + { + dml.push_back(dm_ref_ptr); + } + } + return dml; +} + /////////////////////////////////////////////////////////////////////////////// // // DeviceMemory @@ -72,6 +90,12 @@ DeviceMemory::DeviceMemory(Device* device, const VkMemoryRequirements& memRequir { throw Exception{"Error: Failed to allocate DeviceMemory.", result}; } + + { + std::scoped_lock lock(s_DeviceMemoryListMutex); + s_DeviceMemoryList.emplace_back(this); + vsg::debug("DeviceMemory::DeviceMemory() added to s_DeviceMemoryList, s_DeviceMemoryList.size() = ", s_DeviceMemoryList.size()); + } } DeviceMemory::~DeviceMemory() @@ -84,6 +108,20 @@ DeviceMemory::~DeviceMemory() vkFreeMemory(*_device, _deviceMemory, _device->getAllocationCallbacks()); } + + { + std::scoped_lock lock(s_DeviceMemoryListMutex); + auto itr = std::find(s_DeviceMemoryList.begin(), s_DeviceMemoryList.end(), this); + if (itr != s_DeviceMemoryList.end()) + { + s_DeviceMemoryList.erase(itr); + vsg::debug("DeviceMemory::~DeviceMemory() removed from s_DeviceMemoryList, s_DeviceMemoryList.size() = ", s_DeviceMemoryList.size()); + } + else + { + vsg::warn("DeviceMemory::~DeviceMemory() could not find in s_DeviceMemoryList"); + } + } } VkResult DeviceMemory::map(VkDeviceSize offset, VkDeviceSize size, VkMemoryMapFlags flags, void** ppData) @@ -148,3 +186,8 @@ size_t DeviceMemory::totalReservedSize() const std::scoped_lock lock(_mutex); return _memorySlots.totalReservedSize(); } + +size_t DeviceMemory::totalMemorySize() const +{ + return _memorySlots.totalMemorySize(); +}