Merge pull request #1231 from vsg-dev/InstrusiveAllocator_MemoryPools

Replaced original block vsg::Allocator with vsg::InstrusiveAllocator that is ~6 x faster at allocation/deallocation

CMakeLists.txt

@@ -1,7 +1,7 @@
 cmake_minimum_required(VERSION 3.7)
 
 project(vsg
-    VERSION 1.1.5
+    VERSION 1.1.6
     DESCRIPTION "VulkanSceneGraph library"
     LANGUAGES CXX
 )

cmake/cppcheck-suppression-list.txt

@@ -48,6 +48,7 @@ useStlAlgorithm:*/src/vsg/vk/Instance.cpp
 useStlAlgorithm:*/src/vsg/vk/RenderPass.cpp
 useStlAlgorithm:*/src/vsg/core/Allocator.cpp
 useStlAlgorithm:*/src/vsg/core/MemorySlots.cpp
+useStlAlgorithm:*/src/vsg/core/IntrusiveAllocator.cpp
 useStlAlgorithm:*/src/vsg/state/ArrayState.cpp
 useStlAlgorithm:*/src/vsg/app/CompileTraversal.cpp
 useStlAlgorithm:*/src/vsg/utils/ShaderSet.cpp
@@ -74,6 +75,7 @@ syntaxError:*include/vsg/core/Data.h:51
 unusedStructMember:*include/vsg/core/Data.h
 unusedStructMember:*include/vsg/core/Exception.h
 unusedStructMember:*include/vsg/core/Version.h
+unusedStructMember:*include/vsg/core/IntrusiveAllocator.h
 unusedStructMember:*include/vsg/io/ObjectCache.h
 unusedStructMember:*include/vsg/nodes/Bin.h
 unusedStructMember:*include/vsg/nodes/LOD.h
@@ -145,6 +147,8 @@ returnTempReference:*/include/vsg/core/Inherit.h
 variableScope:*/include/vsg/utils/SharedObjects.h
 variableScope:*/src/vsg/utils/SharedObjects.cpp
 variableScope:*/src/vsg/app/CompileManager.cpp
+variableScope:*/src/vsg/core/IntrusiveAllocator.cpp
 
 // suppress really stupid warning of pointerLessThanZero
 pointerLessThanZero:*/src/vsg/app/Viewer.cpp
+

include/vsg/all.h

@@ -24,6 +24,7 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
 #include <vsg/core/Export.h>
 #include <vsg/core/External.h>
 #include <vsg/core/Inherit.h>
+#include <vsg/core/IntrusiveAllocator.h>
 #include <vsg/core/Mask.h>
 #include <vsg/core/MemorySlots.h>
 #include <vsg/core/Object.h>

include/vsg/core/Allocator.h

@@ -12,11 +12,11 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
 
 </editor-fold> */
 
-#include <vsg/core/MemorySlots.h>
+#include <vsg/core/Export.h>
 
+#include <map>
 #include <memory>
 #include <mutex>
-#include <vector>
 
 namespace vsg
 {
@@ -41,94 +41,46 @@ namespace vsg
     class VSG_DECLSPEC Allocator
     {
     public:
-        explicit Allocator(size_t in_default_alignment = 4);
-        explicit Allocator(std::unique_ptr<Allocator> in_nestedAllocator, size_t in_default_alignment = 4);
-
-        virtual ~Allocator();
+        explicit Allocator(size_t in_defaultAlignment = 4) :
+            defaultAlignment(in_defaultAlignment) {}
+        explicit Allocator(std::unique_ptr<Allocator> in_nestedAllocator, size_t in_defaultAlignment = 4) :
+            defaultAlignment(in_defaultAlignment), nestedAllocator(std::move(in_nestedAllocator)) {}
+        virtual ~Allocator() {}
 
         /// Allocator singleton
         static std::unique_ptr<Allocator>& instance();
 
         /// allocate from the pool of memory blocks, or allocate from a new memory block
-        virtual void* allocate(std::size_t size, AllocatorAffinity allocatorAffinity = ALLOCATOR_AFFINITY_OBJECTS);
+        virtual void* allocate(std::size_t size, AllocatorAffinity allocatorAffinity = ALLOCATOR_AFFINITY_OBJECTS) = 0;
 
         /// deallocate, returning data to pool.
-        virtual bool deallocate(void* ptr, std::size_t size);
+        virtual bool deallocate(void* ptr, std::size_t size) = 0;
 
         /// delete any MemoryBlock that are empty
-        virtual size_t deleteEmptyMemoryBlocks();
+        virtual size_t deleteEmptyMemoryBlocks() = 0;
 
         /// return the total available size of allocated MemoryBlocks
-        virtual size_t totalAvailableSize() const;
+        virtual size_t totalAvailableSize() const = 0;
 
         /// return the total reserved size of allocated MemoryBlocks
-        virtual size_t totalReservedSize() const;
+        virtual size_t totalReservedSize() const = 0;
 
         /// return the total memory size of allocated MemoryBlocks
-        virtual size_t totalMemorySize() const;
-
-        /// report stats about blocks of memory allocated.
-        virtual void report(std::ostream& out) const;
+        virtual size_t totalMemorySize() const = 0;
 
         AllocatorType allocatorType = ALLOCATOR_TYPE_VSG_ALLOCATOR; // use MemoryBlocks by default
-        int memoryTracking = MEMORY_TRACKING_DEFAULT;
 
-        /// set the MemoryTracking member of the vsg::Allocator and all the MemoryBlocks that it manages.
-        void setMemoryTracking(int mt);
-
-        struct MemoryBlock
-        {
-            MemoryBlock(size_t blockSize, int memoryTracking, size_t in_alignment);
-            virtual ~MemoryBlock();
+        virtual void setBlockSize(AllocatorAffinity allocatorAffinity, size_t blockSize) = 0;
 
-            void* allocate(std::size_t size);
-            bool deallocate(void* ptr, std::size_t size);
-
-            vsg::MemorySlots memorySlots;
-            size_t alignment = 4;
-            size_t block_alignment = 16;
-            uint8_t* memory = nullptr;
-        };
-
-        struct MemoryBlocks
-        {
-            Allocator* parent = nullptr;
-            std::string name;
-            size_t blockSize = 0;
-            size_t alignment = 4;
-            std::map<void*, std::shared_ptr<MemoryBlock>> memoryBlocks;
-            std::shared_ptr<MemoryBlock> latestMemoryBlock;
-
-            MemoryBlocks(Allocator* in_parent, const std::string& in_name, size_t in_blockSize, size_t in_alignment);
-            virtual ~MemoryBlocks();
-
-            void* allocate(std::size_t size);
-            bool deallocate(void* ptr, std::size_t size);
-
-            size_t deleteEmptyMemoryBlocks();
-            size_t totalAvailableSize() const;
-            size_t totalReservedSize() const;
-            size_t totalMemorySize() const;
-        };
-
-        MemoryBlocks* getMemoryBlocks(AllocatorAffinity allocatorAffinity);
-
-        MemoryBlocks* getOrCreateMemoryBlocks(AllocatorAffinity allocatorAffinity, const std::string& name, size_t blockSize, size_t in_alignment = 4);
-
-        void setBlockSize(AllocatorAffinity allocatorAffinity, size_t blockSize);
+        /// report stats about blocks of memory allocated.
+        virtual void report(std::ostream& out) const = 0;
 
         mutable std::mutex mutex;
-
-        size_t default_alignment = 4;
-
-        double allocationTime = 0.0;
-        double deallocationTime = 0.0;
+        size_t defaultAlignment = 4;
 
     protected:
         // if you are assigning a custom allocator you must retain the old allocator to manage the memory it allocated and needs to delete
         std::unique_ptr<Allocator> nestedAllocator;
-
-        std::vector<std::unique_ptr<MemoryBlocks>> allocatorMemoryBlocks;
     };
 
     /// allocate memory using vsg::Allocator::instance() if available, otherwise use std::malloc(size)

include/vsg/core/IntrusiveAllocator.h

@@ -0,0 +1,199 @@
+#pragma once
+
+/* <editor-fold desc="MIT License">
+
+Copyright(c) 2024 Robert Osfield
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+</editor-fold> */
+
+#include <vsg/core/Allocator.h>
+
+#include <list>
+#include <vector>
+
+namespace vsg
+{
+    ////////////////////////////////////////////////////////////////////////////////////////////////////
+    //
+    // InstrusiveAllocator is the default Allocator implenentation
+    //
+    // Memory is allocated for fixed sized blocks, with indexing of allocated and available slots of memory
+    // are stored within the same memory block that user memory allocation are made from.  The memory block
+    // is created a contiguous block of 4 bytes Elements, where the Element is a union of bitfield linked list
+    // market the beginning of the previous slot or the begging of the next, the status of whether the slot is
+    // allocated or available, or an index when used as part of doubling linked list of free slots.
+    //
+    // The block allocation is done based on the type of object so all nodes, data or general objects are
+    // allocated within the blocks containing objects of similar type.  This form of block allocation helps
+    // scene graph traversal speeds by improving cache coherency/reducing cache missing as it ensures that
+    // nodes etc. are packed in adjacent memory.
+    //
+    // The instrusive indexing means there is only a 4 byte panalty for each memory allocation, and a minimum
+    // memory use per allocation of 12 bytes (3 Elements - 1 for the slot{previous, next, status} and 2 for the
+    // previous and next free list indices.
+    //
+    // The maximum size of allocations within the block allocation is (2^15-2) * 4, allocations larger than this
+    // are allocated using aligned versions of std::new and std::delete.
+    //
+    class VSG_DECLSPEC IntrusiveAllocator : public Allocator
+    {
+    public:
+        explicit IntrusiveAllocator(size_t in_defaultAlignment = 4);
+        explicit IntrusiveAllocator(std::unique_ptr<Allocator> in_nestedAllocator, size_t in_defaultAlignment = 4);
+
+        ~IntrusiveAllocator();
+
+        void report(std::ostream& out) const override;
+
+        void* allocate(std::size_t size, AllocatorAffinity allocatorAffinity = ALLOCATOR_AFFINITY_OBJECTS) override;
+
+        bool deallocate(void* ptr, std::size_t size) override;
+
+        bool validate() const;
+
+        size_t deleteEmptyMemoryBlocks() override;
+        size_t totalAvailableSize() const override;
+        size_t totalReservedSize() const override;
+        size_t totalMemorySize() const override;
+        void setBlockSize(AllocatorAffinity allocatorAffinity, size_t blockSize) override;
+
+    protected:
+        struct VSG_DECLSPEC MemoryBlock
+        {
+            MemoryBlock(const std::string& in_name, size_t in_blockSize, size_t in_alignment);
+            virtual ~MemoryBlock();
+
+            std::string name;
+
+            void* allocate(std::size_t size);
+            bool deallocate(void* ptr, std::size_t size);
+
+            void report(std::ostream& out) const;
+
+            // bitfield packing of doubly-linked with status field into a 4 byte word
+            struct Element
+            {
+                union
+                {
+                    uint32_t index;
+
+                    struct
+                    {
+                        unsigned int previous : 15;
+                        unsigned int next : 15;
+                        unsigned int status : 2;
+                    };
+                };
+
+                using Offset = decltype(previous);
+                using Status = decltype(status);
+                using Index = decltype(index);
+
+                explicit Element(Index in_index) :
+                    index(static_cast<Offset>(in_index)) {}
+
+                Element(Offset in_previous, Offset in_next, Status in_status) :
+                    previous(static_cast<Offset>(in_previous)),
+                    next(static_cast<Offset>(in_next)),
+                    status(in_status) {}
+
+                Element() = default;
+                Element(const Element&) = default;
+            };
+
+            struct FreeList
+            {
+                Element::Index count = 0;
+                Element::Index head = 0;
+            };
+
+            Element* memory = nullptr;
+            Element* memoryEnd = nullptr;
+
+            size_t alignment = 4; // min aligment is 4 { sizeof(Element) }
+            size_t blockAlignment = 16;
+            size_t blockSize = 0;
+            size_t maximumAllocationSize = 0;
+            Element::Index elementAlignment = 1;
+            Element::Index firstSlot = 1;
+            Element::Index capacity = 0;
+
+            std::vector<FreeList> freeLists;
+
+            bool validate() const;
+
+            bool freeSlotsAvaible(size_t size) const;
+
+            inline bool within(const void* ptr) const { return memory <= ptr && ptr < memoryEnd; }
+
+            size_t totalAvailableSize() const;
+            size_t totalReservedSize() const;
+            size_t totalMemorySize() const;
+
+            // used for debugging only.
+            struct VSG_DECLSPEC SlotTester
+            {
+                SlotTester(Element* in_mem, size_t in_head) :
+                    mem(in_mem), head(in_head){};
+
+                const Element* mem = nullptr;
+                size_t head = 0;
+
+                struct Entry
+                {
+                    std::string name;
+                    size_t position;
+                    Element slot;
+                    size_t previousFree;
+                    size_t nextFree;
+                };
+
+                std::list<Entry> elements;
+
+                void slot(size_t position, const std::string& name);
+
+                void report(std::ostream& out);
+            };
+
+            static inline size_t computeMaxiumAllocationSize(size_t blockSize, size_t alignment)
+            {
+                return std::min(blockSize - alignment, size_t((1 << 15) - 2) * sizeof(Element));
+            }
+        };
+
+        class VSG_DECLSPEC MemoryBlocks
+        {
+        public:
+            MemoryBlocks(IntrusiveAllocator* in_parent, const std::string& in_name, size_t in_blockSize, size_t in_alignment);
+            virtual ~MemoryBlocks();
+
+            IntrusiveAllocator* parent = nullptr;
+            std::string name;
+            size_t alignment = 4;
+            size_t blockSize = 0;
+            size_t maximumAllocationSize = 0;
+            std::vector<std::shared_ptr<MemoryBlock>> memoryBlocks;
+            std::shared_ptr<MemoryBlock> memoryBlockWithSpace;
+
+            void* allocate(std::size_t size);
+            void report(std::ostream& out) const;
+            bool validate() const;
+
+            size_t deleteEmptyMemoryBlocks();
+            size_t totalAvailableSize() const;
+            size_t totalReservedSize() const;
+            size_t totalMemorySize() const;
+        };
+
+        std::vector<std::unique_ptr<MemoryBlocks>> allocatorMemoryBlocks;
+        std::map<void*, std::shared_ptr<MemoryBlock>> memoryBlocks;
+        std::map<void*, std::pair<size_t, size_t>> largeAllocations;
+    };
+
+} // namespace vsg

include/vsg/utils/FindDynamicObjects.h

@@ -24,7 +24,6 @@ namespace vsg
     class VSG_DECLSPEC FindDynamicObjects : public Inherit<ConstVisitor, FindDynamicObjects>
     {
     public:
-
         std::mutex mutex;
         std::set<const Object*> dynamicObjects;
 

include/vsg/vk/Device.h

@@ -23,6 +23,7 @@ namespace vsg
 
     // forward declare
     class WindowTraits;
+    class MemoryBufferPools;
 
     struct QueueSetting
     {
@@ -82,6 +83,10 @@ namespace vsg
         /// return true if Device was created with specified extension
         bool supportsDeviceExtension(const char* extensionName) const;
 
+        // provide observer_ptr to memory buffer pools so that these can be accessed when required
+        observer_ptr<MemoryBufferPools> deviceMemoryBufferPools;
+        observer_ptr<MemoryBufferPools> stagingMemoryBufferPools;
+
     protected:
         virtual ~Device();