From 6538ce5b703fc4d393d13988c961704aafda7ea9 Mon Sep 17 00:00:00 2001 From: Eyal Amir Date: Thu, 29 Oct 2026 16:16:41 -0700 Subject: [PATCH 1/2] Allocators: Added PMR and MultiPoolAllocator tests, fixed over-aligned allocation and PMR::Object assignment, added ObjectPoolResource and PMR::makeOwned --- Tests/Allocators/MultiPoolAllocatorTests.cpp | 830 ++++++++++++++++++ Tests/Allocators/PMRTests.cpp | 629 +++++++++++++ Tests/CMakeLists.txt | 15 +- Tests/Helpers/AllocationHelpers.h | 115 +++ .../Allocators/MultiPoolAllocator.h | 128 ++- ea_data_structures/Allocators/PMR.h | 105 ++- 6 files changed, 1794 insertions(+), 28 deletions(-) create mode 100644 Tests/Allocators/MultiPoolAllocatorTests.cpp create mode 100644 Tests/Allocators/PMRTests.cpp create mode 100644 Tests/Helpers/AllocationHelpers.h diff --git a/Tests/Allocators/MultiPoolAllocatorTests.cpp b/Tests/Allocators/MultiPoolAllocatorTests.cpp new file mode 100644 index 0000000..cebbacf --- /dev/null +++ b/Tests/Allocators/MultiPoolAllocatorTests.cpp @@ -0,0 +1,830 @@ +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace nano; +using EA::Allocators::DynamicMemory; +using EA::Allocators::MemoryPool; +using EA::Allocators::MemoryPoolResource; +using EA::Allocators::ObjectPoolResource; +using EA::Allocators::ResourceContainer; +using EA::TestHelpers::countGlobalHeapCalls; +using EA::TestHelpers::expectedHeapCalls; + +namespace +{ +bool isAligned(const void* ptr, std::size_t alignment) +{ + return reinterpret_cast(ptr) % alignment == 0; +} + +bool allBytesEqual(const void* ptr, std::size_t size, unsigned char value) +{ + auto* bytes = static_cast(ptr); + + for (auto index = std::size_t {0}; index < size; ++index) + { + if (bytes[index] != value) + return false; + } + + return true; +} + +template +bool throwsRuntimeError(Func&& func) +{ + try + { + func(); + } + catch (const std::runtime_error&) + { + return true; + } + + return false; +} +} // namespace + +//DynamicMemory + +auto dynamicMemoryWithinCapacity = + test("DynamicMemory.allocate_within_capacity_reuses_storage") = [] +{ + auto memory = DynamicMemory(256); + void* first = nullptr; + void* second = nullptr; + + auto heapCalls = countGlobalHeapCalls( + [&] + { + first = memory.allocate(128); + second = memory.allocate(256); + }); + + check(heapCalls == 0); + check(first != nullptr); + check(first == second); + check(memory == first); +}; + +auto dynamicMemoryGrows = + test("DynamicMemory.allocate_beyond_capacity_moves_to_a_bigger_buffer") = [] +{ + auto memory = DynamicMemory(64); + auto* original = memory.allocate(32); + std::memset(original, 0xAB, 32); + void* grown = nullptr; + + //Growing past the reserved capacity is a std::vector reallocation: one + //allocation for the new buffer and one free for the old one. + auto heapCalls = countGlobalHeapCalls([&] { grown = memory.allocate(128); }); + + check(heapCalls == expectedHeapCalls(2)); + check(grown != original); + check(memory == grown); + check(!(memory == original)); + check(allBytesEqual(grown, 32, 0xAB)); +}; + +auto dynamicMemoryOverAligned = + test("DynamicMemory.over_aligned_allocate_returns_aligned_pointer") = [] +{ + //48 bytes at 64-byte alignment need 48 + 63 bytes of block. + static_assert(EA::Allocators::defaultAlignment + == __STDCPP_DEFAULT_NEW_ALIGNMENT__); + check(DynamicMemory::getRequiredSpace(48, 64) == 111); + check(DynamicMemory::getRequiredSpace(48, alignof(std::max_align_t)) == 48); + check(DynamicMemory::getRequiredSpace(48, 16) == 48); + + auto memory = DynamicMemory(111); + void* ptr = nullptr; + + auto heapCalls = countGlobalHeapCalls([&] { ptr = memory.allocate(48, 64); }); + + check(heapCalls == 0); + check(isAligned(ptr, 64)); + check(memory == ptr); + std::memset(ptr, 0x11, 48); + check(allBytesEqual(ptr, 48, 0x11)); +}; + +//ResourceContainer + +auto containerIsLifo = test("ResourceContainer.push_and_pop_back_are_lifo") = [] +{ + auto container = ResourceContainer(4); + check(container.empty()); + + auto a = std::make_unique(16); + auto b = std::make_unique(16); + auto c = std::make_unique(16); + auto* rawA = a.get(); + auto* rawB = b.get(); + auto* rawC = c.get(); + + container.push_back(std::move(a)); + container.push_back(std::move(b)); + container.push_back(std::move(c)); + check(container.resources.size() == 3); + + check(container.pop_back().get() == rawC); + check(container.pop(0).get() == rawA); + check(container.pop_back().get() == rawB); + check(container.empty()); +}; + +auto containerPopKeepsOrder = + test("ResourceContainer.pop_from_middle_keeps_order") = [] +{ + auto container = ResourceContainer(4); + + for (auto index = 0; index < 3; ++index) + container.create(16); + + auto* first = container.resources[0].get(); + auto* middle = container.resources[1].get(); + auto* last = container.resources[2].get(); + + check(container.pop(1).get() == middle); + check(container.resources.size() == 2); + check(container.resources[0].get() == first); + check(container.resources[1].get() == last); +}; + +auto containerFindAndPop = + test("ResourceContainer.find_and_pop_locates_block_by_address") = [] +{ + auto container = ResourceContainer(4); + + for (auto index = 0; index < 3; ++index) + container.create(32); + + auto* target = container.resources[1].get(); + auto* address = target->allocate(32); + + auto popped = container.findAndPop(address); + check(popped.get() == target); + check(container.resources.size() == 2); + + auto unknown = 0; + check(throwsRuntimeError([&] { container.findAndPop(&unknown); })); + check(container.resources.size() == 2); +}; + +auto containerReservedNoHeap = + test("ResourceContainer.moving_blocks_within_reserve_skips_heap") = [] +{ + auto container = ResourceContainer(8); + + for (auto index = 0; index < 4; ++index) + container.create(64); + + auto heapCalls = countGlobalHeapCalls( + [&] + { + auto last = container.pop_back(); + auto first = container.pop(0); + container.push_back(std::move(last)); + container.push_back(std::move(first)); + }); + + check(heapCalls == 0); + check(container.resources.size() == 4); +}; + +//MemoryPool + +auto poolWarmNoHeap = test("MemoryPool.allocations_from_warm_pool_skip_heap") = [] +{ + auto pool = MemoryPool(128, 8, 16); + auto pointers = std::vector(8, nullptr); + + auto allocateCalls = countGlobalHeapCalls( + [&] + { + for (auto& ptr: pointers) + ptr = pool.allocate(128); + }); + + auto deallocateCalls = countGlobalHeapCalls( + [&] + { + for (auto* ptr: pointers) + pool.deallocate(ptr); + }); + + check(allocateCalls == 0); + check(deallocateCalls == 0); +}; + +auto poolBlocksAreDistinct = + test("MemoryPool.allocations_get_separate_non_overlapping_blocks") = [] +{ + constexpr auto blockSize = std::size_t {64}; + constexpr auto count = 8; + + auto pool = MemoryPool((int) blockSize, count, 16); + auto pointers = std::vector(); + + for (auto index = 0; index < count; ++index) + { + auto* ptr = static_cast(pool.allocate(blockSize)); + std::memset(ptr, index + 1, blockSize); + pointers.push_back(ptr); + } + + auto disjoint = true; + + for (auto i = 0; i < count; ++i) + { + for (auto j = i + 1; j < count; ++j) + { + auto* a = pointers[(size_t) i]; + auto* b = pointers[(size_t) j]; + + if (a < b + blockSize && b < a + blockSize) + disjoint = false; + } + } + + check(disjoint); + + for (auto index = 0; index < count; ++index) + { + check(allBytesEqual( + pointers[(size_t) index], blockSize, (unsigned char) (index + 1))); + } +}; + +auto poolGrowsWhenExhausted = + test("MemoryPool.exhausted_pool_creates_a_block_then_keeps_it") = [] +{ + auto pool = MemoryPool(64, 2, 16); + auto pointers = std::vector(4, nullptr); + + auto warmCalls = countGlobalHeapCalls( + [&] + { + pointers[0] = pool.allocate(64); + pointers[1] = pool.allocate(64); + }); + + //Once the free list is empty, each allocation creates one block: one + //allocation for the DynamicMemory and one for its reserved storage. + auto thirdCalls = countGlobalHeapCalls([&] { pointers[2] = pool.allocate(64); }); + auto fourthCalls = + countGlobalHeapCalls([&] { pointers[3] = pool.allocate(64); }); + + check(warmCalls == 0); + check(thirdCalls == expectedHeapCalls(2)); + check(fourthCalls == expectedHeapCalls(2)); + + //The grown blocks stay in the pool, so the next round is heap-free. + auto secondRoundCalls = countGlobalHeapCalls( + [&] + { + for (auto* ptr: pointers) + pool.deallocate(ptr); + + for (auto& ptr: pointers) + ptr = pool.allocate(64); + }); + + check(secondRoundCalls == 0); +}; + +auto poolReusesLastFreed = + test("MemoryPool.deallocated_block_is_handed_out_next") = [] +{ + auto pool = MemoryPool(64, 4, 16); + auto* first = pool.allocate(64); + auto* other = pool.allocate(64); + void* reused = nullptr; + + auto heapCalls = countGlobalHeapCalls( + [&] + { + pool.deallocate(first); + reused = pool.allocate(64); + }); + + check(heapCalls == 0); + check(reused == first); + check(reused != other); +}; + +auto poolOversizeOnEmpty = + test("MemoryPool.oversized_request_on_empty_pool_gets_a_fitting_block") = [] +{ + auto pool = MemoryPool(64, 0, 16); + void* ptr = nullptr; + + //The new block is sized max(request, objectSize), so the request fits and + //the block isn't resized afterwards: still exactly one block creation. + auto heapCalls = countGlobalHeapCalls([&] { ptr = pool.allocate(1000); }); + + check(heapCalls == expectedHeapCalls(2)); + std::memset(ptr, 0x5A, 1000); + check(allBytesEqual(ptr, 1000, 0x5A)); +}; + +auto poolOversizeGrowsFreeBlock = + test("MemoryPool.oversized_request_grows_a_free_block_and_it_stays_grown") = [] +{ + auto pool = MemoryPool(64, 1, 16); + void* ptr = nullptr; + + //The only free block holds 64 bytes, so it has to reallocate to fit 1000. + auto growCalls = countGlobalHeapCalls([&] { ptr = pool.allocate(1000); }); + + check(growCalls == expectedHeapCalls(2)); + std::memset(ptr, 0x3C, 1000); + check(allBytesEqual(ptr, 1000, 0x3C)); + + auto reuseCalls = countGlobalHeapCalls( + [&] + { + pool.deallocate(ptr); + ptr = pool.allocate(1000); + }); + + check(reuseCalls == 0); +}; + +auto poolBookkeepingGrows = + test("MemoryPool.exceeding_reserved_objects_grows_bookkeeping") = [] +{ + //4 free blocks, but room for only 2 entries in the used list. + auto pool = MemoryPool(32, 4, 2); + + auto withinReserveCalls = countGlobalHeapCalls( + [&] + { + pool.allocate(32); + pool.allocate(32); + }); + + //No new block is needed, but the used list's vector regrows. + auto beyondReserveCalls = countGlobalHeapCalls([&] { pool.allocate(32); }); + + check(withinReserveCalls == 0); + check(beyondReserveCalls == expectedHeapCalls(2)); +}; + +auto poolRejectsUnknownPointer = + test("MemoryPool.deallocating_foreign_pointer_throws") = [] +{ + auto pool = MemoryPool(64, 2, 16); + auto foreign = 0; + + check(throwsRuntimeError([&] { pool.deallocate(&foreign); })); +}; + +auto poolRejectsDoubleFree = test("MemoryPool.double_deallocate_throws") = [] +{ + auto pool = MemoryPool(64, 2, 16); + auto* ptr = pool.allocate(64); + pool.deallocate(ptr); + + check(throwsRuntimeError([&] { pool.deallocate(ptr); })); +}; + +auto poolDefaults = test("MemoryPool.defaults_prefill_sixteen_blocks") = [] +{ + auto pool = MemoryPool(64); + + auto prefilledCalls = countGlobalHeapCalls( + [&] + { + for (auto index = 0; index < 16; ++index) + pool.allocate(64); + }); + + auto seventeenthCalls = countGlobalHeapCalls([&] { pool.allocate(64); }); + + check(prefilledCalls == 0); + check(seventeenthCalls == expectedHeapCalls(2)); +}; + +auto poolOverAlignedRoundTrip = + test("MemoryPool.over_aligned_allocation_deallocates_and_is_reused") = [] +{ + auto pool = MemoryPool(128, 4, 16); + void* first = nullptr; + void* second = nullptr; + + //128-byte blocks fit 48 bytes at 64-byte alignment (48 + 63 = 111). + auto heapCalls = countGlobalHeapCalls( + [&] + { + first = pool.allocate(48, 64); + pool.deallocate(first); + second = pool.allocate(48, 64); + }); + + check(heapCalls == 0); + check(isAligned(first, 64)); + check(first == second); +}; + +auto poolOverAlignedOnEmpty = + test("MemoryPool.over_aligned_request_on_empty_pool_gets_room_to_align") = [] +{ + auto pool = MemoryPool(64, 0, 16); + void* ptr = nullptr; + + //The new block is sized for the alignment slack too, so it isn't regrown. + auto heapCalls = countGlobalHeapCalls([&] { ptr = pool.allocate(64, 64); }); + + check(heapCalls == expectedHeapCalls(2)); + check(isAligned(ptr, 64)); +}; + +//Requests at or below defaultAlignment take no slack, so the pool relies on +//every block start honouring __STDCPP_DEFAULT_NEW_ALIGNMENT__. +auto poolDefaultAlignment = + test("MemoryPool.blocks_are_aligned_to_default_new_alignment") = [] +{ + auto pool = MemoryPool(24, 32, 64); + auto aligned = true; + + for (auto size = std::size_t {1}; size <= 32; ++size) + { + if (!isAligned(pool.allocate(size), EA::Allocators::defaultAlignment)) + aligned = false; + } + + check(aligned); +}; + +auto poolSixteenAlignedNoSlack = + test("MemoryPool.sixteen_aligned_request_takes_no_slack") = [] +{ + //48-byte blocks hold a 48-byte, 16-aligned request as-is; with any + //alignment slack the block would have to regrow on the heap. + auto pool = MemoryPool(48, 1, 4); + void* ptr = nullptr; + + auto heapCalls = countGlobalHeapCalls([&] { ptr = pool.allocate(48, 16); }); + + check(heapCalls == 0); + check(isAligned(ptr, 16)); +}; + +//MemoryPoolResource + +auto resourceEquality = test("MemoryPoolResource.equal_only_to_itself") = [] +{ + auto first = MemoryPoolResource(64, 1, 4); + auto second = MemoryPoolResource(64, 1, 4); + + check(first.is_equal(first)); + check(!first.is_equal(second)); + check(first != second); +}; + +auto resourceRoundTrip = + test("MemoryPoolResource.allocate_deallocate_round_trip_reuses_block") = [] +{ + auto resource = MemoryPoolResource(256, 4, 16); + void* first = nullptr; + void* second = nullptr; + + auto heapCalls = countGlobalHeapCalls( + [&] + { + first = resource.allocate(200); + resource.deallocate(first, 200); + second = resource.allocate(100); + }); + + check(heapCalls == 0); + check(first == second); +}; + +auto resourcePmrVectorNoHeap = + test("MemoryPoolResource.pmr_vector_growth_skips_heap") = [] +{ + auto resource = MemoryPoolResource(4096, 16, 64); + auto size = std::size_t {0}; + auto sum = 0; + auto usedResource = static_cast(nullptr); + + //1000 ints fit in a 4096-byte block, so every regrowth takes a free block + //and returns the previous one; destruction returns the last one too. + auto heapCalls = countGlobalHeapCalls( + [&] + { + auto values = std::pmr::vector(&resource); + + for (auto index = 0; index < 1000; ++index) + values.push_back(index); + + size = values.size(); + usedResource = values.get_allocator().resource(); + + for (auto value: values) + sum += value; + }); + + check(heapCalls == 0); + check(size == 1000); + check(sum == 999 * 1000 / 2); + check(usedResource == &resource); +}; + +auto resourceVectorsSpanBlocks = + test("MemoryPoolResource.concurrent_vectors_use_separate_blocks") = [] +{ + auto resource = MemoryPoolResource(1024, 8, 16); + auto vectors = std::vector>(); + vectors.reserve(4); + + for (auto index = 0; index < 4; ++index) + vectors.emplace_back(&resource); + + auto heapCalls = countGlobalHeapCalls( + [&] + { + for (auto index = 0; index < 4; ++index) + vectors[(size_t) index].assign(256, index); + }); + + check(heapCalls == 0); + + for (auto i = 0; i < 4; ++i) + { + for (auto j = i + 1; j < 4; ++j) + check(vectors[(size_t) i].data() != vectors[(size_t) j].data()); + } + + for (auto index = 0; index < 4; ++index) + { + auto intact = true; + + for (auto value: vectors[(size_t) index]) + { + if (value != index) + intact = false; + } + + check(intact); + } +}; + +auto resourceVectorOutgrowsBlock = + test("MemoryPoolResource.vector_outgrowing_block_size_touches_heap") = [] +{ + auto resource = MemoryPoolResource(256, 4, 16); + auto values = std::pmr::vector(&resource); + + auto fitsCalls = countGlobalHeapCalls([&] { values.reserve(64); }); + + //512 bytes no longer fit a 256-byte block: the block taken from the free + //list is regrown (one allocation, one free). The old 256-byte buffer goes + //back to the pool, which doesn't touch the heap. + auto outgrowCalls = countGlobalHeapCalls([&] { values.reserve(128); }); + + check(fitsCalls == 0); + check(outgrowCalls == expectedHeapCalls(2)); + check(values.capacity() >= 128); +}; + +//48-byte blocks are too small for 48 bytes at 64-byte alignment, so each one +//regrows; what matters here is that every pointer comes back 64-aligned. +auto resourceOverAligned = + test("MemoryPoolResource.honours_over_aligned_requests") = [] +{ + auto resource = MemoryPoolResource(48, 32, 64); + auto pointers = std::vector(); + auto aligned = true; + + for (auto index = 0; index < 32; ++index) + { + auto* ptr = resource.allocate(48, 64); + pointers.push_back(ptr); + + if (!isAligned(ptr, 64)) + aligned = false; + } + + check(aligned); + + for (auto* ptr: pointers) + resource.deallocate(ptr, 48, 64); +}; + +auto resourceDefaults = + test("MemoryPoolResource.defaults_prefill_sixteen_64k_blocks") = [] +{ + auto resource = MemoryPoolResource(); + constexpr auto blockSize = std::size_t {64 * 1024}; + + auto prefilledCalls = countGlobalHeapCalls( + [&] + { + for (auto index = 0; index < 16; ++index) + (void) resource.allocate(blockSize); + }); + + auto seventeenthCalls = + countGlobalHeapCalls([&] { (void) resource.allocate(blockSize); }); + + check(prefilledCalls == 0); + check(seventeenthCalls == expectedHeapCalls(2)); +}; + +//ObjectPoolResource + +namespace +{ +struct Particle +{ + float x = 0.f; + float y = 0.f; + float z = 0.f; + int id = 0; +}; + +struct alignas(64) CacheLine +{ + std::byte data[64] {}; +}; + +//Every pointer aligned to `alignment`, and no two closer than `size` bytes. +bool alignedAndDisjoint(std::vector pointers, + std::size_t size, + std::size_t alignment) +{ + std::sort(pointers.begin(), pointers.end()); + + for (auto index = std::size_t {0}; index < pointers.size(); ++index) + { + if (!isAligned(pointers[index], alignment)) + return false; + + if (index > 0) + { + auto gap = static_cast(pointers[index]) + - static_cast(pointers[index - 1]); + + if (gap < (std::ptrdiff_t) size) + return false; + } + } + + return true; +} +} // namespace + +auto objectPoolDefaultCount = + test("ObjectPoolResource.default_count_serves_1024_objects_without_heap") = [] +{ + using Pool = ObjectPoolResource; + static_assert(Pool::blockSize == (int) sizeof(Particle)); + + auto pool = Pool(); + auto pointers = std::vector(1024, nullptr); + + auto allocateCalls = countGlobalHeapCalls( + [&] + { + for (auto& ptr: pointers) + ptr = pool.allocate(sizeof(Particle), alignof(Particle)); + }); + + check(allocateCalls == 0); + check(alignedAndDisjoint(pointers, sizeof(Particle), alignof(Particle))); + + auto deallocateCalls = countGlobalHeapCalls( + [&] + { + for (auto* ptr: pointers) + pool.deallocate(ptr, sizeof(Particle), alignof(Particle)); + }); + + check(deallocateCalls == 0); +}; + +auto objectPoolCustomCount = + test("ObjectPoolResource.custom_count_then_next_allocation_grows") = [] +{ + auto pool = ObjectPoolResource(8); + + auto withinCountCalls = countGlobalHeapCalls( + [&] + { + for (auto index = 0; index < 8; ++index) + (void) pool.allocate(sizeof(Particle), alignof(Particle)); + }); + + //The 9th needs a fresh block (the DynamicMemory and its storage) and the + //used list, reserved for exactly 8, regrows (new buffer, old one freed). + auto ninthCalls = countGlobalHeapCalls( + [&] { (void) pool.allocate(sizeof(Particle), alignof(Particle)); }); + + check(withinCountCalls == 0); + check(ninthCalls == expectedHeapCalls(4)); +}; + +auto objectPoolOverAligned = + test("ObjectPoolResource.over_aligned_type_comes_back_aligned") = [] +{ + using Pool = ObjectPoolResource; + static_assert(Pool::blockSize == 64 + 64 - 1); + + //16-byte alignment is what operator new already gives, so no slack. + struct alignas(16) Vec4 + { + float values[4]; + }; + static_assert(ObjectPoolResource::blockSize == 16); + + auto pool = Pool(32); + auto pointers = std::vector(32, nullptr); + + auto heapCalls = countGlobalHeapCalls( + [&] + { + for (auto& ptr: pointers) + ptr = pool.allocate(sizeof(CacheLine), alignof(CacheLine)); + }); + + check(heapCalls == 0); + check(alignedAndDisjoint(pointers, sizeof(CacheLine), 64)); +}; + +//allocate_shared puts the control block and the Particle in one allocation, +//which is bigger than sizeof(Particle). Each block regrows on the heap the +//first time it serves one; after that the grown block is reused heap-free. +auto objectPoolMakeShared = + test("ObjectPoolResource.make_shared_regrows_blocks_once") = [] +{ + auto pool = ObjectPoolResource(8); + auto objects = std::vector>(8); + + auto firstRoundCalls = countGlobalHeapCalls( + [&] + { + for (auto& object: objects) + object = EA::PMR::makeShared(pool); + + for (auto& object: objects) + object.reset(); + }); + + auto secondRoundCalls = countGlobalHeapCalls( + [&] + { + for (auto& object: objects) + object = EA::PMR::makeShared(pool); + + for (auto& object: objects) + object.reset(); + }); + + check(firstRoundCalls == expectedHeapCalls(16)); + check(secondRoundCalls == 0); +}; + +//Same story for list nodes, which carry two links next to the Particle. +auto objectPoolList = test("ObjectPoolResource.pmr_list_regrows_blocks_once") = [] +{ + auto pool = ObjectPoolResource(8); + auto particles = std::pmr::list(&pool); + + auto fill = [&] + { + for (auto index = 0; index < 8; ++index) + particles.push_back(Particle {0.f, 0.f, 0.f, index}); + }; + + auto firstRoundCalls = countGlobalHeapCalls( + [&] + { + fill(); + particles.clear(); + }); + + auto secondRoundCalls = countGlobalHeapCalls( + [&] + { + fill(); + particles.clear(); + }); + + check(firstRoundCalls == expectedHeapCalls(16)); + check(secondRoundCalls == 0); +}; diff --git a/Tests/Allocators/PMRTests.cpp b/Tests/Allocators/PMRTests.cpp new file mode 100644 index 0000000..fec320f --- /dev/null +++ b/Tests/Allocators/PMRTests.cpp @@ -0,0 +1,629 @@ +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace nano; +using EA::TestHelpers::countGlobalHeapCalls; +using EA::TestHelpers::CountingResource; +using EA::TestHelpers::StackResource; + +namespace +{ +//A PMR-aware type whose constructor takes the resource first, which is the +//shape PMR::Object forwards its extra arguments to. +struct FilledBuffer +{ + FilledBuffer(std::pmr::memory_resource* resource, std::size_t size, int fill) + : values(size, fill, resource) + { + } + + std::pmr::vector values; +}; + +//Long enough to defeat the small-string optimisation on every standard library, +//so each string really allocates. +constexpr auto longText = "a string that is far too long for any SSO buffer"; +} // namespace + +//PMR::Object + +auto objectStoresResource = + test("PMR.Object.constructed_from_reference_or_pointer") = [] +{ + auto resource = CountingResource(); + + auto fromReference = EA::PMR::Object>(resource); + auto fromPointer = EA::PMR::Object>(&resource); + + check(fromReference.getResource() == &resource); + check(fromPointer.getResource() == &resource); + check(fromReference->get_allocator().resource() == &resource); + check(fromPointer->get_allocator().resource() == &resource); +}; + +auto objectAccessors = + test("PMR.Object.get_arrow_and_star_reach_the_same_object") = [] +{ + auto resource = CountingResource(); + auto object = EA::PMR::Object>(resource); + + object->push_back(1); + (*object).push_back(2); + object.get().push_back(3); + + const auto& constObject = object; + + check(&constObject.get() == &object.get()); + check(&*constObject == &object.get()); + check(constObject->size() == 3); + check(constObject.get()[0] == 1); + check(constObject.get()[2] == 3); +}; + +auto objectForwardsArgs = test("PMR.Object.forwards_args_after_the_resource") = [] +{ + auto stack = StackResource<1024>(); + + auto object = EA::PMR::Object(stack.counter, 8, 7); + + check(object->values.size() == 8); + check(object->values[0] == 7); + check(object->values[7] == 7); + check(object->values.get_allocator().resource() == &stack.counter); + check(stack.counter.allocations == 1); +}; + +auto objectNoGlobalHeap = + test("PMR.Object.allocates_through_its_resource_not_global_new") = [] +{ + auto stack = StackResource<1024>(); + + auto heapCalls = countGlobalHeapCalls( + [&] + { + auto object = EA::PMR::Object(stack.counter, 32, 1); + object->values.push_back(2); + }); + + check(heapCalls == 0); + check(stack.counter.allocations >= 1); + check(stack.counter.live() == 0); +}; + +auto objectCopyKeepsResource = test("PMR.Object.copy_uses_the_source_resource") = [] +{ + auto stack = StackResource<4096>(); + auto source = EA::PMR::Vector(stack.counter); + + for (auto index = 0; index < 16; ++index) + source->push_back(index); + + auto allocationsBefore = stack.counter.allocations; + auto copySize = std::size_t {0}; + auto sameContents = false; + auto copyResource = static_cast(nullptr); + auto copyAllocatorResource = static_cast(nullptr); + + auto heapCalls = countGlobalHeapCalls( + [&] + { + auto copy = source; + copySize = copy->size(); + sameContents = *copy == *source; + copyResource = copy.getResource(); + copyAllocatorResource = copy->get_allocator().resource(); + }); + + check(heapCalls == 0); + check(copySize == 16); + check(sameContents); + check(copyResource == &stack.counter); + check(copyAllocatorResource == &stack.counter); + check(stack.counter.allocations == allocationsBefore + 1); +}; + +auto objectMoveKeepsResource = + test("PMR.Object.move_construction_keeps_resource") = [] +{ + auto stack = StackResource<1024>(); + auto source = EA::PMR::Vector(stack.counter); + source->assign({1, 2, 3}); + + const auto* buffer = source->data(); + auto allocationsBefore = stack.counter.allocations; + auto stoleBuffer = false; + auto sameContents = false; + auto movedResource = static_cast(nullptr); + auto movedAllocatorResource = static_cast(nullptr); + + auto heapCalls = countGlobalHeapCalls( + [&] + { + auto moved = std::move(source); + stoleBuffer = moved->data() == buffer; + sameContents = *moved == std::pmr::vector({1, 2, 3}, &stack.arena); + movedResource = moved.getResource(); + movedAllocatorResource = moved->get_allocator().resource(); + }); + + check(heapCalls == 0); + check(stack.counter.allocations == allocationsBefore); + check(stoleBuffer); + check(sameContents); + check(movedResource == &stack.counter); + check(movedAllocatorResource == &stack.counter); +}; + +auto objectAssignSameResource = + test("PMR.Object.copy_assignment_with_shared_resource") = [] +{ + auto stack = StackResource<4096>(); + auto source = EA::PMR::Vector(stack.counter); + auto target = EA::PMR::Vector(stack.counter); + source->assign({4, 5, 6}); + target->assign({9}); + + auto heapCalls = countGlobalHeapCalls([&] { target = source; }); + + check(heapCalls == 0); + check(*target == *source); + check(target.getResource() == &stack.counter); + check(target->get_allocator().resource() == &stack.counter); +}; + +//std::pmr containers never propagate their allocator on assignment, so neither +//does Object: across different resources, the target keeps its own resource +//and the contents are copied (or moved element-wise) into it. +auto objectAssignAcrossResources = + test("PMR.Object.copy_assignment_keeps_resource_in_sync") = [] +{ + auto first = StackResource<1024>(); + auto second = StackResource<1024>(); + auto copyTarget = EA::PMR::Vector(first.counter); + auto moveTarget = EA::PMR::Vector(first.counter); + auto source = EA::PMR::Vector(second.counter); + auto moveSource = EA::PMR::Vector(second.counter); + source->assign({1, 2, 3}); + moveSource->assign({4, 5}); + auto secondAllocationsBefore = second.counter.allocations; + + auto heapCalls = countGlobalHeapCalls( + [&] + { + copyTarget = source; + moveTarget = std::move(moveSource); + }); + + check(heapCalls == 0); + + check(*copyTarget == *source); + check(copyTarget.getResource() == &first.counter); + check(copyTarget->get_allocator().resource() == &first.counter); + + check(moveTarget->size() == 2); + check(moveTarget.get()[0] == 4); + check(moveTarget.get()[1] == 5); + check(moveTarget.getResource() == &first.counter); + check(moveTarget->get_allocator().resource() == &first.counter); + + //Both targets allocated from their own resource, never from the source's. + check(first.counter.allocations == 2); + check(second.counter.allocations == secondAllocationsBefore); +}; + +//PMR::Vector + +auto vectorAlias = test("PMR.Vector.is_an_object_wrapping_pmr_vector") = [] +{ + using IntVector = EA::PMR::Vector; + + static_assert(std::is_same_v>>); + static_assert(std::is_same_v>); +}; + +auto vectorNoGlobalHeap = + test("PMR.Vector.growth_allocates_through_resource_not_global_new") = [] +{ + auto stack = StackResource<8192>(); + auto sum = 0; + + auto heapCalls = countGlobalHeapCalls( + [&] + { + auto values = EA::PMR::Vector(stack.counter); + + for (auto index = 0; index < 100; ++index) + values->push_back(index); + + for (auto value: *values) + sum += value; + }); + + check(heapCalls == 0); + check(sum == 99 * 100 / 2); + check(stack.counter.allocations > 1); + check(stack.counter.live() == 0); +}; + +auto vectorPropagatesToElements = + test("PMR.Vector.pmr_elements_share_the_vector_resource") = [] +{ + auto stack = StackResource<4096>(); + auto elementResource = static_cast(nullptr); + auto textIntact = false; + + auto heapCalls = countGlobalHeapCalls( + [&] + { + auto strings = EA::PMR::Vector(stack.counter); + strings->reserve(2); + strings->emplace_back(longText); + strings->emplace_back(longText); + elementResource = strings->back().get_allocator().resource(); + textIntact = strings->front() == longText; + }); + + check(heapCalls == 0); + check(textIntact); + check(elementResource == &stack.counter); + //One buffer for the vector, one per string. + check(stack.counter.allocations == 3); + check(stack.counter.live() == 0); +}; + +auto vectorWithMemoryPool = + test("PMR.Vector.on_a_memory_pool_resource_skips_heap") = [] +{ + auto pool = EA::Allocators::MemoryPoolResource(4096, 16, 64); + auto size = std::size_t {0}; + + auto heapCalls = countGlobalHeapCalls( + [&] + { + auto values = EA::PMR::Vector(pool); + + for (auto index = 0; index < 256; ++index) + values->push_back(index * 0.5); + + auto copy = values; + size = copy->size(); + }); + + check(heapCalls == 0); + check(size == 256); +}; + +//PMR::makeShared + +auto makeSharedThroughResource = + test("PMR.makeShared.allocates_and_frees_through_resource") = [] +{ + auto stack = StackResource<1024>(); + auto first = 0; + auto second = 0.0; + auto allocationsWhileAlive = 0; + + auto heapCalls = countGlobalHeapCalls( + [&] + { + auto shared = + EA::PMR::makeShared>(stack.counter, 3, 2.5); + first = shared->first; + second = shared->second; + allocationsWhileAlive = stack.counter.allocations; + }); + + check(heapCalls == 0); + check(first == 3); + check(second == 2.5); + //Object and control block share a single allocation. + check(allocationsWhileAlive == 1); + check(stack.counter.deallocations == 1); +}; + +auto makeSharedOnPool = + test("PMR.makeShared.released_block_is_reused_by_the_pool") = [] +{ + auto pool = EA::Allocators::MemoryPoolResource(256, 4, 16); + const void* firstAddress = nullptr; + const void* secondAddress = nullptr; + + auto heapCalls = countGlobalHeapCalls( + [&] + { + auto shared = EA::PMR::makeShared(pool, 42); + firstAddress = shared.get(); + shared.reset(); + + auto again = EA::PMR::makeShared(pool, 43); + secondAddress = again.get(); + }); + + check(heapCalls == 0); + check(firstAddress == secondAddress); +}; + +//PMR::makeOwned + +namespace +{ +//Counts its own destructions through a pointer the test owns. +struct Tracked +{ + Tracked(int valueToUse, int& destructionsToUse) + : value(valueToUse) + , destructions(&destructionsToUse) + { + } + + ~Tracked() { ++*destructions; } + + int value; + int* destructions; +}; + +struct ThrowsOnConstruction +{ + ThrowsOnConstruction() { throw std::runtime_error("constructor failed"); } +}; + +struct Voice +{ + double phase = 0.0; + float gain = 1.f; + int note = 0; +}; + +struct Base +{ + Base(int& destructionsToUse) + : destructions(&destructionsToUse) + { + } + + virtual ~Base() { ++*destructions; } + + int* destructions; +}; + +//A polymorphic Mixin listed first becomes the primary base, so the Base +//subobject isn't at the start of the allocation. +struct Mixin +{ + virtual ~Mixin() = default; + + long long padding[3] {}; +}; + +struct alignas(32) Derived + : Mixin + , Base +{ + Derived(int& destructionsToUse, int valueToUse) + : Base(destructionsToUse) + , value(valueToUse) + { + } + + ~Derived() override { ++*destructions; } + + int value; +}; + +struct PlainBase +{ + int value = 0; +}; + +struct PlainDerived : PlainBase +{ + int extra = 0; +}; +} // namespace + +auto makeOwnedThroughResource = + test("PMR.makeOwned.allocates_constructs_and_frees_through_resource") = [] +{ + auto stack = StackResource<1024>(); + auto destructions = 0; + auto value = 0; + auto allocationsWhileAlive = 0; + auto bytesWhileAlive = std::size_t {0}; + + auto heapCalls = countGlobalHeapCalls( + [&] + { + auto owned = + EA::PMR::makeOwned(stack.counter, 42, destructions); + value = owned->value; + allocationsWhileAlive = stack.counter.allocations; + bytesWhileAlive = stack.counter.bytesAllocated; + }); + + check(heapCalls == 0); + check(value == 42); + check(allocationsWhileAlive == 1); + check(bytesWhileAlive == sizeof(Tracked)); + check(destructions == 1); + check(stack.counter.live() == 0); + check(stack.counter.bytesDeallocated == sizeof(Tracked)); + check(stack.counter.lastDeallocation.pointer + == stack.counter.lastAllocation.pointer); + check(stack.counter.lastDeallocation.alignment == alignof(Tracked)); +}; + +auto makeOwnedUsesAllocator = + test("PMR.makeOwned.pmr_aware_object_gets_the_same_resource") = [] +{ + auto stack = StackResource<1024>(); + auto innerResource = static_cast(nullptr); + + auto heapCalls = countGlobalHeapCalls( + [&] + { + auto owned = EA::PMR::makeOwned>(stack.counter); + owned->assign({1, 2, 3}); + innerResource = owned->get_allocator().resource(); + }); + + check(heapCalls == 0); + check(innerResource == &stack.counter); + check(stack.counter.allocations == 2); + check(stack.counter.live() == 0); +}; + +auto makeOwnedThrowingConstructor = + test("PMR.makeOwned.throwing_constructor_returns_the_memory") = [] +{ + //Throwing allocates the exception object on the heap, so only the + //resource's books are checked here. + auto resource = CountingResource(); + auto threw = false; + + try + { + auto owned = EA::PMR::makeOwned(resource); + } + catch (const std::runtime_error&) + { + threw = true; + } + + check(threw); + check(resource.allocations == 1); + check(resource.deallocations == 1); + check(resource.bytesDeallocated == sizeof(ThrowsOnConstruction)); +}; + +auto makeOwnedObjectPool = + test("PMR.makeOwned.object_pool_fits_exactly_and_reuses_blocks") = [] +{ + auto pool = EA::Allocators::ObjectPoolResource(8); + auto voices = std::vector>(8); + auto firstAddresses = std::vector(8, nullptr); + auto secondAddresses = std::vector(8, nullptr); + + //Each Voice is exactly one block, so not even the first allocation regrows. + auto firstRoundCalls = countGlobalHeapCalls( + [&] + { + for (auto index = 0; index < 8; ++index) + { + voices[(size_t) index] = EA::PMR::makeOwned(pool); + firstAddresses[(size_t) index] = voices[(size_t) index].get(); + } + }); + + void* released = voices[3].get(); + void* reused = nullptr; + + auto reuseCalls = countGlobalHeapCalls( + [&] + { + voices[3].reset(); + voices[3] = EA::PMR::makeOwned(pool, 0.5, 0.25f, 60); + reused = voices[3].get(); + }); + + auto secondRoundCalls = countGlobalHeapCalls( + [&] + { + for (auto& voice: voices) + voice.reset(); + + for (auto index = 0; index < 8; ++index) + { + voices[(size_t) index] = EA::PMR::makeOwned(pool); + secondAddresses[(size_t) index] = voices[(size_t) index].get(); + } + }); + + check(firstRoundCalls == 0); + check(reuseCalls == 0); + check(secondRoundCalls == 0); + check(reused == released); + + std::sort(firstAddresses.begin(), firstAddresses.end()); + std::sort(secondAddresses.begin(), secondAddresses.end()); + check(firstAddresses == secondAddresses); +}; + +auto makeOwnedBaseConversion = + test("PMR.makeOwned.base_conversion_frees_with_derived_size_and_alignment") = [] +{ + static_assert( + std::is_convertible_v, EA::PMR::Owned>); + static_assert(std::is_convertible_v, + EA::PMR::Owned>); + //No virtual destructor: converting would destroy only the base, so it + //doesn't compile. + static_assert(!std::is_convertible_v, + EA::PMR::Owned>); + + auto stack = StackResource<1024>(); + auto destructions = 0; + auto baseIsOffset = false; + + auto heapCalls = countGlobalHeapCalls( + [&] + { + EA::PMR::Owned owned = + EA::PMR::makeOwned(stack.counter, destructions, 7); + baseIsOffset = static_cast(owned.get()) + != stack.counter.lastAllocation.pointer; + }); + + check(heapCalls == 0); + check(baseIsOffset); + //~Derived and ~Base both ran. + check(destructions == 2); + check(stack.counter.live() == 0); + check(stack.counter.lastDeallocation.pointer + == stack.counter.lastAllocation.pointer); + check(stack.counter.lastDeallocation.bytes == sizeof(Derived)); + check(stack.counter.lastDeallocation.alignment == alignof(Derived)); +}; + +auto makeOwnedMoveOnly = test("PMR.makeOwned.is_move_only_and_moves_freely") = [] +{ + static_assert(!std::is_copy_constructible_v>); + static_assert(!std::is_copy_assignable_v>); + static_assert(std::is_nothrow_move_constructible_v>); + + auto stack = StackResource<1024>(); + auto source = EA::PMR::makeOwned(stack.counter); + auto* original = source.get(); + auto movedFromIsNull = false; + auto pointerKept = false; + auto resourceKept = false; + + auto heapCalls = countGlobalHeapCalls( + [&] + { + auto moved = std::move(source); + movedFromIsNull = source == nullptr; + pointerKept = moved.get() == original; + resourceKept = moved.get_deleter().resource == &stack.counter; + + //Hand it back so destruction happens outside the measured scope. + source = std::move(moved); + }); + + check(heapCalls == 0); + check(movedFromIsNull); + check(pointerKept); + check(resourceKept); + check(stack.counter.allocations == 1); + check(stack.counter.deallocations == 0); +}; diff --git a/Tests/CMakeLists.txt b/Tests/CMakeLists.txt index 5ef5fab..c37d707 100644 --- a/Tests/CMakeLists.txt +++ b/Tests/CMakeLists.txt @@ -11,8 +11,20 @@ FetchContent_Declare( ) FetchContent_MakeAvailable(NanoTest) +#Test-only: interposes malloc/free and global new/delete so tests can assert +#that a code path doesn't touch the global heap. This file is only reached when +#this repo is the top-level project, so consumers never fetch or link it. +FetchContent_Declare( + ScopedMemoryAllocations + GIT_REPOSITORY https://github.com/eyalamirmusic/ScopedMemoryAllocations.git + GIT_TAG main +) +FetchContent_MakeAvailable(ScopedMemoryAllocations) + nano_add_executable(ea_data_structures_tests SOURCES + Allocators/MultiPoolAllocatorTests.cpp + Allocators/PMRTests.cpp Flags/BoolTests.cpp Flags/CopyableAtomicTests.cpp Flags/LocksTests.cpp @@ -54,7 +66,8 @@ nano_add_executable(ea_data_structures_tests ValueWrapper/VariantTests.cpp ) -target_link_libraries(ea_data_structures_tests PRIVATE ea_data_structures) +target_link_libraries(ea_data_structures_tests + PRIVATE ea_data_structures AllocationsChecker) target_include_directories(ea_data_structures_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) diff --git a/Tests/Helpers/AllocationHelpers.h b/Tests/Helpers/AllocationHelpers.h new file mode 100644 index 0000000..13e53aa --- /dev/null +++ b/Tests/Helpers/AllocationHelpers.h @@ -0,0 +1,115 @@ +#pragma once + +#include + +#include +#include +#include + +namespace EA::TestHelpers +{ +//True when ScopedMemoryAllocations actually interposes malloc/free and global +//new/delete. On Windows (or with SCOPED_ALLOCATIONS_TEST=OFF) the library is a +//no-op, so "the heap was touched" can't be observed - only "it wasn't". +#ifdef EA_SCOPED_ALLOCATIONS_TEST +constexpr bool heapTrackingEnabled = true; +#else +constexpr bool heapTrackingEnabled = false; +#endif + +//Runs func with global heap use banned on this thread and returns how many +//times the global heap was touched while it ran. Every malloc/calloc/realloc/ +//free/posix_memalign and global new/delete counts as one call, so a vector +//regrowing its buffer counts twice: the new allocation and the old free. +template +int countGlobalHeapCalls(Func&& func) +{ + auto calls = 0; + Allocations::setViolationHandler([&calls] { ++calls; }); + + { + auto noAllocs = Allocations::ScopedSetter(); + std::forward(func)(); + } + + Allocations::setViolationHandler({}); + return calls; +} + +//Expected result of countGlobalHeapCalls() for a path that does touch the +//heap: the real count where tracking works, zero where it is compiled out. +constexpr int expectedHeapCalls(int calls) +{ + return heapTrackingEnabled ? calls : 0; +} + +//A memory_resource that forwards to an upstream resource and counts the +//calls, so tests can tell which resource an allocation actually went through. +class CountingResource final : public std::pmr::memory_resource +{ +public: + explicit CountingResource( + std::pmr::memory_resource* upstreamToUse = std::pmr::new_delete_resource()) + : upstream(upstreamToUse) + { + } + + //Pointer, size and alignment of a single allocate/deallocate call. + struct Call + { + void* pointer = nullptr; + std::size_t bytes = 0; + std::size_t alignment = 0; + }; + + int allocations = 0; + int deallocations = 0; + std::size_t bytesAllocated = 0; + std::size_t bytesDeallocated = 0; + Call lastAllocation; + Call lastDeallocation; + + int live() const { return allocations - deallocations; } + +private: + void* do_allocate(std::size_t bytes, std::size_t alignment) override + { + ++allocations; + bytesAllocated += bytes; + auto* ptr = upstream->allocate(bytes, alignment); + lastAllocation = {ptr, bytes, alignment}; + return ptr; + } + + void do_deallocate(void* ptr, std::size_t bytes, std::size_t alignment) override + { + ++deallocations; + bytesDeallocated += bytes; + lastDeallocation = {ptr, bytes, alignment}; + upstream->deallocate(ptr, bytes, alignment); + } + + bool do_is_equal(const memory_resource& other) const noexcept override + { + return &other == this; + } + + std::pmr::memory_resource* upstream; +}; + +//A CountingResource over a fixed stack buffer with no upstream, so anything +//allocated through it provably never reaches the global heap (exhausting the +//buffer throws std::bad_alloc instead of falling back to new). +template +struct StackResource +{ + StackResource() = default; + StackResource(const StackResource&) = delete; + StackResource& operator=(const StackResource&) = delete; + + alignas(std::max_align_t) std::byte buffer[Size] {}; + std::pmr::monotonic_buffer_resource arena { + buffer, Size, std::pmr::null_memory_resource()}; + CountingResource counter {&arena}; +}; +} // namespace EA::TestHelpers diff --git a/ea_data_structures/Allocators/MultiPoolAllocator.h b/ea_data_structures/Allocators/MultiPoolAllocator.h index 1fb2d5a..84a23d8 100644 --- a/ea_data_structures/Allocators/MultiPoolAllocator.h +++ b/ea_data_structures/Allocators/MultiPoolAllocator.h @@ -10,23 +10,44 @@ namespace EA::Allocators { +//Alignment that the heap storage behind a DynamicMemory gets from operator new. +//Anything stricter is handled by over-allocating and aligning inside the block. +constexpr size_t defaultAlignment = __STDCPP_DEFAULT_NEW_ALIGNMENT__; + class DynamicMemory { public: - DynamicMemory(size_t size) { data.reserve(size); } + DynamicMemory(size_t size) + { + data.reserve(size); + handedOut = data.data(); + } - void* allocate(size_t size) + //Bytes a block needs so that `size` bytes aligned to `alignment` fit in it. + static constexpr size_t getRequiredSpace(size_t size, size_t alignment) { - data.resize(size); - return data.data(); + return alignment > defaultAlignment ? size + alignment - 1 : size; } - bool operator==(const void* other) const { return getAddr() == other; } + void* allocate(size_t size, size_t alignment = defaultAlignment) + { + auto space = getRequiredSpace(size, alignment); + data.resize(space); + handedOut = data.data(); + + if (alignment > defaultAlignment) + handedOut = std::align(alignment, size, handedOut, space); -private: - const void* getAddr() const { return data.data(); } + return handedOut; + } + + //Matches the pointer last returned by allocate(), which for an + //over-aligned request sits a few bytes into the block. + bool operator==(const void* other) const { return handedOut == other; } +private: std::vector data; + void* handedOut = nullptr; }; using SingleResource = std::unique_ptr; @@ -58,7 +79,6 @@ struct ResourceContainer } throw std::runtime_error("Trying to deallocate a non-existing memory"); - return nullptr; } bool empty() const { return resources.empty(); } @@ -71,12 +91,21 @@ struct ResourceContainer //creates one if the free list is empty) and deallocate() returns it to the //free list without actually freeing it. Intended for contexts where you want //to avoid malloc/free calls in a hot path after an initial warm-up. +// +//Heap-backed, not an arena: every block is its own heap allocation, made up +//front or on demand. The heap is still hit when the free list runs dry or a +//request is bigger than the block it gets (that block regrows). +// +//Pointer stability: a pointer stays valid and never moves until it is +//deallocated (or the pool is destroyed). Only blocks taken from the free list +//are ever resized; a block in use is never touched, and growing the pool only +//moves the unique_ptrs that own blocks between the free and used lists. class MemoryPool { public: MemoryPool(int poolObjectSizeToUse, - int freeListInitialSize = 1000, - int numReservedObjects = 10000) + int freeListInitialSize = 16, + int numReservedObjects = 1024) : poolObjectSize((size_t) poolObjectSizeToUse) , free((size_t) numReservedObjects) , used((size_t) numReservedObjects) @@ -85,12 +114,18 @@ class MemoryPool free.create(poolObjectSize); } - void* allocate(size_t size) + //Alignments above defaultAlignment cost alignment - 1 extra bytes of the + //block, so a block only fits `size` bytes at that alignment if it holds + //DynamicMemory::getRequiredSpace(size, alignment); a smaller one regrows. + void* allocate(size_t size, size_t alignment = defaultAlignment) { if (free.empty()) - free.create(std::max(size, poolObjectSize)); + { + auto space = DynamicMemory::getRequiredSpace(size, alignment); + free.create(std::max(space, poolObjectSize)); + } - return allocateFromFreePool(size); + return allocateFromFreePool(size, alignment); } void deallocate(const void* data) @@ -100,10 +135,10 @@ class MemoryPool } private: - void* allocateFromFreePool(size_t size) + void* allocateFromFreePool(size_t size, size_t alignment) { auto last = free.pop_back(); - auto addr = last->allocate(size); + auto addr = last->allocate(size, alignment); used.push_back(std::move(last)); return addr; @@ -116,24 +151,30 @@ class MemoryPool ResourceContainer used; }; -//A std::pmr::memory_resource adapter that routes allocations through a -//MemoryPool. Lets you plug the pool into PMR-aware containers. -class MemoryPoolResource final : public std::pmr::memory_resource +namespace Detail { -public: - MemoryPoolResource(int objectSize = 1024 * 1024, - int freeListInitial = 1000, - int reservedObjects = 10000) +//The std::pmr::memory_resource plumbing shared by MemoryPoolResource and +//ObjectPoolResource: every request, including its alignment, goes to a +//MemoryPool. +class PoolResourceBase : public std::pmr::memory_resource +{ +protected: + PoolResourceBase(int objectSize, int freeListInitial, int reservedObjects) : pool(objectSize, freeListInitial, reservedObjects) { } private: - void* do_allocate(size_t bytes, size_t) override { return pool.allocate(bytes); } + void* do_allocate(size_t bytes, size_t alignment) override + { + return pool.allocate(bytes, alignment); + } + void do_deallocate(void* data, size_t, size_t) override { pool.deallocate(data); } + bool do_is_equal(const memory_resource& other) const noexcept override { return &other == this; @@ -141,4 +182,45 @@ class MemoryPoolResource final : public std::pmr::memory_resource MemoryPool pool; }; +} // namespace Detail + +//A std::pmr::memory_resource adapter that routes allocations through a +//MemoryPool. Lets you plug the pool into PMR-aware containers. The defaults +//pre-fill 16 blocks of 64 KB (about 1 MB up front). +// +//Backed by ordinary heap memory, not a stack or static arena: each block is a +//separate heap allocation, and an empty free list or a request bigger than a +//block still reaches the heap. Pointers are stable: what allocate() returns +//never moves until it is deallocated, however the pool grows (see MemoryPool). +class MemoryPoolResource final : public Detail::PoolResourceBase +{ +public: + MemoryPoolResource(int objectSize = 64 * 1024, + int freeListInitial = 16, + int reservedObjects = 1024) + : PoolResourceBase(objectSize, freeListInitial, reservedObjects) + { + } +}; + +//A pool resource whose blocks each hold exactly one T, honouring alignof(T), +//pre-filled with numObjects blocks and with bookkeeping reserved for as many, +//so up to numObjects live allocate(sizeof(T), alignof(T)) calls never touch +//the heap. Anything larger than a T - a shared_ptr control block, a list +//node - regrows the block it lands in the first time it uses it. +//Like MemoryPoolResource, each block is its own heap allocation, the heap is +//hit again past numObjects live allocations, and pointers never move until +//they are deallocated. +template +class ObjectPoolResource final : public Detail::PoolResourceBase +{ +public: + static constexpr int blockSize = + (int) DynamicMemory::getRequiredSpace(sizeof(T), alignof(T)); + + explicit ObjectPoolResource(int numObjects = 1024) + : PoolResourceBase(blockSize, numObjects, numObjects) + { + } +}; } // namespace EA::Allocators diff --git a/ea_data_structures/Allocators/PMR.h b/ea_data_structures/Allocators/PMR.h index 4e4f2e0..fa117d3 100644 --- a/ea_data_structures/Allocators/PMR.h +++ b/ea_data_structures/Allocators/PMR.h @@ -1,14 +1,22 @@ #pragma once +#include +#include #include +#include +#include #include namespace EA::PMR { using Resource = std::pmr::memory_resource; -//Pairs a PMR-aware object T with the memory_resource it uses, so the resource -//stays alive for as long as the object does and is carried along by copies. +//Pairs a PMR-aware object T with the memory_resource it was constructed with. +//The resource pointer is non-owning: the caller must keep the resource alive +//for longer than the Object and every copy of it. +//Copies and moves are constructed on the source's resource. Assignment follows +//std::pmr semantics: the target keeps its own resource and only the contents +//are assigned, so getResource() always names the resource T allocates from. //Typical T is a std::pmr container; see the `Vector` alias below. template struct Object @@ -35,8 +43,25 @@ struct Object object = other.object; } - Object& operator=(const Object& other) = default; - Object& operator=(Object&& other) noexcept = default; + Object(Object&& other) noexcept(std::is_nothrow_move_constructible_v) + : resource(other.resource) + , object(std::move(other.object)) + { + } + + Object& operator=(const Object& other) + { + if (this != &other) + object = other.object; + + return *this; + } + + Object& operator=(Object&& other) noexcept(std::is_nothrow_move_assignable_v) + { + object = std::move(other.object); + return *this; + } T& get() { return object; } const T& get() const { return object; } @@ -64,4 +89,76 @@ std::shared_ptr makeShared(Resource& resource, Args&&... args) return std::allocate_shared(allocator, std::forward(args)...); } +//An Owned may become an Owned when only cv-qualifiers change, or when +//To is a base with a virtual destructor, so deleting through it is complete. +template +concept OwnedConvertibleTo = + std::is_convertible_v + && (std::is_same_v, std::remove_cv_t> + || std::has_virtual_destructor_v); + +//Deleter for Owned: destroys the object and hands its memory back to the +//resource it came from, with the size and alignment captured at creation. +//Converting to Owned keeps the original size and alignment, and is only +//allowed when Base has a virtual destructor, so the whole object is destroyed. +template +struct Deleter +{ + Deleter() = default; + + Deleter(Resource* resourceToUse, size_t sizeToUse, size_t alignmentToUse) + : resource(resourceToUse) + , size(sizeToUse) + , alignment(alignmentToUse) + { + } + + template U> + Deleter(const Deleter& other) noexcept + : resource(other.resource) + , size(other.size) + , alignment(other.alignment) + { + } + + void operator()(T* object) const + { + //The allocation starts at the most-derived object, which a Base* + //into a multiply-inherited Derived doesn't necessarily point to. + auto* mutableObject = const_cast*>(object); + void* memory = nullptr; + + if constexpr (std::is_polymorphic_v) + memory = dynamic_cast(mutableObject); + else + memory = static_cast(mutableObject); + + std::destroy_at(object); + resource->deallocate(memory, size, alignment); + } + + Resource* resource = nullptr; + size_t size = 0; + size_t alignment = 0; +}; + +//Uniquely owned T living in a memory_resource; see makeOwned(). +template +using Owned = std::unique_ptr>; + +//The unique-ownership counterpart to makeShared(): allocates and constructs a +//T in `resource`, and the result destroys it and frees the memory back to the +//same resource. A PMR-aware T (e.g. a std::pmr container) is given the +//resource too, through uses-allocator construction. If T's constructor +//throws, the memory is returned to the resource before the exception leaves. +//The resource is not owned: it must outlive the result. +template +Owned makeOwned(Resource& resource, Args&&... args) +{ + auto allocator = std::pmr::polymorphic_allocator<>(&resource); + auto* object = allocator.new_object(std::forward(args)...); + + return Owned(object, Deleter(&resource, sizeof(T), alignof(T))); +} + } // namespace EA::PMR From 5dfd7148665c59b9574b46a352853ad6ecef1d5f Mon Sep 17 00:00:00 2001 From: Eyal Amir Date: Thu, 29 Oct 2026 16:21:22 -0700 Subject: [PATCH 2/2] Allocator tests: Fixed GCC, libstdc++ and MSVC CI failures --- Tests/Allocators/MultiPoolAllocatorTests.cpp | 2 +- Tests/Allocators/PMRTests.cpp | 46 ++++++++++++++------ Tests/CMakeLists.txt | 10 +++++ Tests/Helpers/AllocationHelpers.h | 17 +++++++- 4 files changed, 59 insertions(+), 16 deletions(-) diff --git a/Tests/Allocators/MultiPoolAllocatorTests.cpp b/Tests/Allocators/MultiPoolAllocatorTests.cpp index cebbacf..ee9ecb9 100644 --- a/Tests/Allocators/MultiPoolAllocatorTests.cpp +++ b/Tests/Allocators/MultiPoolAllocatorTests.cpp @@ -630,7 +630,7 @@ auto resourceDefaults = test("MemoryPoolResource.defaults_prefill_sixteen_64k_blocks") = [] { auto resource = MemoryPoolResource(); - constexpr auto blockSize = std::size_t {64 * 1024}; + auto blockSize = std::size_t {64 * 1024}; auto prefilledCalls = countGlobalHeapCalls( [&] diff --git a/Tests/Allocators/PMRTests.cpp b/Tests/Allocators/PMRTests.cpp index fec320f..1fa41d9 100644 --- a/Tests/Allocators/PMRTests.cpp +++ b/Tests/Allocators/PMRTests.cpp @@ -14,7 +14,7 @@ using namespace nano; using EA::TestHelpers::countGlobalHeapCalls; -using EA::TestHelpers::CountingResource; +using EA::TestHelpers::proxyAllocationsPerContainer; using EA::TestHelpers::StackResource; namespace @@ -41,7 +41,8 @@ constexpr auto longText = "a string that is far too long for any SSO buffer"; auto objectStoresResource = test("PMR.Object.constructed_from_reference_or_pointer") = [] { - auto resource = CountingResource(); + auto stack = StackResource<1024>(); + auto& resource = stack.counter; auto fromReference = EA::PMR::Object>(resource); auto fromPointer = EA::PMR::Object>(&resource); @@ -55,8 +56,8 @@ auto objectStoresResource = auto objectAccessors = test("PMR.Object.get_arrow_and_star_reach_the_same_object") = [] { - auto resource = CountingResource(); - auto object = EA::PMR::Object>(resource); + auto stack = StackResource<1024>(); + auto object = EA::PMR::Object>(stack.counter); object->push_back(1); (*object).push_back(2); @@ -81,7 +82,7 @@ auto objectForwardsArgs = test("PMR.Object.forwards_args_after_the_resource") = check(object->values[0] == 7); check(object->values[7] == 7); check(object->values.get_allocator().resource() == &stack.counter); - check(stack.counter.allocations == 1); + check(stack.counter.allocations == 1 + proxyAllocationsPerContainer); }; auto objectNoGlobalHeap = @@ -130,7 +131,8 @@ auto objectCopyKeepsResource = test("PMR.Object.copy_uses_the_source_resource") check(sameContents); check(copyResource == &stack.counter); check(copyAllocatorResource == &stack.counter); - check(stack.counter.allocations == allocationsBefore + 1); + check(stack.counter.allocations + == allocationsBefore + 1 + proxyAllocationsPerContainer); }; auto objectMoveKeepsResource = @@ -157,8 +159,11 @@ auto objectMoveKeepsResource = movedAllocatorResource = moved->get_allocator().resource(); }); + //The move takes over the buffer; only MSVC's debug STL allocates anything, + //a fresh container proxy for the moved-to vector. check(heapCalls == 0); - check(stack.counter.allocations == allocationsBefore); + check(stack.counter.allocations + == allocationsBefore + proxyAllocationsPerContainer); check(stoleBuffer); check(sameContents); check(movedResource == &stack.counter); @@ -217,8 +222,9 @@ auto objectAssignAcrossResources = check(moveTarget.getResource() == &first.counter); check(moveTarget->get_allocator().resource() == &first.counter); - //Both targets allocated from their own resource, never from the source's. - check(first.counter.allocations == 2); + //Both targets allocated from their own resource, never from the source's + //(plus a debug-STL proxy each when they were constructed). + check(first.counter.allocations == 2 + 2 * proxyAllocationsPerContainer); check(second.counter.allocations == secondAllocationsBefore); }; @@ -277,8 +283,8 @@ auto vectorPropagatesToElements = check(heapCalls == 0); check(textIntact); check(elementResource == &stack.counter); - //One buffer for the vector, one per string. - check(stack.counter.allocations == 3); + //One buffer for the vector, one per string (and a debug-STL proxy each). + check(stack.counter.allocations == 3 + 3 * proxyAllocationsPerContainer); check(stack.counter.live() == 0); }; @@ -406,6 +412,13 @@ struct Mixin long long padding[3] {}; }; +//The over-alignment (and the padding it brings) is the point of this type, so +//silence MSVC's C4324 "structure was padded due to alignment specifier". +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable : 4324) +#endif + struct alignas(32) Derived : Mixin , Base @@ -421,6 +434,10 @@ struct alignas(32) Derived int value; }; +#ifdef _MSC_VER +#pragma warning(pop) +#endif + struct PlainBase { int value = 0; @@ -479,7 +496,8 @@ auto makeOwnedUsesAllocator = check(heapCalls == 0); check(innerResource == &stack.counter); - check(stack.counter.allocations == 2); + //The Owned allocation, the vector's buffer, and any debug-STL proxy. + check(stack.counter.allocations == 2 + proxyAllocationsPerContainer); check(stack.counter.live() == 0); }; @@ -488,7 +506,8 @@ auto makeOwnedThrowingConstructor = { //Throwing allocates the exception object on the heap, so only the //resource's books are checked here. - auto resource = CountingResource(); + auto stack = StackResource<1024>(); + auto& resource = stack.counter; auto threw = false; try @@ -626,4 +645,5 @@ auto makeOwnedMoveOnly = test("PMR.makeOwned.is_move_only_and_moves_freely") = [ check(resourceKept); check(stack.counter.allocations == 1); check(stack.counter.deallocations == 0); + //unique_ptr isn't a container, so no debug-STL proxy is involved here. }; diff --git a/Tests/CMakeLists.txt b/Tests/CMakeLists.txt index c37d707..eb1e8bd 100644 --- a/Tests/CMakeLists.txt +++ b/Tests/CMakeLists.txt @@ -21,6 +21,16 @@ FetchContent_Declare( ) FetchContent_MakeAvailable(ScopedMemoryAllocations) +#AllocationsChecker compiles its sources straight into the test executable, so +#keep our warnings-as-errors off third-party code (GCC rejects its operator +#delete replacements under -Wextra's -Wsized-deallocation). +set_source_files_properties( + ${ScopedMemoryAllocations_SOURCE_DIR}/Source/ScopedMemoryAllocations/Allocations.cpp + ${ScopedMemoryAllocations_SOURCE_DIR}/Source/ScopedMemoryAllocations/Malloc.cpp + ${ScopedMemoryAllocations_SOURCE_DIR}/Source/ScopedMemoryAllocations/Operators.cpp + PROPERTIES COMPILE_OPTIONS $<$:-w> +) + nano_add_executable(ea_data_structures_tests SOURCES Allocators/MultiPoolAllocatorTests.cpp diff --git a/Tests/Helpers/AllocationHelpers.h b/Tests/Helpers/AllocationHelpers.h index 13e53aa..1d57882 100644 --- a/Tests/Helpers/AllocationHelpers.h +++ b/Tests/Helpers/AllocationHelpers.h @@ -43,13 +43,26 @@ constexpr int expectedHeapCalls(int calls) return heapTrackingEnabled ? calls : 0; } +//MSVC's debug STL (_ITERATOR_DEBUG_LEVEL > 0) gives every container a +//"container proxy" allocated through the container's own allocator - so +//through its pmr resource. Exact resource allocation counts add this once per +//container constructed on the resource. +#if defined(_ITERATOR_DEBUG_LEVEL) && _ITERATOR_DEBUG_LEVEL > 0 +constexpr int proxyAllocationsPerContainer = 1; +#else +constexpr int proxyAllocationsPerContainer = 0; +#endif + //A memory_resource that forwards to an upstream resource and counts the //calls, so tests can tell which resource an allocation actually went through. +//The upstream is explicit on purpose: std::pmr::new_delete_resource() goes +//through aligned operator new on libstdc++, and ScopedMemoryAllocations' +//replacement of it passes alignments below sizeof(void*) straight to +//posix_memalign, which rejects them with std::bad_alloc. class CountingResource final : public std::pmr::memory_resource { public: - explicit CountingResource( - std::pmr::memory_resource* upstreamToUse = std::pmr::new_delete_resource()) + explicit CountingResource(std::pmr::memory_resource* upstreamToUse) : upstream(upstreamToUse) { }