diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index bef43104..a948ca96 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -24,6 +24,7 @@ set(PUBLIC_HEADERS include/join/io_ring_buffer.hpp include/join/proactor.hpp include/join/memory.hpp + include/join/bump_allocator.hpp include/join/slab_allocator.hpp include/join/queue.hpp include/join/backoff.hpp diff --git a/core/include/join/bump_allocator.hpp b/core/include/join/bump_allocator.hpp new file mode 100644 index 00000000..e2d4c749 --- /dev/null +++ b/core/include/join/bump_allocator.hpp @@ -0,0 +1,326 @@ +/** + * MIT License + * + * Copyright (c) 2026 Mathieu Rabine + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef JOIN_CORE_BUMP_ALLOCATOR_HPP +#define JOIN_CORE_BUMP_ALLOCATOR_HPP + +// libjoin. +#include +#include + +// C++. +#include +#include + +// C. +#include +#include + +namespace join +{ + /** + * @brief bump allocator owning a backend of fixed capacity. + */ + template + class BasicBumpAllocator + { + static_assert (Capacity > 0, "capacity must be at least 1"); + + public: + /** + * @brief create instance. + * @param args arguments forwarded to the backend. + * @throw std::system_error if the backend cannot be mapped. + */ + template + BasicBumpAllocator (Args&&... args) + : _backend (Capacity, std::forward (args)...) + , _base (static_cast (_backend.get ())) + { + } + + /** + * @brief copy constructor. + * @param other other object to copy. + */ + BasicBumpAllocator (const BasicBumpAllocator& other) = delete; + + /** + * @brief copy assignment operator. + * @param other other object to copy. + */ + BasicBumpAllocator& operator= (const BasicBumpAllocator& other) = delete; + + /** + * @brief move constructor. + * @param other other object to move. + */ + BasicBumpAllocator (BasicBumpAllocator&& other) noexcept + : _backend (std::move (other._backend)) + , _base (other._base) + , _cursor (std::move (other._cursor)) + { + other._base = nullptr; + other._cursor.store (0); + } + + /** + * @brief move assignment operator. + * @param other other object to move. + * @return this. + */ + BasicBumpAllocator& operator= (BasicBumpAllocator&& other) noexcept + { + _backend = std::move (other._backend); + _base = other._base; + _cursor = std::move (other._cursor); + + other._base = nullptr; + other._cursor.store (0); + + return *this; + } + + /** + * @brief destroy instance. + */ + ~BasicBumpAllocator () = default; + + /** + * @brief allocate memory at the cursor, aligned on the largest power of 2 dividing size, up to max_align_t. + * @param size size of the allocation request in bytes. + * @return pointer to the allocated memory, or nullptr on failure. + */ + void* allocate (size_t size) noexcept + { + const size_t bits = size | alignof (std::max_align_t); + return allocate (size, bits & (~bits + 1)); + } + + /** + * @brief allocate memory at the cursor with an explicit alignment. + * @param size size of the allocation request in bytes. + * @param alignment alignment of the allocation, must be a power of 2. + * @return pointer to the allocated memory, or nullptr on failure. + */ + void* allocate (size_t size, size_t alignment) noexcept + { + if (JOIN_UNLIKELY ((_base == nullptr) || !isPow2 (alignment))) + { + return nullptr; + } + + const uintptr_t base = reinterpret_cast (_base); + uint64_t current = _cursor.load (); + uint64_t offset; + + do + { + offset = ((base + current + alignment - 1) & ~(alignment - 1)) - base; + + if (JOIN_UNLIKELY ((offset > Capacity) || (size > Capacity - offset))) + { + return nullptr; + } + } + while (!_cursor.advance (current, offset + size)); + + return _base + offset; + } + + /** + * @brief release all allocations at once. + */ + void reset () noexcept + { + _cursor.store (0); + } + + /** + * @brief get the capacity. + * @return capacity in bytes. + */ + static constexpr size_t capacity () noexcept + { + return Capacity; + } + + /** + * @brief get the number of bytes consumed, alignment padding included. + * @return consumed bytes. + */ + size_t used () const noexcept + { + return _cursor.load (); + } + + /** + * @brief check if the bump allocator still owns a memory region. + * @return true if the bump allocator owns a memory region, false if it has been moved from. + */ + bool hasBackend () const noexcept + { + return _backend.mapped (); + } + +#ifdef JOIN_HAS_NUMA + /** + * @brief bind memory to a NUMA node. + * @param numa NUMA node ID. + * @return 0 on success, -1 on failure. + */ + int mbind (int numa) const noexcept + { + return _backend.mbind (numa); + } +#endif + + /** + * @brief lock memory in RAM. + * @return 0 on success, -1 on failure. + */ + int mlock () const noexcept + { + return _backend.mlock (); + } + + private: + /// memory backend. + Backend _backend; + + /// start of the backend region. + char* _base = nullptr; + + /// allocation cursor. + SyncPolicy _cursor; + }; + + /** + * @brief bump sync policy using a lock-free cursor. + */ + class BumpAtomic + { + public: + /** + * @brief default constructor. + */ + BumpAtomic () noexcept = default; + + /** + * @brief move constructor. + * @param other other object to move. + */ + BumpAtomic (BumpAtomic&& other) noexcept + : _value (other._value.load (std::memory_order_relaxed)) + { + } + + /** + * @brief move assignment operator. + * @param other other object to move. + * @return this. + */ + BumpAtomic& operator= (BumpAtomic&& other) noexcept + { + _value.store (other._value.load (std::memory_order_relaxed), std::memory_order_relaxed); + return *this; + } + + /** + * @brief load the cursor. + * @return cursor value. + */ + uint64_t load () const noexcept + { + return _value.load (std::memory_order_relaxed); + } + + /** + * @brief store the cursor. + * @param value new cursor value. + */ + void store (uint64_t value) noexcept + { + _value.store (value, std::memory_order_relaxed); + } + + /** + * @brief advance the cursor. + * @param expected expected cursor value, updated on failure. + * @param desired new cursor value. + * @return true if the cursor was advanced, false if another thread moved it first. + */ + bool advance (uint64_t& expected, uint64_t desired) noexcept + { + return _value.compare_exchange_weak (expected, desired, std::memory_order_relaxed); + } + + private: + /// cursor value. + alignas (64) std::atomic_uint64_t _value{0}; + }; + + /** + * @brief bump sync policy using a plain cursor, single thread only. + */ + class BumpPlain + { + public: + /** + * @brief load the cursor. + * @return cursor value. + */ + uint64_t load () const noexcept + { + return _value; + } + + /** + * @brief store the cursor. + * @param value new cursor value. + */ + void store (uint64_t value) noexcept + { + _value = value; + } + + /** + * @brief advance the cursor. + * @param expected expected cursor value. + * @param desired new cursor value. + * @return always true. + */ + bool advance ([[maybe_unused]] uint64_t& expected, uint64_t desired) noexcept + { + _value = desired; + return true; + } + + private: + /// cursor value. + uint64_t _value = 0; + }; +} + +#endif diff --git a/core/include/join/memory.hpp b/core/include/join/memory.hpp index e12c7dde..60501d1c 100644 --- a/core/include/join/memory.hpp +++ b/core/include/join/memory.hpp @@ -49,6 +49,11 @@ namespace join { /// allocator forward declarations. + class BumpAtomic; + + template + class BasicBumpAllocator; + template class BasicSlabAllocator; @@ -155,6 +160,9 @@ namespace join class LocalMem { public: + template + using BumpAllocator = BasicBumpAllocator; + template using SlabAllocator = BasicSlabAllocator; diff --git a/core/tests/CMakeLists.txt b/core/tests/CMakeLists.txt index 96e33c3e..f3c06c9e 100644 --- a/core/tests/CMakeLists.txt +++ b/core/tests/CMakeLists.txt @@ -116,6 +116,11 @@ target_link_libraries(local_mem.gtest ${JOIN_CORE} GTest::gtest_main rt) add_test(NAME local_mem.gtest COMMAND local_mem.gtest) install(TARGETS local_mem.gtest RUNTIME DESTINATION ${CMAKE_INSTALL_DATADIR}/${PROJECT_NAME}/test) +add_executable(local_bump.gtest local_bump_test.cpp) +target_link_libraries(local_bump.gtest ${JOIN_CORE} GTest::gtest_main rt) +add_test(NAME local_bump.gtest COMMAND local_bump.gtest) +install(TARGETS local_bump.gtest RUNTIME DESTINATION ${CMAKE_INSTALL_DATADIR}/${PROJECT_NAME}/test) + add_executable(local_slab.gtest local_slab_test.cpp) target_link_libraries(local_slab.gtest ${JOIN_CORE} GTest::gtest_main rt) add_test(NAME local_slab.gtest COMMAND local_slab.gtest) diff --git a/core/tests/local_bump_test.cpp b/core/tests/local_bump_test.cpp new file mode 100644 index 00000000..e76f856c --- /dev/null +++ b/core/tests/local_bump_test.cpp @@ -0,0 +1,309 @@ +/** + * MIT License + * + * Copyright (c) 2026 Mathieu Rabine + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +// libjoin. +#include +#include +#include + +// Libraries. +#include + +using join::ScopedStats; +using join::BumpPlain; +using join::LocalMem; +using join::Thread; +using join::Rdtsc; + +/** + * @brief test move. + */ +TEST (LocalBump, move) +{ + LocalMem::BumpAllocator<64> allocator1; + LocalMem::BumpAllocator<64> allocator2; + + ASSERT_NE (allocator1.allocate (32), nullptr); + + allocator2 = std::move (allocator1); + + ASSERT_FALSE (allocator1.hasBackend ()); + ASSERT_EQ (allocator1.used (), 0); + ASSERT_EQ (allocator1.allocate (1), nullptr); + ASSERT_EQ (allocator2.used (), 32); + + LocalMem::BumpAllocator<64> allocator3 (std::move (allocator2)); + + ASSERT_FALSE (allocator2.hasBackend ()); + ASSERT_EQ (allocator2.used (), 0); + ASSERT_TRUE (allocator3.hasBackend ()); + ASSERT_EQ (allocator3.used (), 32); + ASSERT_NE (allocator3.allocate (32), nullptr); + ASSERT_EQ (allocator3.allocate (1), nullptr); + + LocalMem::BumpAllocator<64, BumpPlain> plain1; + LocalMem::BumpAllocator<64, BumpPlain> plain2; + + ASSERT_NE (plain1.allocate (32), nullptr); + + plain2 = std::move (plain1); + + ASSERT_FALSE (plain1.hasBackend ()); + ASSERT_EQ (plain1.used (), 0); + ASSERT_EQ (plain1.allocate (1), nullptr); + ASSERT_EQ (plain2.used (), 32); + + LocalMem::BumpAllocator<64, BumpPlain> plain3 (std::move (plain2)); + + ASSERT_FALSE (plain2.hasBackend ()); + ASSERT_EQ (plain2.used (), 0); + ASSERT_TRUE (plain3.hasBackend ()); + ASSERT_EQ (plain3.used (), 32); + ASSERT_NE (plain3.allocate (32), nullptr); + ASSERT_EQ (plain3.allocate (1), nullptr); +} + +/** + * @brief test allocate. + */ +TEST (LocalBump, allocate) +{ + LocalMem::BumpAllocator<256> allocator; + + char* p1 = static_cast (allocator.allocate (1)); + ASSERT_NE (p1, nullptr); + + char* p2 = static_cast (allocator.allocate (24)); + ASSERT_NE (p2, nullptr); + ASSERT_EQ (reinterpret_cast (p2) % 8, 0); + ASSERT_EQ (p2 - p1, 8); + + char* p3 = static_cast (allocator.allocate (3)); + ASSERT_NE (p3, nullptr); + ASSERT_EQ (p3 - p2, 24); + + char* p4 = static_cast (allocator.allocate (64)); + ASSERT_NE (p4, nullptr); + ASSERT_EQ (reinterpret_cast (p4) % 16, 0); + ASSERT_EQ (p4 - p1, 48); + + ASSERT_EQ (allocator.allocate (SIZE_MAX), nullptr); + + ASSERT_NE (allocator.allocate (144), nullptr); + ASSERT_EQ (allocator.allocate (1), nullptr); + + LocalMem::BumpAllocator<256> aligned; + + ASSERT_EQ (aligned.allocate (8, 0), nullptr); + ASSERT_EQ (aligned.allocate (8, 3), nullptr); + + char* a1 = static_cast (aligned.allocate (1, 1)); + ASSERT_NE (a1, nullptr); + + char* a2 = static_cast (aligned.allocate (1, 64)); + ASSERT_NE (a2, nullptr); + ASSERT_EQ (reinterpret_cast (a2) % 64, 0); + ASSERT_EQ (a2 - a1, 64); + + ASSERT_EQ (aligned.allocate (1, 4096), nullptr); + + LocalMem::BumpAllocator<256, BumpPlain> plain; + + char* u1 = static_cast (plain.allocate (1)); + ASSERT_NE (u1, nullptr); + + char* u2 = static_cast (plain.allocate (24)); + ASSERT_NE (u2, nullptr); + ASSERT_EQ (u2 - u1, 8); + + ASSERT_EQ (plain.allocate (8, 3), nullptr); + ASSERT_EQ (plain.allocate (SIZE_MAX), nullptr); + + char* u3 = static_cast (plain.allocate (1, 64)); + ASSERT_NE (u3, nullptr); + ASSERT_EQ (u3 - u1, 64); + + ASSERT_NE (plain.allocate (176), nullptr); + ASSERT_EQ (plain.allocate (1), nullptr); +} + +/** + * @brief test reset. + */ +TEST (LocalBump, reset) +{ + LocalMem::BumpAllocator<64> allocator; + + void* p1 = allocator.allocate (64); + ASSERT_NE (p1, nullptr); + ASSERT_EQ (allocator.allocate (1), nullptr); + + allocator.reset (); + ASSERT_EQ (allocator.used (), 0); + ASSERT_EQ (allocator.allocate (64), p1); + + LocalMem::BumpAllocator<64, BumpPlain> plain; + + void* u1 = plain.allocate (64); + ASSERT_NE (u1, nullptr); + ASSERT_EQ (plain.allocate (1), nullptr); + + plain.reset (); + ASSERT_EQ (plain.used (), 0); + ASSERT_EQ (plain.allocate (64), u1); +} + +/** + * @brief test capacity. + */ +TEST (LocalBump, capacity) +{ + ASSERT_EQ (LocalMem::BumpAllocator<128>::capacity (), 128); + ASSERT_EQ ((LocalMem::BumpAllocator<128, BumpPlain>::capacity ()), 128); +} + +/** + * @brief test used. + */ +TEST (LocalBump, used) +{ + LocalMem::BumpAllocator<64> allocator; + ASSERT_EQ (allocator.used (), 0); + + ASSERT_NE (allocator.allocate (1), nullptr); + ASSERT_EQ (allocator.used (), 1); + + ASSERT_NE (allocator.allocate (8), nullptr); + ASSERT_EQ (allocator.used (), 16); + + LocalMem::BumpAllocator<64, BumpPlain> plain; + ASSERT_EQ (plain.used (), 0); + + ASSERT_NE (plain.allocate (1), nullptr); + ASSERT_EQ (plain.used (), 1); + + ASSERT_NE (plain.allocate (8), nullptr); + ASSERT_EQ (plain.used (), 16); +} + +/** + * @brief test hasBackend. + */ +TEST (LocalBump, hasBackend) +{ + LocalMem::BumpAllocator<64> allocator; + ASSERT_TRUE (allocator.hasBackend ()); +} + +#ifdef JOIN_HAS_NUMA +/** + * @brief test mbind. + */ +TEST (LocalBump, mbind) +{ + LocalMem::BumpAllocator<64> allocator; + ASSERT_EQ (allocator.mbind (0), 0) << join::lastError.message (); +} +#endif + +/** + * @brief test mlock. + */ +TEST (LocalBump, mlock) +{ + LocalMem::BumpAllocator<64> allocator; + ASSERT_EQ (allocator.mlock (), 0) << join::lastError.message (); +} + +/** + * @brief test concurrent access. + */ +TEST (LocalBump, benchmark) +{ + const int iterations = 100000; + const int numThreads = 4; + + LocalMem::BumpAllocator allocator; + allocator.mlock (); + + std::vector threads; + for (int i = 0; i < numThreads - 1; ++i) + { + threads.emplace_back ([&] () { + for (int j = 0; j < iterations; ++j) + { + EXPECT_NE (allocator.allocate (16), nullptr); + } + }); + } + + Rdtsc::Stats aStats ("allocate"); + + for (int i = 0; i < iterations; ++i) + { + void* p = nullptr; + { + ScopedStats guard (aStats); + p = allocator.allocate (16); + } + EXPECT_NE (p, nullptr); + } + + for (auto& t : threads) + { + t.join (); + } + + EXPECT_EQ (allocator.used (), allocator.capacity ()); + EXPECT_EQ (allocator.allocate (1), nullptr); + + LocalMem::BumpAllocator plain; + plain.mlock (); + + Rdtsc::Stats pStats ("allocate plain"); + + for (int i = 0; i < iterations; ++i) + { + void* p = nullptr; + { + ScopedStats guard (pStats); + p = plain.allocate (16); + } + EXPECT_NE (p, nullptr); + } + + std::cout << join::statsHeader << "\n"; + std::cout << join::kops << join::nsec << std::fixed << std::setprecision (0) << aStats << "\n"; + std::cout << join::kops << join::nsec << std::fixed << std::setprecision (0) << pStats << "\n"; +} + +/** + * @brief main function. + */ +int main (int argc, char** argv) +{ + Rdtsc::calibrate (); + testing::InitGoogleTest (&argc, argv); + return RUN_ALL_TESTS (); +}