diff --git a/toolbelt/payload_buffer.cc b/toolbelt/payload_buffer.cc index cf7a221..0ac6d37 100644 --- a/toolbelt/payload_buffer.cc +++ b/toolbelt/payload_buffer.cc @@ -53,6 +53,9 @@ char *PayloadBuffer::SetString(PayloadBuffer **self, const char *s, size_t len, BufferOffset header_offset) { // Get address of the string header BufferOffset *hdr = (*self)->ToAddress(header_offset); + if (hdr == nullptr) { + return nullptr; + } void *str = nullptr; // Load the pointer and convert to address. @@ -66,6 +69,9 @@ char *PayloadBuffer::SetString(PayloadBuffer **self, const char *s, size_t len, } else { str = Allocate(self, len + 4, 4, false); } + if (str == nullptr) { + return nullptr; + } uint32_t *p = reinterpret_cast(str); p[0] = uint32_t(len); memcpy(p + 1, s, len); @@ -73,6 +79,10 @@ char *PayloadBuffer::SetString(PayloadBuffer **self, const char *s, size_t len, // The buffer may have moved. Reassign the address of the string // back into the header. BufferOffset *oldp = (*self)->ToAddress(header_offset); + if (oldp == nullptr) { + (*self)->Free(str); + return nullptr; + } *oldp = (*self)->ToOffset(str); return reinterpret_cast(str); } @@ -80,6 +90,9 @@ char *PayloadBuffer::SetString(PayloadBuffer **self, const char *s, size_t len, void PayloadBuffer::ClearString(PayloadBuffer **self, BufferOffset header_offset) { BufferOffset *hdr = (*self)->ToAddress(header_offset); + if (hdr == nullptr) { + return; + } if (*hdr != 0) { (*self)->Free((*self)->ToAddress(*hdr)); // Free doesn't move the buffer so the address is still valid. @@ -89,42 +102,69 @@ void PayloadBuffer::ClearString(PayloadBuffer **self, // 'addr' is the address of the pointer to the string data. std::string PayloadBuffer::GetString(const StringHeader *addr) const { - const uint32_t *p = reinterpret_cast(ToAddress(*addr)); - if (p == nullptr) { + if (addr == nullptr) { + return ""; + } + const uint32_t *p = ToAddress(*addr); + if (p == nullptr || (*p > 0 && !IsValidAddress(p + 1, *p))) { return ""; } return std::string(reinterpret_cast(p + 1), *p); } std::string_view PayloadBuffer::GetStringView(const StringHeader *addr) const { - const uint32_t *p = reinterpret_cast(ToAddress(*addr)); - if (p == nullptr) { + if (addr == nullptr) { + return ""; + } + const uint32_t *p = ToAddress(*addr); + if (p == nullptr || (*p > 0 && !IsValidAddress(p + 1, *p))) { return ""; } return std::string_view(reinterpret_cast(p + 1), *p); } size_t PayloadBuffer::StringSize(const StringHeader *addr) const { - const uint32_t *p = reinterpret_cast(ToAddress(*addr)); - if (p == nullptr) { + if (addr == nullptr) { + return 0; + } + const uint32_t *p = ToAddress(*addr); + if (p == nullptr || (*p > 0 && !IsValidAddress(p + 1, *p))) { return 0; } return size_t(*p); } const char *PayloadBuffer::StringData(const StringHeader *addr) const { - const uint32_t *p = reinterpret_cast(ToAddress(*addr)); - if (p == nullptr) { + if (addr == nullptr) { + return nullptr; + } + const uint32_t *p = ToAddress(*addr); + if (p == nullptr || (*p > 0 && !IsValidAddress(p + 1, *p))) { return nullptr; } return reinterpret_cast(p + 1); } +bool PayloadBuffer::StringWithinBounds(const StringHeader *addr) const { + if (addr == nullptr) { + return false; + } + // An unset string has a zero body offset and serializes as empty. + if (*addr == 0) { + return true; + } + const uint32_t *p = ToAddress(*addr); + return p != nullptr && (*p == 0 || IsValidAddress(p + 1, *p)); +} + absl::Span PayloadBuffer::AllocateString(PayloadBuffer **self, size_t len, BufferOffset header_offset, bool clear) { // Get address of the string header BufferOffset *hdr = (*self)->ToAddress(header_offset); + if (hdr == nullptr) { + return {}; + } void *str = nullptr; // Load the pointer and convert to address. @@ -138,12 +178,19 @@ absl::Span PayloadBuffer::AllocateString(PayloadBuffer **self, size_t len, } else { str = Allocate(self, len + 4, 4, clear); } + if (str == nullptr) { + return {}; + } uint32_t *p = reinterpret_cast(str); p[0] = uint32_t(len); // The buffer may have moved. Reassign the address of the string // back into the header. BufferOffset *oldp = (*self)->ToAddress(header_offset); + if (oldp == nullptr) { + (*self)->Free(str); + return {}; + } *oldp = (*self)->ToOffset(str); // The span returned is the string data, not the address of the length. return absl::Span(reinterpret_cast(str) + 4, len); @@ -679,31 +726,49 @@ void *PayloadBuffer::Realloc(PayloadBuffer **buffer, void *p, uint32_t n, (*buffer)->Free(p); return newp; } -bool PayloadBuffer::PrimeBitmapAllocator(PayloadBuffer **self, size_t size) { - int index = BitmapRunIndex(size); - if (index < 0) { - return true; - } - if ((*self)->bitmaps[index] != 0) { - return true; - } + +static bool InitializeBitMapRunVector(PayloadBuffer **self, int index, + uint32_t size, uint32_t num) { BufferOffset offset = (*self)->AllocateBitMapRunVector(self); if (offset == 0) { return false; } - (*self)->bitmaps[index] = offset; + auto free_bitmap_vector = [self, offset]() { + VectorHeader *hdr = (*self)->ToAddress(offset); + PayloadBuffer::VectorClear(self, hdr); + (*self)->Free((*self)->ToAddress(offset)); + }; - BitMapRun *run = PayloadBuffer::AllocateBitMapRun( - self, bitmp_run_infos[index].size, bitmp_run_infos[index].num); + BitMapRun *run = PayloadBuffer::AllocateBitMapRun(self, size, num); if (run == nullptr) { + free_bitmap_vector(); return false; } + // Re-derive hdr since AllocateBitMapRun may have triggered a buffer resize. - VectorHeader *hdr = (*self)->ToAddress((*self)->bitmaps[index]); - (*self)->VectorPush(self, hdr, (*self)->ToOffset(run), false); + VectorHeader *hdr = (*self)->ToAddress(offset); + BufferOffset run_offset = (*self)->ToOffset(run); + if (!(*self)->VectorPush(self, hdr, run_offset, false)) { + (*self)->Free((*self)->ToAddress(run_offset)); + free_bitmap_vector(); + return false; + } + (*self)->bitmaps[index] = offset; return true; } +bool PayloadBuffer::PrimeBitmapAllocator(PayloadBuffer **self, size_t size) { + int index = BitmapRunIndex(size); + if (index < 0) { + return true; + } + if ((*self)->bitmaps[index] != 0) { + return true; + } + return InitializeBitMapRunVector(self, index, bitmp_run_infos[index].size, + bitmp_run_infos[index].num); +} + BufferOffset PayloadBuffer::AllocateBitMapRunVector(PayloadBuffer **self) { // Allocate space for the VectorHeader. Although this is a small block, we // can't use the small block allocator because this is initializing it. @@ -714,8 +779,11 @@ BufferOffset PayloadBuffer::AllocateBitMapRunVector(PayloadBuffer **self) { BufferOffset hdr_offset = (*self)->ToOffset(hdr); // Preallocate space for 8 elements. - VectorReserve(self, reinterpret_cast(hdr), 8, - false); + if (!VectorReserve(self, reinterpret_cast(hdr), + 8, false)) { + (*self)->Free((*self)->ToAddress(hdr_offset)); + return 0; + } return hdr_offset; } @@ -741,11 +809,9 @@ void *BitMapRun::Allocate(PayloadBuffer **pb, int index, uint32_t, int size, int num, bool clear) { // Lazy init of vector. if ((*pb)->bitmaps[index] == 0) { - BufferOffset offset = (*pb)->AllocateBitMapRunVector(pb); - if (offset == 0) { + if (!InitializeBitMapRunVector(pb, index, size, num)) { return nullptr; } - (*pb)->bitmaps[index] = offset; } for (;;) { // Re-derive hdr each iteration since allocations below may trigger a @@ -791,7 +857,11 @@ void *BitMapRun::Allocate(PayloadBuffer **pb, int index, uint32_t, int size, // Re-derive hdr since AllocateBitMapRun may have triggered a buffer // resize, invalidating the previous pointer. hdr = (*pb)->ToAddress((*pb)->bitmaps[index]); - (*pb)->VectorPush(pb, hdr, (*pb)->ToOffset(run), false); + BufferOffset run_offset = (*pb)->ToOffset(run); + if (!(*pb)->VectorPush(pb, hdr, run_offset, false)) { + (*pb)->Free((*pb)->ToAddress(run_offset)); + return nullptr; + } } } diff --git a/toolbelt/payload_buffer.h b/toolbelt/payload_buffer.h index 08b1775..f112a86 100644 --- a/toolbelt/payload_buffer.h +++ b/toolbelt/payload_buffer.h @@ -9,6 +9,7 @@ #include #include #include +#include namespace toolbelt { @@ -105,7 +106,7 @@ inline constexpr int kBitmapRunSize2 = 32; inline constexpr int kBitmapRunSize3 = 64; inline constexpr int kBitmapRunSize4 = 128; -// In order to allow free to work without searching, we use the 8 bytes +// In order to allow free to work without searching, we use the 4 bytes // preceding the allocated block in the run to store the size of the block, the // index into the BitMapRun vector and the bit number in the bitmap. In order // to distinguish this between small blocks and regular blocks allocated from @@ -246,6 +247,7 @@ struct PayloadBuffer { // The string is copied in. // C-string style (allows for no allocation of std::string). + // Returns nullptr without changing the header if allocation fails. static char *SetString(PayloadBuffer **self, const char *s, size_t len, BufferOffset header_offset); @@ -257,6 +259,7 @@ struct PayloadBuffer { static void ClearString(PayloadBuffer **self, BufferOffset header_offset); + // Returns an empty span without changing the header if allocation fails. static absl::Span AllocateString(PayloadBuffer **self, size_t len, BufferOffset header_offset, bool clear = false); @@ -269,16 +272,22 @@ struct PayloadBuffer { template void Set(BufferOffset offset, T v); template T &Get(BufferOffset offset); + // Appends v to the vector. Returns false without modifying hdr if allocation + // fails. template - static void VectorPush(PayloadBuffer **self, VectorHeader *hdr, T v, + static bool VectorPush(PayloadBuffer **self, VectorHeader *hdr, T v, bool enable_small_block = true); + // Reserves space for at least n elements. Returns false without modifying hdr + // if allocation fails. template - static void VectorReserve(PayloadBuffer **self, VectorHeader *hdr, size_t n, + static bool VectorReserve(PayloadBuffer **self, VectorHeader *hdr, size_t n, bool enable_small_block = true); + // Resizes the vector to n elements. Returns false without modifying hdr if + // allocation fails. template - static void VectorResize(PayloadBuffer **self, VectorHeader *hdr, size_t n); + static bool VectorResize(PayloadBuffer **self, VectorHeader *hdr, size_t n); template static void VectorClear(PayloadBuffer **self, VectorHeader *hdr); @@ -298,10 +307,19 @@ struct PayloadBuffer { return StringData(ToAddress(header_offset)); } + // True when the string at header_offset is safe to serialize: either unset + // (body offset 0, serializes as empty) or its length prefix and declared body + // lie wholly within the buffer. Lets the serializer reject a forged string + // rather than silently emitting empty as the general readers do. + bool StringWithinBounds(BufferOffset header_offset) const { + return StringWithinBounds(ToAddress(header_offset)); + } + std::string GetString(const StringHeader *addr) const; std::string_view GetStringView(const StringHeader *addr) const; size_t StringSize(const StringHeader *addr) const; const char *StringData(const StringHeader *addr) const; + bool StringWithinBounds(const StringHeader *addr) const; template T VectorGet(const VectorHeader *hdr, size_t index) const; @@ -340,12 +358,27 @@ struct PayloadBuffer { return (magic & kBitMapMask) == kMovableBufferMagic; } - bool IsValidAddress(const void *addr, size_t size) const { + // Integer-only bounds check on a [offset, offset + size) range. size == 0 + // means the access extent is unknown (e.g. a void* or a variable-length + // region); only the start is validated. The range form is written as a + // subtraction to avoid overflow when offset + size would wrap. + bool IsValidOffset(size_t offset, size_t size) const { if (size == 0) { - size = full_size; + return offset < full_size; + } + return offset <= full_size && size <= full_size - offset; + } + + bool IsValidAddress(const void *addr, size_t size) const { + // Compare and subtract through uintptr_t rather than pointers: 'addr' may + // come from an unrelated allocation, and comparing/subtracting unrelated + // pointers is undefined behavior. + const uintptr_t base = reinterpret_cast(this); + const uintptr_t a = reinterpret_cast(addr); + if (a < base) { + return false; } - return addr >= reinterpret_cast(this) && - addr < reinterpret_cast(this) + size; + return IsValidOffset(static_cast(a - base), size); } // Given the address of a block, return the size of the block. This is @@ -367,13 +400,20 @@ struct PayloadBuffer { if (!IsValidMagic()) { return nullptr; } - // Validate that we don't go outside the buffer. - char *addr = reinterpret_cast(this) + offset; - if (!IsValidAddress(addr, size)) { + // Without an explicit size, bound the access by sizeof(T) so a value that + // starts in-bounds but extends past the buffer end is rejected. + if constexpr (!std::is_void_v) { + if (size == 0) { + size = sizeof(T); + } + } + // Validate with integer arithmetic before forming any pointer past the + // buffer, which would itself be undefined behavior. + if (!IsValidOffset(offset, size)) { return nullptr; } - return reinterpret_cast(addr); + return reinterpret_cast(reinterpret_cast(this) + offset); } template BufferOffset ToOffset(T *addr, size_t size = 0) { @@ -383,11 +423,16 @@ struct PayloadBuffer { if (!IsValidMagic()) { return 0; } + if constexpr (!std::is_void_v) { + if (size == 0) { + size = sizeof(T); + } + } if (!IsValidAddress(addr, size)) { return 0; } - return reinterpret_cast(addr) - - reinterpret_cast(this); + return static_cast(reinterpret_cast(addr) - + reinterpret_cast(this)); } template @@ -398,9 +443,16 @@ struct PayloadBuffer { if (!IsValidMagic()) { return nullptr; } - // Validate that we don't go outside the buffer. - const char *addr = reinterpret_cast(this) + offset; - if (!IsValidAddress(addr, size)) { + // Without an explicit size, bound the access by sizeof(T) so a value that + // starts in-bounds but extends past the buffer end is rejected. + if constexpr (!std::is_void_v) { + if (size == 0) { + size = sizeof(T); + } + } + // Validate with integer arithmetic before forming any pointer past the + // buffer, which would itself be undefined behavior. + if (!IsValidOffset(offset, size)) { return nullptr; } @@ -416,11 +468,16 @@ struct PayloadBuffer { if (!IsValidMagic()) { return 0; } + if constexpr (!std::is_void_v) { + if (size == 0) { + size = sizeof(T); + } + } if (!IsValidAddress(addr, size)) { return 0; } - return reinterpret_cast(addr) - - reinterpret_cast(this); + return static_cast(reinterpret_cast(addr) - + reinterpret_cast(this)); } void InsertNewFreeBlockAtEnd(FreeBlockHeader *free_block, @@ -474,7 +531,7 @@ template inline T &PayloadBuffer::Get(BufferOffset offset) { } template -inline void PayloadBuffer::VectorPush(PayloadBuffer **self, VectorHeader *hdr, +inline bool PayloadBuffer::VectorPush(PayloadBuffer **self, VectorHeader *hdr, T v, bool enable_small_block) { // hdr points to a VectorHeader: // uint32_t num_elements; - number of elements in the vector @@ -487,6 +544,9 @@ inline void PayloadBuffer::VectorPush(PayloadBuffer **self, VectorHeader *hdr, if (hdr->data == 0) { // The vector is empty, allocate it with a default size of 2. void *vecp = Allocate(self, 2 * sizeof(T), true, enable_small_block); + if (vecp == nullptr) { + return false; + } VectorHeader *new_hdr = (*self)->ToAddress(hdr_offset); new_hdr->data = (*self)->ToOffset(vecp); hdr = new_hdr; @@ -494,11 +554,14 @@ inline void PayloadBuffer::VectorPush(PayloadBuffer **self, VectorHeader *hdr, // Vector has some values in it. Retrieve the total size from // the allocated block header (before the start of the memory) uint32_t *block = (*self)->ToAddress(hdr->data); - uint32_t current_size = DecodeSize(block); + uint32_t current_size = DecodedSize(block); if (current_size == total_size) { // Need to double the size of the memory. void *vecp = Realloc(self, block, 2 * hdr->num_elements * sizeof(T), true, enable_small_block); + if (vecp == nullptr) { + return false; + } VectorHeader *new_hdr = (*self)->ToAddress(hdr_offset); new_hdr->data = (*self)->ToOffset(vecp); hdr = new_hdr; @@ -510,57 +573,77 @@ inline void PayloadBuffer::VectorPush(PayloadBuffer **self, VectorHeader *hdr, *valuep = v; // Increment the number of elements. hdr->num_elements++; + return true; } template -inline void PayloadBuffer::VectorReserve(PayloadBuffer **self, +inline bool PayloadBuffer::VectorReserve(PayloadBuffer **self, VectorHeader *hdr, size_t n, bool enable_small_block) { + if (n == 0) { + return true; + } BufferOffset hdr_offset = (*self)->ToOffset(hdr); if (hdr->data == 0) { void *vecp = Allocate(self, n * sizeof(T), false, enable_small_block); - VectorHeader* new_hdr = (*self)->ToAddress(hdr_offset); + if (vecp == nullptr) { + return false; + } + VectorHeader *new_hdr = (*self)->ToAddress(hdr_offset); new_hdr->data = (*self)->ToOffset(vecp); - hdr = new_hdr; } else { // Vector has some values in it. Retrieve the total size from // the allocated block header (before the start of the memory) uint32_t *block = (*self)->ToAddress(hdr->data); - uint32_t current_size = DecodeSize(block); + uint32_t current_size = DecodedSize(block); if (current_size < n * sizeof(T)) { // Need to expand the memory to the size given. void *vecp = Realloc(self, block, n * sizeof(T), false, enable_small_block); - VectorHeader* new_hdr = (*self)->ToAddress(hdr_offset); + if (vecp == nullptr) { + return false; + } + VectorHeader *new_hdr = (*self)->ToAddress(hdr_offset); new_hdr->data = (*self)->ToOffset(vecp); - hdr = new_hdr; } } + return true; } template -inline void PayloadBuffer::VectorResize(PayloadBuffer **self, VectorHeader *hdr, +inline bool PayloadBuffer::VectorResize(PayloadBuffer **self, VectorHeader *hdr, size_t n) { + if (n == 0 && hdr->data == 0) { + hdr->num_elements = 0; + return true; + } BufferOffset hdr_offset = (*self)->ToOffset(hdr); if (hdr->data == 0) { void *vecp = Allocate(self, n * sizeof(T)); - VectorHeader* new_hdr = (*self)->ToAddress(hdr_offset); + if (vecp == nullptr) { + return false; + } + VectorHeader *new_hdr = (*self)->ToAddress(hdr_offset); new_hdr->data = (*self)->ToOffset(vecp); hdr = new_hdr; } else { // Vector has some values in it. Retrieve the total size from // the allocated block header (before the start of the memory) uint32_t *block = (*self)->ToAddress(hdr->data); - uint32_t current_size = DecodeSize(block); + uint32_t current_size = DecodedSize(block); if (current_size < n * sizeof(T)) { // Need to expand the memory to the size given. void *vecp = Realloc(self, block, n * sizeof(T), 8); - VectorHeader* new_hdr = (*self)->ToAddress(hdr_offset); + if (vecp == nullptr) { + return false; + } + VectorHeader *new_hdr = (*self)->ToAddress(hdr_offset); new_hdr->data = (*self)->ToOffset(vecp); hdr = new_hdr; } } hdr->num_elements = n; + return true; } template @@ -578,7 +661,7 @@ inline T PayloadBuffer::VectorGet(const VectorHeader *hdr, size_t index) const { if (index >= hdr->num_elements) { return static_cast(0); } - const T *addr = ToAddress(hdr->data); + const T *addr = ToAddress(hdr->data, (index + 1) * sizeof(T)); if (addr == nullptr) { return static_cast(0); } diff --git a/toolbelt/payload_buffer_test.cc b/toolbelt/payload_buffer_test.cc index cd7d9db..2c8aa2e 100644 --- a/toolbelt/payload_buffer_test.cc +++ b/toolbelt/payload_buffer_test.cc @@ -3,6 +3,7 @@ #include "toolbelt/payload_buffer.h" #include #include +#include #include using PayloadBuffer = toolbelt::PayloadBuffer; @@ -180,6 +181,167 @@ TEST(BufferTest, SmallBlockAllocFree) { free(buffer); } +TEST(BufferTest, PrimeBitmapAllocatorReserveFailureReclaimsVectorHeader) { + constexpr size_t kReserveAllocationSize = + 8 * sizeof(BufferOffset) + sizeof(uint64_t); + constexpr size_t kSize = sizeof(PayloadBuffer) + kReserveAllocationSize; + char *buffer = (char *)calloc(kSize, 1); + PayloadBuffer *pb = new (buffer) PayloadBuffer(kSize); + + const BufferOffset initial_free_list = pb->free_list; + toolbelt::FreeBlockHeader *initial_free_block = + pb->ToAddress(initial_free_list); + ASSERT_NE(nullptr, initial_free_block); + const uint32_t initial_free_length = initial_free_block->length; + + EXPECT_FALSE(PayloadBuffer::PrimeBitmapAllocator( + &pb, toolbelt::kBitmapRunSize1)); + EXPECT_EQ(0u, pb->bitmaps[0]); + EXPECT_EQ(initial_free_list, pb->free_list); + toolbelt::FreeBlockHeader *restored_free_block = + pb->ToAddress(pb->free_list); + ASSERT_NE(nullptr, restored_free_block); + EXPECT_EQ(initial_free_length, restored_free_block->length); + + free(buffer); +} + +TEST(BufferTest, PrimeBitmapAllocatorRunFailureRollsBackInitialization) { + constexpr size_t kVectorHeaderAllocationSize = + sizeof(VectorHeader) + sizeof(uint64_t); + constexpr size_t kReserveAllocationSize = + 8 * sizeof(BufferOffset) + sizeof(uint64_t); + constexpr size_t kSize = + sizeof(PayloadBuffer) + kVectorHeaderAllocationSize + + kReserveAllocationSize + sizeof(toolbelt::FreeBlockHeader); + char *buffer = (char *)calloc(kSize, 1); + PayloadBuffer *pb = new (buffer) PayloadBuffer(kSize); + + const BufferOffset initial_free_list = pb->free_list; + toolbelt::FreeBlockHeader *initial_free_block = + pb->ToAddress(initial_free_list); + ASSERT_NE(nullptr, initial_free_block); + const uint32_t initial_free_length = initial_free_block->length; + + for (int attempt = 0; attempt < 2; attempt++) { + EXPECT_FALSE(PayloadBuffer::PrimeBitmapAllocator( + &pb, toolbelt::kBitmapRunSize1)); + EXPECT_EQ(0u, pb->bitmaps[0]); + EXPECT_EQ(initial_free_list, pb->free_list); + toolbelt::FreeBlockHeader *restored_free_block = + pb->ToAddress(pb->free_list); + ASSERT_NE(nullptr, restored_free_block); + EXPECT_EQ(initial_free_length, restored_free_block->length); + } + + free(buffer); +} + +TEST(BufferTest, LazyBitmapAllocatorRunFailureRollsBackInitialization) { + constexpr size_t kVectorHeaderAllocationSize = + sizeof(VectorHeader) + sizeof(uint64_t); + constexpr size_t kReserveAllocationSize = + 8 * sizeof(BufferOffset) + sizeof(uint64_t); + constexpr size_t kSize = + sizeof(PayloadBuffer) + kVectorHeaderAllocationSize + + kReserveAllocationSize + sizeof(toolbelt::FreeBlockHeader); + char *buffer = (char *)calloc(kSize, 1); + PayloadBuffer *pb = new (buffer) PayloadBuffer(kSize); + + const BufferOffset initial_free_list = pb->free_list; + toolbelt::FreeBlockHeader *initial_free_block = + pb->ToAddress(initial_free_list); + ASSERT_NE(nullptr, initial_free_block); + const uint32_t initial_free_length = initial_free_block->length; + + for (int attempt = 0; attempt < 2; attempt++) { + EXPECT_EQ(nullptr, + PayloadBuffer::Allocate(&pb, toolbelt::kBitmapRunSize1)); + EXPECT_EQ(0u, pb->bitmaps[0]); + EXPECT_EQ(initial_free_list, pb->free_list); + toolbelt::FreeBlockHeader *restored_free_block = + pb->ToAddress(pb->free_list); + ASSERT_NE(nullptr, restored_free_block); + EXPECT_EQ(initial_free_length, restored_free_block->length); + } + + free(buffer); +} + +TEST(BufferTest, BitmapRunGrowthFailureReclaimsUnappendedRun) { + constexpr size_t kSize = 8192; + char *buffer = (char *)calloc(kSize, 1); + PayloadBuffer *pb = new (buffer) PayloadBuffer(kSize); + + ASSERT_TRUE( + PayloadBuffer::PrimeBitmapAllocator(&pb, toolbelt::kBitmapRunSize1)); + const BufferOffset bitmap_vector_offset = pb->bitmaps[0]; + VectorHeader *hdr = pb->ToAddress(bitmap_vector_offset); + ASSERT_NE(nullptr, hdr); + const size_t bitmap_capacity = + PayloadBuffer::DecodedSize(pb->ToAddress(hdr->data)) / + sizeof(BufferOffset); + ASSERT_GT(bitmap_capacity, hdr->num_elements); + + for (size_t i = hdr->num_elements; i < bitmap_capacity; i++) { + toolbelt::BitMapRun *run = PayloadBuffer::AllocateBitMapRun( + &pb, toolbelt::kBitmapRunSize1, toolbelt::kRunSize1); + ASSERT_NE(nullptr, run); + const BufferOffset run_offset = pb->ToOffset(run); + hdr = pb->ToAddress(bitmap_vector_offset); + ASSERT_TRUE( + PayloadBuffer::VectorPush(&pb, hdr, run_offset, false)); + } + hdr = pb->ToAddress(bitmap_vector_offset); + ASSERT_EQ(bitmap_capacity, hdr->num_elements); + const BufferOffset bitmap_data_offset = hdr->data; + for (size_t i = 0; i < hdr->num_elements; i++) { + toolbelt::BitMapRun *run = + pb->ToAddress(pb->VectorGet(hdr, i)); + ASSERT_NE(nullptr, run); + run->bits = run->num == 32 ? std::numeric_limits::max() + : (uint32_t{1} << run->num) - 1; + run->free = 0; + } + + toolbelt::BitMapRun *probe_run = PayloadBuffer::AllocateBitMapRun( + &pb, toolbelt::kBitmapRunSize1, toolbelt::kRunSize1); + ASSERT_NE(nullptr, probe_run); + const size_t run_allocation_size = + PayloadBuffer::DecodedSize(reinterpret_cast(probe_run)) + + sizeof(uint64_t); + pb->Free(probe_run); + + toolbelt::FreeBlockHeader *free_block = pb->FreeList(); + ASSERT_NE(nullptr, free_block); + ASSERT_EQ(0u, free_block->next); + ASSERT_GT(free_block->length, run_allocation_size + sizeof(uint64_t)); + const size_t drain_size = + free_block->length - run_allocation_size - sizeof(uint64_t); + ASSERT_EQ(0u, drain_size % sizeof(uint64_t)); + ASSERT_NE(nullptr, PayloadBuffer::Allocate(&pb, drain_size, false, false)); + + const BufferOffset initial_free_list = pb->free_list; + free_block = pb->FreeList(); + ASSERT_NE(nullptr, free_block); + ASSERT_EQ(run_allocation_size, free_block->length); + + for (int attempt = 0; attempt < 2; attempt++) { + EXPECT_EQ(nullptr, + PayloadBuffer::Allocate(&pb, toolbelt::kBitmapRunSize1)); + hdr = pb->ToAddress(bitmap_vector_offset); + ASSERT_NE(nullptr, hdr); + EXPECT_EQ(bitmap_capacity, hdr->num_elements); + EXPECT_EQ(bitmap_data_offset, hdr->data); + EXPECT_EQ(initial_free_list, pb->free_list); + free_block = pb->FreeList(); + ASSERT_NE(nullptr, free_block); + EXPECT_EQ(run_allocation_size, free_block->length); + } + + free(buffer); +} + // This performance test compares the performance of the small block allocator // against the regular allocator. It is a best-case test where we are not // stressing the small block allocator by allocating more blocks than a single @@ -618,6 +780,145 @@ TEST(BufferTest, VectorResizeWithResize) { free(buffer); } +TEST(BufferTest, EmptyVectorZeroSizeOperationsSucceed) { + constexpr size_t kSize = 256; + char *buffer = (char *)calloc(kSize, 1); + PayloadBuffer *pb = new (buffer) PayloadBuffer(kSize, false); + + ASSERT_NE(nullptr, + PayloadBuffer::AllocateMainMessage(&pb, sizeof(VectorHeader))); + VectorHeader *hdr = pb->ToAddress(pb->message); + + EXPECT_TRUE(PayloadBuffer::VectorReserve(&pb, hdr, 0, false)); + EXPECT_TRUE(PayloadBuffer::VectorResize(&pb, hdr, 0)); + EXPECT_EQ(0u, hdr->data); + EXPECT_EQ(0u, hdr->num_elements); + + free(buffer); +} + +TEST(BufferTest, VectorPushFixedBufferAllocationFailurePreservesHeader) { + constexpr size_t kSize = 256; + char *buffer = (char *)calloc(kSize, 1); + PayloadBuffer *pb = new (buffer) PayloadBuffer(kSize, false); + + ASSERT_NE(nullptr, + PayloadBuffer::AllocateMainMessage(&pb, sizeof(VectorHeader))); + const BufferOffset msg_offset = pb->message; + VectorHeader *hdr = pb->ToAddress(msg_offset); + + toolbelt::FreeBlockHeader *free_block = + pb->ToAddress(pb->free_list); + ASSERT_NE(nullptr, free_block); + ASSERT_NE(nullptr, + PayloadBuffer::Allocate(&pb, free_block->length - sizeof(uint64_t), + false, false)); + ASSERT_EQ(0u, pb->free_list); + + EXPECT_FALSE( + PayloadBuffer::VectorPush(&pb, hdr, 0x12345678, false)); + hdr = pb->ToAddress(msg_offset); + EXPECT_EQ(0u, hdr->data); + EXPECT_EQ(0u, hdr->num_elements); + + free(buffer); +} + +TEST(BufferTest, VectorPushFixedBufferGrowthFailurePreservesHeader) { + constexpr size_t kSize = 256; + char *buffer = (char *)calloc(kSize, 1); + PayloadBuffer *pb = new (buffer) PayloadBuffer(kSize, false); + + ASSERT_NE(nullptr, + PayloadBuffer::AllocateMainMessage(&pb, sizeof(VectorHeader))); + const BufferOffset msg_offset = pb->message; + VectorHeader *hdr = pb->ToAddress(msg_offset); + ASSERT_TRUE( + PayloadBuffer::VectorPush(&pb, hdr, 0x12345678, false)); + ASSERT_TRUE( + PayloadBuffer::VectorPush(&pb, hdr, 0x9abcdef0, false)); + + hdr = pb->ToAddress(msg_offset); + const BufferOffset original_data = hdr->data; + toolbelt::FreeBlockHeader *free_block = + pb->ToAddress(pb->free_list); + ASSERT_NE(nullptr, free_block); + ASSERT_NE(nullptr, + PayloadBuffer::Allocate(&pb, free_block->length - sizeof(uint64_t), + false, false)); + ASSERT_EQ(0u, pb->free_list); + + EXPECT_FALSE( + PayloadBuffer::VectorPush(&pb, hdr, 0xdeadbeef, false)); + hdr = pb->ToAddress(msg_offset); + EXPECT_EQ(original_data, hdr->data); + ASSERT_EQ(2u, hdr->num_elements); + EXPECT_EQ(0x12345678u, pb->VectorGet(hdr, 0)); + EXPECT_EQ(0x9abcdef0u, pb->VectorGet(hdr, 1)); + + free(buffer); +} + +TEST(BufferTest, VectorReserveFixedBufferFailurePreservesHeader) { + constexpr size_t kSize = 256; + char *buffer = (char *)calloc(kSize, 1); + PayloadBuffer *pb = new (buffer) PayloadBuffer(kSize, false); + + ASSERT_NE(nullptr, + PayloadBuffer::AllocateMainMessage(&pb, sizeof(VectorHeader))); + const BufferOffset msg_offset = pb->message; + VectorHeader *hdr = pb->ToAddress(msg_offset); + + EXPECT_FALSE(PayloadBuffer::VectorReserve(&pb, hdr, kSize, false)); + EXPECT_EQ(0u, hdr->data); + EXPECT_EQ(0u, hdr->num_elements); + + ASSERT_TRUE( + PayloadBuffer::VectorPush(&pb, hdr, 0x12345678, false)); + hdr = pb->ToAddress(msg_offset); + const BufferOffset original_data = hdr->data; + + EXPECT_FALSE(PayloadBuffer::VectorReserve(&pb, hdr, kSize, false)); + hdr = pb->ToAddress(msg_offset); + EXPECT_EQ(original_data, hdr->data); + ASSERT_EQ(1u, hdr->num_elements); + EXPECT_EQ(0x12345678u, pb->VectorGet(hdr, 0)); + + free(buffer); +} + +TEST(BufferTest, VectorResizeFixedBufferFailurePreservesHeader) { + constexpr size_t kSize = 256; + char *buffer = (char *)calloc(kSize, 1); + PayloadBuffer *pb = new (buffer) PayloadBuffer(kSize, false); + + ASSERT_NE(nullptr, + PayloadBuffer::AllocateMainMessage(&pb, sizeof(VectorHeader))); + const BufferOffset msg_offset = pb->message; + VectorHeader *hdr = pb->ToAddress(msg_offset); + + EXPECT_FALSE(PayloadBuffer::VectorResize(&pb, hdr, kSize)); + EXPECT_EQ(0u, hdr->data); + EXPECT_EQ(0u, hdr->num_elements); + + ASSERT_TRUE(PayloadBuffer::VectorResize(&pb, hdr, 2)); + hdr = pb->ToAddress(msg_offset); + const BufferOffset original_data = hdr->data; + uint32_t *values = pb->ToAddress(original_data, 2 * sizeof(uint32_t)); + ASSERT_NE(nullptr, values); + values[0] = 0x12345678; + values[1] = 0x9abcdef0; + + EXPECT_FALSE(PayloadBuffer::VectorResize(&pb, hdr, kSize)); + hdr = pb->ToAddress(msg_offset); + EXPECT_EQ(original_data, hdr->data); + ASSERT_EQ(2u, hdr->num_elements); + EXPECT_EQ(0x12345678u, pb->VectorGet(hdr, 0)); + EXPECT_EQ(0x9abcdef0u, pb->VectorGet(hdr, 1)); + + free(buffer); +} + TEST(BufferTest, Resizeable) { char *buffer = (char *)calloc(1, 512); bool resized = false; @@ -668,6 +969,321 @@ TEST(BufferTest, Resizeable) { free(pb); } +TEST(BufferTest, ToAddressRejectsTypedReadPastEnd) { + constexpr size_t kSize = 4096; + char *buffer = (char *)calloc(kSize, 1); + PayloadBuffer *pb = new (buffer) PayloadBuffer(kSize); + + EXPECT_EQ(pb->ToAddress(kSize - 2), nullptr); + EXPECT_NE(pb->ToAddress(kSize - sizeof(uint32_t)), nullptr); + + free(buffer); +} + +TEST(BufferTest, StringHelpersRejectLengthHeaderPastEnd) { + constexpr size_t kSize = 4093; + char *buffer = (char *)calloc(kSize, 1); + PayloadBuffer *pb = new (buffer) PayloadBuffer(kSize); + + toolbelt::StringHeader header = static_cast(4092); + + EXPECT_EQ(pb->StringSize(&header), 0u); + EXPECT_EQ(pb->GetString(&header), ""); + + free(buffer); +} + +TEST(BufferTest, StringSizeRejectsBodyPastEnd) { + constexpr size_t kSize = 4096; + char *buffer = (char *)calloc(kSize, 1); + PayloadBuffer *pb = new (buffer) PayloadBuffer(kSize); + + toolbelt::StringHeader header = + static_cast(kSize - sizeof(uint32_t)); + uint32_t declared = 1; + memcpy(buffer + header, &declared, sizeof(declared)); + + EXPECT_EQ(pb->StringData(&header), nullptr); + EXPECT_EQ(pb->StringSize(&header), 0u); + + free(buffer); +} + +TEST(BufferTest, EmptyStringAtBufferTailAccepted) { + constexpr size_t kSize = 4096; + char *buffer = (char *)calloc(kSize, 1); + PayloadBuffer *pb = new (buffer) PayloadBuffer(kSize); + + toolbelt::StringHeader header = + static_cast(kSize - sizeof(uint32_t)); + uint32_t declared = 0; + memcpy(buffer + header, &declared, sizeof(declared)); + + EXPECT_NE(pb->StringData(&header), nullptr); + EXPECT_EQ(pb->StringSize(&header), 0u); + EXPECT_EQ(pb->GetString(&header), ""); + EXPECT_EQ(pb->GetStringView(&header), ""); + + free(buffer); +} + +TEST(BufferTest, StringWithinBoundsAcceptsValidString) { + char *buffer = (char *)calloc(4096, 1); + PayloadBuffer *pb = new (buffer) PayloadBuffer(4096); + + PayloadBuffer::AllocateMainMessage(&pb, 32); + BufferOffset offset = pb->ToOffset(pb->ToAddress(pb->message)); + PayloadBuffer::SetString(&pb, std::string("foobar"), offset); + + EXPECT_TRUE(pb->StringWithinBounds(offset)); + + free(buffer); +} + +TEST(BufferTest, StringWithinBoundsAcceptsUnsetString) { + char *buffer = (char *)calloc(4096, 1); + PayloadBuffer *pb = new (buffer) PayloadBuffer(4096); + + PayloadBuffer::AllocateMainMessage(&pb, sizeof(BufferOffset)); + + EXPECT_TRUE(pb->StringWithinBounds(pb->message)); + + free(buffer); +} + +TEST(BufferTest, SetStringFixedBufferFailurePreservesHeader) { + constexpr uint32_t kBufferSize = 4096; + char *buffer = (char *)calloc(kBufferSize, 1); + PayloadBuffer *pb = + new (buffer) PayloadBuffer(kBufferSize, /*bitmap_allocator=*/false); + PayloadBuffer::AllocateMainMessage(&pb, sizeof(toolbelt::StringHeader)); + + const uint32_t free_len = pb->FreeList()->length; + constexpr uint32_t kRemainingBytes = 2 * sizeof(uint64_t); + ASSERT_GT(free_len, kRemainingBytes + sizeof(uint64_t)); + const uint32_t drain = free_len - kRemainingBytes - sizeof(uint64_t); + ASSERT_NE(PayloadBuffer::Allocate(&pb, drain), nullptr); + + EXPECT_EQ(PayloadBuffer::SetString(&pb, "too large", pb->message), nullptr); + EXPECT_EQ(*pb->ToAddress(pb->message), + BufferOffset(0)); + EXPECT_TRUE(PayloadBuffer::AllocateString(&pb, 9, pb->message).empty()); + EXPECT_EQ(*pb->ToAddress(pb->message), + BufferOffset(0)); + + free(buffer); +} + +TEST(BufferTest, StringReallocFixedBufferFailurePreservesValue) { + constexpr uint32_t kBufferSize = 4096; + char *buffer = (char *)calloc(kBufferSize, 1); + PayloadBuffer *pb = + new (buffer) PayloadBuffer(kBufferSize, /*bitmap_allocator=*/false); + PayloadBuffer::AllocateMainMessage(&pb, sizeof(toolbelt::StringHeader)); + ASSERT_NE(PayloadBuffer::SetString(&pb, "x", pb->message), nullptr); + + const BufferOffset original_offset = + *pb->ToAddress(pb->message); + ASSERT_NE(original_offset, BufferOffset(0)); + + const uint32_t free_len = pb->FreeList()->length; + constexpr uint32_t kRemainingBytes = 2 * sizeof(uint64_t); + ASSERT_GT(free_len, kRemainingBytes + sizeof(uint64_t)); + const uint32_t drain = free_len - kRemainingBytes - sizeof(uint64_t); + ASSERT_NE(PayloadBuffer::Allocate(&pb, drain), nullptr); + + EXPECT_EQ(PayloadBuffer::SetString(&pb, "too large", pb->message), nullptr); + EXPECT_EQ(*pb->ToAddress(pb->message), + original_offset); + EXPECT_EQ(pb->GetString(pb->ToAddress(pb->message)), + "x"); + EXPECT_TRUE(PayloadBuffer::AllocateString(&pb, 9, pb->message).empty()); + EXPECT_EQ(*pb->ToAddress(pb->message), + original_offset); + EXPECT_EQ(pb->GetString(pb->ToAddress(pb->message)), + "x"); + + free(buffer); +} + +TEST(BufferTest, StringWithinBoundsRejectsNullHeader) { + char *buffer = (char *)calloc(4096, 1); + PayloadBuffer *pb = new (buffer) PayloadBuffer(4096); + + EXPECT_FALSE( + pb->StringWithinBounds(static_cast(nullptr))); + + free(buffer); +} + +TEST(BufferTest, StringReadersReturnEmptyForNullHeader) { + char *buffer = (char *)calloc(4096, 1); + PayloadBuffer *pb = new (buffer) PayloadBuffer(4096); + + const toolbelt::StringHeader *header = nullptr; + + EXPECT_EQ(pb->GetString(header), ""); + EXPECT_EQ(pb->GetStringView(header), ""); + EXPECT_EQ(pb->StringSize(header), 0u); + EXPECT_EQ(pb->StringData(header), nullptr); + + free(buffer); +} + +TEST(BufferTest, StringReadersRejectHeaderOffsetPastEnd) { + constexpr size_t kSize = 4096; + char *buffer = (char *)calloc(kSize, 1); + PayloadBuffer *pb = new (buffer) PayloadBuffer(kSize); + + const BufferOffset straddling = static_cast(kSize - 2); + + EXPECT_EQ(pb->GetString(straddling), ""); + EXPECT_EQ(pb->GetStringView(straddling), ""); + EXPECT_EQ(pb->StringSize(straddling), 0u); + EXPECT_EQ(pb->StringData(straddling), nullptr); + + const BufferOffset unset = static_cast(0); + + EXPECT_EQ(pb->GetString(unset), ""); + EXPECT_EQ(pb->GetStringView(unset), ""); + EXPECT_EQ(pb->StringSize(unset), 0u); + EXPECT_EQ(pb->StringData(unset), nullptr); + + free(buffer); +} + +TEST(BufferTest, StringWritersRejectHeaderOffsetPastEnd) { + constexpr size_t kSize = 4096; + char *buffer = (char *)calloc(kSize, 1); + PayloadBuffer *pb = + new (buffer) PayloadBuffer(kSize, /*bitmap_allocator=*/false); + const uint32_t free_len = pb->FreeList()->length; + + const BufferOffset straddling = static_cast(kSize - 2); + + EXPECT_EQ(PayloadBuffer::SetString(&pb, "x", 1, straddling), nullptr); + EXPECT_TRUE(PayloadBuffer::AllocateString(&pb, 1, straddling).empty()); + PayloadBuffer::ClearString(&pb, straddling); + + const BufferOffset unset = static_cast(0); + + EXPECT_EQ(PayloadBuffer::SetString(&pb, "x", 1, unset), nullptr); + EXPECT_TRUE(PayloadBuffer::AllocateString(&pb, 1, unset).empty()); + PayloadBuffer::ClearString(&pb, unset); + + EXPECT_EQ(pb->FreeList()->length, free_len); + + free(buffer); +} + +TEST(BufferTest, AllocateStringReturnsWritableSpanAndStoresOffset) { + constexpr size_t kSize = 4096; + char *buffer = (char *)calloc(kSize, 1); + PayloadBuffer *pb = + new (buffer) PayloadBuffer(kSize, /*bitmap_allocator=*/false); + PayloadBuffer::AllocateMainMessage(&pb, sizeof(toolbelt::StringHeader)); + + absl::Span str = PayloadBuffer::AllocateString(&pb, 3, pb->message); + ASSERT_EQ(str.size(), 3u); + EXPECT_NE(*pb->ToAddress(pb->message), + BufferOffset(0)); + + memcpy(str.data(), "abc", 3); + EXPECT_EQ(pb->GetString(pb->message), "abc"); + + absl::Span grown = PayloadBuffer::AllocateString(&pb, 5, pb->message); + ASSERT_EQ(grown.size(), 5u); + + memcpy(grown.data(), "abcde", 5); + EXPECT_EQ(pb->GetString(pb->message), "abcde"); + + free(buffer); +} + +TEST(BufferTest, StringWithinBoundsRejectsBodyOffsetPastEnd) { + constexpr size_t kSize = 4096; + char *buffer = (char *)calloc(kSize, 1); + PayloadBuffer *pb = new (buffer) PayloadBuffer(kSize); + + toolbelt::StringHeader header = static_cast(kSize - 2); + + EXPECT_FALSE(pb->StringWithinBounds(&header)); + + free(buffer); +} + +TEST(BufferTest, StringWithinBoundsRejectsBodyPastEnd) { + constexpr size_t kSize = 4096; + char *buffer = (char *)calloc(kSize, 1); + PayloadBuffer *pb = new (buffer) PayloadBuffer(kSize); + + toolbelt::StringHeader header = + static_cast(kSize - sizeof(uint32_t)); + uint32_t declared = 1; + memcpy(buffer + header, &declared, sizeof(declared)); + + EXPECT_FALSE(pb->StringWithinBounds(&header)); + + free(buffer); +} + +TEST(BufferTest, VectorGetRejectsIndexPastEnd) { + constexpr size_t kSize = 4096; + char *buffer = (char *)calloc(kSize, 1); + PayloadBuffer *pb = new (buffer) PayloadBuffer(kSize); + + VectorHeader hdr; + hdr.num_elements = 2; + hdr.data = static_cast(kSize - sizeof(uint32_t)); + + EXPECT_EQ(pb->VectorGet(&hdr, 1), 0u); + + free(buffer); +} + +TEST(BufferTest, ToOffsetAndToAddressAgreeOnTrailingExtent) { + constexpr size_t kSize = 4096; + char *buffer = (char *)calloc(kSize, 1); + PayloadBuffer *pb = new (buffer) PayloadBuffer(kSize); + + uint32_t *p = + reinterpret_cast(reinterpret_cast(pb) + (kSize - 2)); + + EXPECT_EQ(pb->ToAddress(kSize - 2), nullptr); + EXPECT_EQ(pb->ToOffset(p), 0u); + + free(buffer); +} + +TEST(BufferTest, ToAddressVoidStartOnlyBoundary) { + constexpr size_t kSize = 4096; + char *buffer = (char *)calloc(kSize, 1); + PayloadBuffer *pb = new (buffer) PayloadBuffer(kSize); + + EXPECT_NE(pb->ToAddress(kSize - 1), nullptr); + EXPECT_EQ(pb->ToAddress(kSize), nullptr); + + free(buffer); +} + +TEST(BufferTest, ToAddressAndToOffsetRejectOutOfRangeInputs) { + constexpr size_t kSize = 4096; + char *buffer = (char *)calloc(kSize, 1); + PayloadBuffer *pb = new (buffer) PayloadBuffer(kSize); + char *other_buffer = (char *)calloc(kSize, 1); + + const BufferOffset far_offset = std::numeric_limits::max(); + EXPECT_EQ(pb->ToAddress(far_offset), nullptr); + EXPECT_EQ(pb->ToAddress(far_offset), nullptr); + + const uint32_t *foreign_address = + reinterpret_cast(other_buffer); + EXPECT_EQ(pb->ToOffset(foreign_address), 0u); + + free(other_buffer); + free(buffer); +} + int main(int argc, char **argv) { testing::InitGoogleTest(&argc, argv);