diff --git a/ARENA.md b/ARENA.md index 3409f98..b41cdf2 100644 --- a/ARENA.md +++ b/ARENA.md @@ -259,8 +259,19 @@ Setting the limit to zero disables external-buffer caching. Reducing the limit immediately trims the cache. A larger limit can improve throughput for repeated large payloads but may increase retained memory and RSS. -The arena also maintains size classes for commonly-sized owned SDS values. -Callers do not need to select a class. +The arena also maintains size classes for owned SDS values up to 1 KiB. +Callers do not need to select a class. Classes below 128 bytes grow in +alignment-sized steps of the complete allocation, so short strings such as map +keys use at most one alignment unit more than they need; larger classes grow +by about 25 percent. A released string is recycled for the next string of the +same class. `cfl_sds_alloc()` reports the class capacity, which can exceed the +requested length. + +Variants, kvpairs, key/value lists, arrays, and SDS headers are aligned for the +pointers, sizes, 64-bit integers, and doubles they contain. This can be less +strict than the raw allocation alignment, which also covers `long double`, so +CFL objects pack more densely in arena chunks. Raw allocations keep the public +alignment guarantee. ## Memory statistics diff --git a/CHANGELOG.md b/CHANGELOG.md index 37b1f89..30e6583 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,15 @@ This file records the notable changes in each CFL release. - Added public request-lifetime arena allocation, duplication helpers, allocator callbacks, and optional bounded geometric chunk growth. +- Reduced arena memory for CFL object graphs: objects use their natural + alignment instead of the raw allocation alignment, and owned SDS strings use + finer size classes. Decoded log records use 27 to 31 percent less arena + memory. Owned SDS blocks are no longer zeroed before initialization. +- Added a bump-allocation fast path for the current arena chunk. +- Fixed the alignment of external arena allocations, which returned storage + at a misaligned offset for large owned SDS strings. +- Added the `cfl-benchmark-arena-records` benchmark for decoded record + batches. ## 1.0.0 - 2026-07-11 diff --git a/benchmarks/CMakeLists.txt b/benchmarks/CMakeLists.txt index f1148df..0a7aefa 100644 --- a/benchmarks/CMakeLists.txt +++ b/benchmarks/CMakeLists.txt @@ -3,3 +3,6 @@ target_link_libraries(cfl-benchmark-variant-arena cfl-static) add_executable(cfl-benchmark-variant-mutable variant_mutable.c) target_link_libraries(cfl-benchmark-variant-mutable cfl-static) + +add_executable(cfl-benchmark-arena-records records.c) +target_link_libraries(cfl-benchmark-arena-records cfl-static) diff --git a/benchmarks/README.md b/benchmarks/README.md index d4cb58f..3697ad8 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -88,3 +88,20 @@ document is still alive. Compare them with `arena_initial_used` and `arena_used`. `max_rss_kb` is process-level high-water RSS and can remain the same for small workloads because it is page-granular and includes executable, library, and allocator-retained pages. + +## Decoded record batches + +The records benchmark builds batches of small log records, each with short +keys, a nested map, an array, and string, integer, and double values, as +produced by decoding a buffer of telemetry records. Arena batches are reset +between iterations; heap batches destroy every record: + +```sh +build-bench/benchmarks/cfl-benchmark-arena-records arena 512 3000 +build-bench/benchmarks/cfl-benchmark-arena-records heap 512 3000 +``` + +Optional arguments select `referenced` or `owned` values, enable record edits +(`1` removes, inserts, and renames keys), and set the arena chunk size. The +tool reports nanoseconds per record and, for arenas, used and reserved bytes +per record. diff --git a/benchmarks/records.c b/benchmarks/records.c new file mode 100644 index 0000000..4c2cd85 --- /dev/null +++ b/benchmarks/records.c @@ -0,0 +1,195 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* + * Decoded telemetry batch benchmark: every batch holds many small records + * with short keys, a nested map and an array, as produced by decoding a + * buffer of log records. Values can reference the source buffer or be owned + * copies, and records can be edited after construction. + */ + +#include +#include +#include +#include + +#if !defined(CFL_SYSTEM_WINDOWS) +#include +#endif + +#include + +struct batch_options { + size_t records; + int referenced; + int mutate; +}; + +static int insert_string(struct cfl_kvlist *list, char *key, char *value, + int referenced) +{ + return cfl_kvlist_insert_string_s(list, key, strlen(key), value, + strlen(value), referenced); +} + +static struct cfl_variant *build_record(struct cfl_arena *arena, size_t index, + struct batch_options *options) +{ + char message[64]; + struct cfl_kvlist *record; + struct cfl_kvlist *http; + struct cfl_array *tags; + struct cfl_variant *root; + + snprintf(message, sizeof(message), "request %zu completed", index); + + record = cfl_kvlist_create_in(arena); + http = cfl_kvlist_create_in(arena); + tags = cfl_array_create_in(arena, 2); + if (record == NULL || http == NULL || tags == NULL) { + return NULL; + } + + /* referenced values must outlive the record, keep them static */ + if (insert_string(record, "msg", options->referenced ? + "request completed" : message, + options->referenced) != 0 || + insert_string(record, "level", "info", options->referenced) != 0 || + insert_string(record, "user", "u-1234", options->referenced) != 0 || + insert_string(record, "password", "secret", options->referenced) != 0 || + insert_string(http, "path", "/api/v1/items", options->referenced) != 0 || + insert_string(http, "method", "GET", options->referenced) != 0 || + cfl_kvlist_insert_int64(http, "status", 200) != 0 || + cfl_kvlist_insert_kvlist(record, "http", http) != 0 || + cfl_kvlist_insert_double(record, "duration_ms", 12.5) != 0 || + cfl_array_append_string_s(tags, "edge", 4, options->referenced) != 0 || + cfl_array_append_string_s(tags, "eu-west-1", 9, + options->referenced) != 0 || + cfl_kvlist_insert_array(record, "tags", tags) != 0) { + return NULL; + } + + if (options->mutate) { + if (cfl_kvlist_remove(record, "password") != CFL_TRUE || + cfl_kvlist_insert_int64(record, "x", 1) != 0 || + cfl_kvlist_rename_s(record, "user", 4, "user.id", 7) != 0) { + return NULL; + } + } + + root = cfl_variant_create_from_kvlist_in(arena, record); + return root; +} + +static int run_batch(struct cfl_arena *arena, struct cfl_variant **roots, + struct batch_options *options) +{ + size_t index; + int result; + + result = 0; + for (index = 0; index < options->records; index++) { + roots[index] = build_record(arena, index, options); + if (roots[index] == NULL) { + result = -1; + break; + } + } + + /* arena records are released by the reset, heap ones one by one */ + if (arena == NULL) { + for (index = 0; index < options->records; index++) { + if (roots[index] != NULL) { + cfl_variant_destroy(roots[index]); + } + } + } + + return result; +} + +int main(int argc, char **argv) +{ + const char *mode; + size_t batches; + size_t batch; + size_t chunk_size; + size_t reserved; + size_t used; + uint64_t start; + uint64_t elapsed; + struct cfl_arena *arena; + struct cfl_variant **roots; + struct batch_options options; +#if !defined(CFL_SYSTEM_WINDOWS) + struct rusage usage; +#endif + + mode = argc > 1 ? argv[1] : "arena"; + options.records = argc > 2 ? strtoull(argv[2], NULL, 10) : 512; + batches = argc > 3 ? strtoull(argv[3], NULL, 10) : 2000; + options.referenced = argc > 4 ? strcmp(argv[4], "owned") != 0 : CFL_TRUE; + options.mutate = argc > 5 ? atoi(argv[5]) : 0; + chunk_size = argc > 6 ? strtoull(argv[6], NULL, 10) : 16384; + + if ((strcmp(mode, "arena") != 0 && strcmp(mode, "heap") != 0) || + options.records == 0) { + fprintf(stderr, + "usage: %s arena|heap [records] [batches] " + "[referenced|owned] [mutate] [chunk-size]\n", argv[0]); + return EXIT_FAILURE; + } + + roots = calloc(options.records, sizeof(struct cfl_variant *)); + if (roots == NULL) { + return EXIT_FAILURE; + } + + arena = NULL; + if (strcmp(mode, "arena") == 0) { + arena = cfl_arena_create(chunk_size); + if (arena == NULL) { + free(roots); + return EXIT_FAILURE; + } + } + + reserved = 0; + used = 0; + start = cfl_time_now(); + for (batch = 0; batch < batches; batch++) { + if (run_batch(arena, roots, &options) != 0) { + fprintf(stderr, "batch construction failed\n"); + cfl_arena_destroy(arena); + free(roots); + return EXIT_FAILURE; + } + if (arena != NULL) { + reserved = cfl_arena_bytes_reserved(arena); + used = cfl_arena_bytes_used(arena); + cfl_arena_reset(arena); + } + } + elapsed = cfl_time_now() - start; + + printf("mode=%s records=%zu batches=%zu values=%s mutate=%d " + "elapsed_ns=%llu ns_per_record=%.2f", + mode, options.records, batches, + options.referenced ? "referenced" : "owned", options.mutate, + (unsigned long long) elapsed, + (double) elapsed / (double) (batches * options.records)); + if (arena != NULL) { + printf(" arena_reserved=%zu arena_used=%zu used_per_record=%.1f " + "reserved_per_record=%.1f", reserved, used, + (double) used / (double) options.records, + (double) reserved / (double) options.records); + } +#if !defined(CFL_SYSTEM_WINDOWS) + getrusage(RUSAGE_SELF, &usage); + printf(" max_rss_kb=%ld", usage.ru_maxrss); +#endif + putchar('\n'); + + cfl_arena_destroy(arena); + free(roots); + return EXIT_SUCCESS; +} diff --git a/src/cfl_arena.c b/src/cfl_arena.c index 62fa165..a3c80e8 100644 --- a/src/cfl_arena.c +++ b/src/cfl_arena.c @@ -5,11 +5,15 @@ #include #include +#include +#include +#include #include "cfl_arena_internal.h" #define CFL_ARENA_DEFAULT_CHUNK_SIZE 8192 -#define CFL_ARENA_SDS_CLASS_COUNT 6 +#define CFL_ARENA_SDS_SMALL_PAYLOAD 128 +#define CFL_ARENA_SDS_MEDIUM_CLASS_COUNT 10 #define CFL_ARENA_EXTERNAL_CLASS_COUNT 24 #define CFL_ARENA_EXTERNAL_EXACT_CLASS UINT8_MAX @@ -24,6 +28,72 @@ struct cfl_arena_alignment_probe { union cfl_arena_max_align value; }; +/* + * CFL objects only hold pointers, sizes, integers and doubles, so they are + * packed with the strictest of those alignments instead of the raw allocation + * alignment, which can be larger because of long double. + */ +union cfl_arena_object_align { + void *pointer_value; + size_t size_value; + uint64_t uint64_value; + double double_value; +}; + +struct cfl_arena_object_alignment_probe { + char byte; + union cfl_arena_object_align value; +}; + +#define CFL_ARENA_MAX_ALIGNMENT \ + offsetof(struct cfl_arena_alignment_probe, value) +#define CFL_ARENA_OBJECT_ALIGNMENT \ + offsetof(struct cfl_arena_object_alignment_probe, value) + +/* small SDS classes cover payloads below 128 bytes in alignment steps */ +#define CFL_ARENA_SDS_SMALL_CLASS_COUNT \ + (CFL_ARENA_SDS_SMALL_PAYLOAD / CFL_ARENA_OBJECT_ALIGNMENT) +#define CFL_ARENA_SDS_CLASS_COUNT \ + (CFL_ARENA_SDS_SMALL_CLASS_COUNT + CFL_ARENA_SDS_MEDIUM_CLASS_COUNT) + +struct cfl_arena_variant_probe { + char byte; + struct cfl_variant value; +}; + +struct cfl_arena_kvpair_probe { + char byte; + struct cfl_kvpair value; +}; + +struct cfl_arena_kvlist_probe { + char byte; + struct cfl_kvlist value; +}; + +struct cfl_arena_array_probe { + char byte; + struct cfl_array value; +}; + +/* compile time checks: every object type fits the object alignment */ +typedef char cfl_arena_check_object_alignment[ + (CFL_ARENA_MAX_ALIGNMENT % CFL_ARENA_OBJECT_ALIGNMENT == 0 && + (CFL_ARENA_OBJECT_ALIGNMENT & (CFL_ARENA_OBJECT_ALIGNMENT - 1)) == 0 && + CFL_ARENA_SDS_CLASS_COUNT < UINT8_MAX) ? 1 : -1]; +typedef char cfl_arena_check_variant_alignment[ + (CFL_ARENA_OBJECT_ALIGNMENT % + offsetof(struct cfl_arena_variant_probe, value) == 0) ? 1 : -1]; +typedef char cfl_arena_check_kvpair_alignment[ + (CFL_ARENA_OBJECT_ALIGNMENT % + offsetof(struct cfl_arena_kvpair_probe, value) == 0) ? 1 : -1]; +typedef char cfl_arena_check_kvlist_alignment[ + (CFL_ARENA_OBJECT_ALIGNMENT % + offsetof(struct cfl_arena_kvlist_probe, value) == 0) ? 1 : -1]; +typedef char cfl_arena_check_array_alignment[ + (CFL_ARENA_OBJECT_ALIGNMENT % + offsetof(struct cfl_arena_array_probe, value) == 0) ? 1 : -1]; + struct cfl_arena_chunk { struct cfl_arena_chunk *next; size_t capacity; @@ -37,6 +107,7 @@ struct cfl_arena_external { struct cfl_arena_external *previous; size_t size; uint8_t allocation_class; + union cfl_arena_max_align alignment; unsigned char data[]; }; @@ -276,24 +347,20 @@ void cfl_arena_reset(struct cfl_arena *arena) memset(arena->free_sds, 0, sizeof(arena->free_sds)); } -void *cfl_arena_malloc(struct cfl_arena *arena, size_t size) +/* walks the remaining chunks, then reserves a new one */ +static void *arena_allocate_slow(struct cfl_arena *arena, size_t size, + size_t alignment) { struct cfl_arena_chunk *chunk; - size_t alignment; size_t offset; size_t remainder; size_t capacity; void *result; - if (arena == NULL || size == 0) { - return NULL; - } - - alignment = offsetof(struct cfl_arena_alignment_probe, value); chunk = arena->current; while (chunk != NULL) { offset = chunk->used; - remainder = offset % alignment; + remainder = offset & (alignment - 1); if (remainder != 0) { if (offset > SIZE_MAX - (alignment - remainder)) { return NULL; @@ -346,6 +413,62 @@ void *cfl_arena_malloc(struct cfl_arena *arena, size_t size) return chunk->data; } +/* bump allocation from the current chunk, the common case */ +static inline void *arena_allocate(struct cfl_arena *arena, size_t size, + size_t alignment) +{ + struct cfl_arena_chunk *chunk; + size_t offset; + void *result; + + if (arena == NULL || size == 0) { + return NULL; + } + + chunk = arena->current; + if (chunk != NULL) { + offset = (chunk->used + alignment - 1) & ~(alignment - 1); + if (offset >= chunk->used && offset <= chunk->capacity && + size <= chunk->capacity - offset) { + result = &chunk->data[offset]; + chunk->used = offset + size; + arena->bytes_used += size; + return result; + } + } + + return arena_allocate_slow(arena, size, alignment); +} + +void *cfl_arena_malloc(struct cfl_arena *arena, size_t size) +{ + return arena_allocate(arena, size, CFL_ARENA_MAX_ALIGNMENT); +} + +void *cfl_arena_alloc_object(struct cfl_arena *arena, size_t size) +{ + return arena_allocate(arena, size, CFL_ARENA_OBJECT_ALIGNMENT); +} + +void *cfl_arena_calloc_object(struct cfl_arena *arena, + size_t count, size_t size) +{ + void *result; + size_t total; + + if (count != 0 && size > SIZE_MAX / count) { + return NULL; + } + + total = count * size; + result = arena_allocate(arena, total, CFL_ARENA_OBJECT_ALIGNMENT); + if (result != NULL) { + memset(result, 0, total); + } + + return result; +} + void *cfl_arena_alloc(struct cfl_arena *arena, size_t size) { return cfl_arena_malloc(arena, size); @@ -604,7 +727,7 @@ void cfl_arena_free_external(struct cfl_arena *arena, } static void *arena_reusable_alloc(struct cfl_arena *arena, - void **free_list, size_t size) + void **free_list, size_t size, int zero) { void *result; @@ -616,11 +739,19 @@ static void *arena_reusable_alloc(struct cfl_arena *arena, result = *free_list; *free_list = *((void **) result); arena->bytes_used += size; + } + else { + result = arena_allocate(arena, size, CFL_ARENA_OBJECT_ALIGNMENT); + if (result == NULL) { + return NULL; + } + } + + if (zero) { memset(result, 0, size); - return result; } - return cfl_arena_calloc(arena, 1, size); + return result; } static void arena_reusable_free(struct cfl_arena *arena, @@ -638,7 +769,7 @@ static void arena_reusable_free(struct cfl_arena *arena, void *cfl_arena_alloc_variant(struct cfl_arena *arena, size_t size) { - return arena_reusable_alloc(arena, &arena->free_variants, size); + return arena_reusable_alloc(arena, &arena->free_variants, size, CFL_TRUE); } void cfl_arena_free_variant(struct cfl_arena *arena, @@ -650,7 +781,7 @@ void cfl_arena_free_variant(struct cfl_arena *arena, void *cfl_arena_alloc_kvpair(struct cfl_arena *arena, size_t size) { - return arena_reusable_alloc(arena, &arena->free_kvpairs, size); + return arena_reusable_alloc(arena, &arena->free_kvpairs, size, CFL_TRUE); } void cfl_arena_free_kvpair(struct cfl_arena *arena, @@ -659,32 +790,80 @@ void cfl_arena_free_kvpair(struct cfl_arena *arena, arena_reusable_free(arena, &arena->free_kvpairs, pointer, size); } +/* + * Owned SDS strings are recycled through size classes. Small classes grow in + * object alignment steps of the complete allocation up to about 128 bytes of + * payload, so short strings such as map keys waste less than one alignment + * unit. Medium classes grow by roughly 25 percent up to the largest pooled + * payload. + */ +static int arena_sds_class(size_t payload_size, size_t overhead_size, + size_t *index, size_t *payload_capacity) +{ + static const size_t medium_payloads[CFL_ARENA_SDS_MEDIUM_CLASS_COUNT] = { + 160, 200, 248, 312, 392, 488, 608, 760, 952, CFL_ARENA_SDS_MAX_PAYLOAD + }; + size_t alignment; + size_t smallest; + size_t total; + size_t class_index; + + alignment = CFL_ARENA_OBJECT_ALIGNMENT; + if (payload_size > CFL_ARENA_SDS_MAX_PAYLOAD || + overhead_size > SIZE_MAX - CFL_ARENA_SDS_MAX_PAYLOAD - 1 - alignment) { + return -1; + } + + /* smallest allocation holds the overhead, the terminator and one byte */ + smallest = (overhead_size + 2 + alignment - 1) & ~(alignment - 1); + total = (overhead_size + payload_size + 1 + alignment - 1) & + ~(alignment - 1); + if (total < smallest) { + total = smallest; + } + + if (total <= smallest + (CFL_ARENA_SDS_SMALL_CLASS_COUNT - 1) * alignment) { + *index = (total - smallest) / alignment; + *payload_capacity = total - overhead_size - 1; + return 0; + } + + for (class_index = 0; class_index < CFL_ARENA_SDS_MEDIUM_CLASS_COUNT; + class_index++) { + total = (overhead_size + medium_payloads[class_index] + 1 + + alignment - 1) & ~(alignment - 1); + if (payload_size <= total - overhead_size - 1) { + *index = CFL_ARENA_SDS_SMALL_CLASS_COUNT + class_index; + *payload_capacity = total - overhead_size - 1; + return 0; + } + } + + return -1; +} + void *cfl_arena_alloc_sds(struct cfl_arena *arena, size_t payload_size, size_t overhead_size, uint8_t *allocation_class, size_t *payload_capacity) { - static const size_t class_sizes[CFL_ARENA_SDS_CLASS_COUNT] = { - 32, 64, 128, 256, 512, 1024 - }; size_t index; if (arena == NULL || allocation_class == NULL || payload_capacity == NULL) { return NULL; } - for (index = 0; index < CFL_ARENA_SDS_CLASS_COUNT; index++) { - if (payload_size <= class_sizes[index]) { - *allocation_class = (uint8_t) (index + 1); - *payload_capacity = class_sizes[index]; - return arena_reusable_alloc(arena, &arena->free_sds[index], - overhead_size + class_sizes[index] + 1); - } + if (arena_sds_class(payload_size, overhead_size, &index, + payload_capacity) != 0) { + *allocation_class = 0; + *payload_capacity = payload_size; + return NULL; } - *allocation_class = 0; - *payload_capacity = payload_size; - return NULL; + /* the SDS layer initializes the headers and the payload it uses */ + *allocation_class = (uint8_t) (index + 1); + return arena_reusable_alloc(arena, &arena->free_sds[index], + overhead_size + *payload_capacity + 1, CFL_FALSE); } void cfl_arena_free_sds(struct cfl_arena *arena, diff --git a/src/cfl_arena_internal.h b/src/cfl_arena_internal.h index e41fe3a..5959a17 100644 --- a/src/cfl_arena_internal.h +++ b/src/cfl_arena_internal.h @@ -2,10 +2,23 @@ #define CFL_ARENA_INTERNAL_H #include +#include #include +/* largest owned SDS payload recycled through arena size classes */ +#define CFL_ARENA_SDS_MAX_PAYLOAD 1024 + void *cfl_arena_alloc(struct cfl_arena *arena, size_t size); + +/* + * CFL object allocations: aligned for the pointers, sizes, integers and + * doubles held by variants, kvpairs, kvlists, arrays and SDS headers, which + * can be less strict than the public raw allocation alignment. + */ +void *cfl_arena_alloc_object(struct cfl_arena *arena, size_t size); +void *cfl_arena_calloc_object(struct cfl_arena *arena, + size_t count, size_t size); void *cfl_arena_alloc_external(struct cfl_arena *arena, size_t size); void cfl_arena_free_external(struct cfl_arena *arena, diff --git a/src/cfl_array.c b/src/cfl_array.c index bac55ab..d7b0a99 100644 --- a/src/cfl_array.c +++ b/src/cfl_array.c @@ -49,7 +49,7 @@ struct cfl_array *cfl_array_create_in(struct cfl_arena *arena, array = malloc(sizeof(struct cfl_array)); } else { - array = cfl_arena_malloc(arena, sizeof(struct cfl_array)); + array = cfl_arena_alloc_object(arena, sizeof(struct cfl_array)); } if (array == NULL) { cfl_errno(); @@ -64,8 +64,8 @@ struct cfl_array *cfl_array_create_in(struct cfl_arena *arena, array->entries = calloc(alloc_count, sizeof(void *)); } else { - array->entries = cfl_arena_calloc(arena, alloc_count, - sizeof(void *)); + array->entries = cfl_arena_calloc_object(arena, alloc_count, + sizeof(void *)); } if (array->entries == NULL) { cfl_errno(); @@ -222,7 +222,7 @@ int cfl_array_append(struct cfl_array *array, tmp = realloc(array->entries, new_size); } else { - tmp = cfl_arena_malloc(array->arena, new_size); + tmp = cfl_arena_alloc_object(array->arena, new_size); if (tmp != NULL) { memcpy(tmp, array->entries, array->entry_count * sizeof(void *)); diff --git a/src/cfl_kvlist.c b/src/cfl_kvlist.c index 204ab77..3993e48 100644 --- a/src/cfl_kvlist.c +++ b/src/cfl_kvlist.c @@ -99,7 +99,7 @@ struct cfl_kvlist *cfl_kvlist_create_in(struct cfl_arena *arena) list = malloc(sizeof(struct cfl_kvlist)); } else { - list = cfl_arena_malloc(arena, sizeof(struct cfl_kvlist)); + list = cfl_arena_alloc_object(arena, sizeof(struct cfl_kvlist)); } if (list == NULL) { cfl_report_runtime_error(); diff --git a/src/cfl_sds.c b/src/cfl_sds.c index afe89c2..13e3923 100644 --- a/src/cfl_sds.c +++ b/src/cfl_sds.c @@ -82,8 +82,9 @@ static cfl_sds_t sds_alloc(struct cfl_arena *arena, size_t size) if (arena == NULL) { buf = malloc(allocation_size); } - else if (size <= 1024) { - external_threshold = cfl_arena_large_object_threshold(arena); + else if (size <= CFL_ARENA_SDS_MAX_PAYLOAD) { + /* pooled strings never use external storage */ + external_threshold = SIZE_MAX; buf = cfl_arena_alloc_sds(arena, size, CFL_SDS_HEADER_SIZE + sizeof(struct cfl_sds_arena_header), @@ -99,7 +100,7 @@ static cfl_sds_t sds_alloc(struct cfl_arena *arena, size_t size) buf = cfl_arena_alloc_external(arena, allocation_size); } else { - buf = cfl_arena_malloc(arena, allocation_size); + buf = cfl_arena_alloc_object(arena, allocation_size); } } if (!buf) { diff --git a/tests/arena.c b/tests/arena.c index 494fa63..77e4969 100644 --- a/tests/arena.c +++ b/tests/arena.c @@ -17,6 +17,56 @@ struct test_alignment_probe { union test_max_align value; }; +/* CFL objects hold pointers, sizes, integers and doubles */ +union test_object_align { + void *pointer_value; + size_t size_value; + uint64_t uint64_value; + double double_value; +}; + +struct test_object_alignment_probe { + char byte; + union test_object_align value; +}; + +struct test_variant_probe { + char byte; + struct cfl_variant value; +}; + +struct test_kvpair_probe { + char byte; + struct cfl_kvpair value; +}; + +struct test_kvlist_probe { + char byte; + struct cfl_kvlist value; +}; + +struct test_array_probe { + char byte; + struct cfl_array value; +}; + +#define TEST_MAX_ALIGNMENT offsetof(struct test_alignment_probe, value) +#define TEST_OBJECT_ALIGNMENT offsetof(struct test_object_alignment_probe, value) + +/* owned SDS payloads up to this size are recycled through size classes */ +#define TEST_SDS_POOLED_PAYLOAD 1024 + +static int is_aligned(const void *pointer, size_t alignment) +{ + return ((uintptr_t) pointer % alignment) == 0; +} + +static uint32_t test_random(uint32_t *state) +{ + *state = (*state * 1103515245u) + 12345u; + return (*state >> 8) & 0xffffff; +} + struct test_allocator_context { size_t allocation_count; size_t free_count; @@ -344,6 +394,7 @@ static void reclaim_large_string_on_remove(void) value = cfl_kvlist_fetch(list, "body"); TEST_CHECK(value != NULL); first_buffer = value->data.as_string; + TEST_CHECK(is_aligned(CFL_SDS_HEADER(first_buffer), TEST_OBJECT_ALIGNMENT)); before_remove = cfl_arena_bytes_used(arena); reserved_before_remove = cfl_arena_bytes_reserved(arena); TEST_CHECK(cfl_kvlist_remove(list, "body") == CFL_TRUE); @@ -428,13 +479,15 @@ static void create_like_and_rename(void) static void reuse_sds_size_classes(void) { - static const size_t sizes[] = {1, 32, 33, 64, 65, 128, - 129, 256, 257, 512, 513, 1024}; + static const size_t sizes[] = {0, 1, 7, 8, 15, 16, 31, 32, 33, 64, 65, + 127, 128, 129, 256, 257, 512, 513, 1023, + 1024}; struct cfl_arena *arena; cfl_sds_t first; cfl_sds_t second; cfl_sds_t grown; char payload[1024]; + size_t capacity; size_t index; memset(payload, 's', sizeof(payload)); @@ -444,28 +497,565 @@ static void reuse_sds_size_classes(void) for (index = 0; index < sizeof(sizes) / sizeof(sizes[0]); index++) { first = cfl_sds_create_len_in(arena, payload, (int) sizes[index]); TEST_CHECK(first != NULL); + TEST_CHECK(cfl_sds_alloc(first) >= sizes[index]); cfl_sds_destroy(first); second = cfl_sds_create_len_in(arena, payload, (int) sizes[index]); TEST_CHECK(second == first); cfl_sds_destroy(second); } + /* growing past the capacity moves to the next class, keeping the data */ first = cfl_sds_create_len_in(arena, payload, 31); TEST_CHECK(first != NULL); - TEST_CHECK(cfl_sds_alloc(first) == 32); - grown = cfl_sds_increase(first, 1); + capacity = cfl_sds_alloc(first); + TEST_CHECK(capacity >= 31); + grown = cfl_sds_increase(first, capacity - 31 + 1); TEST_CHECK(grown != NULL); - TEST_CHECK(cfl_sds_alloc(grown) == 64); + TEST_CHECK(grown != first); + TEST_CHECK(cfl_sds_alloc(grown) > capacity); TEST_CHECK(cfl_sds_len(grown) == 31); TEST_CHECK(memcmp(grown, payload, 31) == 0); + TEST_CHECK(grown[31] == '\0'); - second = cfl_sds_create_len_in(arena, payload, 32); + /* the released slot serves the next string of the same class */ + second = cfl_sds_create_len_in(arena, payload, 31); TEST_CHECK(second == first); cfl_sds_destroy(second); cfl_sds_destroy(grown); cfl_arena_destroy(arena); } +static void sds_class_boundaries(void) +{ + struct cfl_arena *arena; + cfl_sds_t value; + cfl_sds_t again; + char payload[TEST_SDS_POOLED_PAYLOAD + 64]; + size_t previous_capacity; + size_t capacity; + size_t used; + size_t size; + + memset(payload, 'b', sizeof(payload)); + arena = cfl_arena_create(4096); + TEST_CHECK(arena != NULL); + if (arena == NULL) { + return; + } + + previous_capacity = 0; + for (size = 0; size <= TEST_SDS_POOLED_PAYLOAD + 32; size++) { + used = cfl_arena_bytes_used(arena); + value = cfl_sds_create_len_in(arena, payload, (int) size); + TEST_CHECK(value != NULL); + if (value == NULL) { + break; + } + + capacity = cfl_sds_alloc(value); + TEST_CHECK(cfl_sds_len(value) == size); + TEST_CHECK(capacity >= size); + TEST_CHECK(cfl_sds_avail(value) == capacity - size); + TEST_CHECK(value[size] == '\0'); + TEST_CHECK(memcmp(value, payload, size) == 0); + TEST_CHECK(is_aligned(CFL_SDS_HEADER(value), TEST_OBJECT_ALIGNMENT)); + + if (size <= TEST_SDS_POOLED_PAYLOAD) { + /* class capacities never shrink as the payload grows */ + TEST_CHECK(capacity >= previous_capacity); + previous_capacity = capacity; + + /* short strings are sized within one alignment unit */ + if (size < 128) { + TEST_CHECK(capacity - size < TEST_OBJECT_ALIGNMENT); + } + else { + TEST_CHECK(capacity - size <= size / 4 + TEST_OBJECT_ALIGNMENT); + } + } + else { + TEST_CHECK(capacity == size); + } + + cfl_sds_destroy(value); + + /* larger strings below the external threshold live until reset */ + if (size <= TEST_SDS_POOLED_PAYLOAD) { + TEST_CHECK(cfl_arena_bytes_used(arena) == used); + again = cfl_sds_create_len_in(arena, payload, (int) size); + TEST_CHECK(again == value); + cfl_sds_destroy(again); + } + } + + cfl_arena_destroy(arena); +} + +static void sds_capacity_is_writable(void) +{ + struct cfl_arena *arena; + cfl_sds_t values[256]; + char payload[2048]; + size_t capacity; + size_t index; + size_t offset; + int intact; + + memset(payload, 'p', sizeof(payload)); + arena = cfl_arena_create(1024); + TEST_CHECK(arena != NULL); + if (arena == NULL) { + return; + } + + /* fill every string to its reported capacity, neighbours must survive */ + for (index = 0; index < 256; index++) { + values[index] = cfl_sds_create_len_in(arena, payload, + (int) ((index * 37) % 1500)); + TEST_CHECK(values[index] != NULL); + if (values[index] == NULL) { + cfl_arena_destroy(arena); + return; + } + capacity = cfl_sds_alloc(values[index]); + memset(values[index], (int) ('A' + (index % 26)), capacity); + cfl_sds_set_len(values[index], capacity); + TEST_CHECK(values[index][capacity] == '\0'); + } + + for (index = 0; index < 256; index++) { + capacity = cfl_sds_alloc(values[index]); + TEST_CHECK(cfl_sds_len(values[index]) == capacity); + intact = 1; + for (offset = 0; offset < capacity; offset++) { + if (values[index][offset] != (char) ('A' + (index % 26))) { + intact = 0; + break; + } + } + TEST_CHECK(intact == 1); + TEST_CHECK(values[index][capacity] == '\0'); + TEST_MSG("string %zu with capacity %zu", index, capacity); + } + + for (index = 0; index < 256; index += 2) { + cfl_sds_destroy(values[index]); + } + for (index = 1; index < 256; index += 2) { + TEST_CHECK(values[index][0] == (char) ('A' + (index % 26))); + cfl_sds_destroy(values[index]); + } + TEST_CHECK(cfl_arena_bytes_used(arena) == 0); + + cfl_arena_destroy(arena); +} + +static void sds_grow_across_classes(void) +{ + struct cfl_arena *arena; + cfl_sds_t value; + cfl_sds_t result; + char expected[3001]; + size_t index; + + arena = cfl_arena_create_ex(2048, 1500); + TEST_CHECK(arena != NULL); + if (arena == NULL) { + return; + } + + /* small classes, medium classes, chunk storage and external storage */ + value = cfl_sds_create_len_in(arena, "", 0); + TEST_CHECK(value != NULL); + for (index = 0; index < 3000 && value != NULL; index++) { + expected[index] = (char) ('a' + (index % 26)); + result = cfl_sds_cat(value, &expected[index], 1); + TEST_CHECK(result != NULL); + if (result == NULL) { + break; + } + value = result; + TEST_CHECK(is_aligned(CFL_SDS_HEADER(value), TEST_OBJECT_ALIGNMENT)); + TEST_CHECK(cfl_sds_len(value) == index + 1); + TEST_CHECK(cfl_sds_alloc(value) >= index + 1); + TEST_CHECK(value[index + 1] == '\0'); + } + expected[3000] = '\0'; + TEST_CHECK(value != NULL && strcmp(value, expected) == 0); + + /* self append and printf keep working on arena storage */ + result = cfl_sds_cat(value, value, 10); + TEST_CHECK(result != NULL); + if (result != NULL) { + value = result; + TEST_CHECK(cfl_sds_len(value) == 3010); + TEST_CHECK(memcmp(value + 3000, expected, 10) == 0); + } + result = cfl_sds_printf(&value, "-%d", 42); + TEST_CHECK(result != NULL); + TEST_CHECK(cfl_sds_len(value) == 3013); + TEST_CHECK(strcmp(value + 3010, "-42") == 0); + + cfl_sds_destroy(value); + cfl_arena_destroy(arena); +} + +static void object_alignment(void) +{ + struct cfl_arena *arena; + struct cfl_variant *variant; + struct cfl_kvlist *list; + struct cfl_kvpair *pair; + struct cfl_array *array; + struct cfl_variant *entry; + struct cfl_list *head; + char key[8]; + size_t index; + + arena = cfl_arena_create(512); + TEST_CHECK(arena != NULL); + if (arena == NULL) { + return; + } + + list = cfl_kvlist_create_in(arena); + array = cfl_array_create_in(arena, 1); + TEST_CHECK(list != NULL && array != NULL); + if (list == NULL || array == NULL) { + cfl_arena_destroy(arena); + return; + } + TEST_CHECK(cfl_array_resizable(array, CFL_TRUE) == 0); + + /* odd sized keys and payloads shift the bump pointer between objects */ + for (index = 0; index < 64; index++) { + snprintf(key, sizeof(key), "k%zu", index); + TEST_CHECK(cfl_kvlist_insert_string_s(list, key, strlen(key), "abc", + index % 4, CFL_FALSE) == 0); + TEST_CHECK(cfl_kvlist_insert_double(list, "", (double) index) == 0); + TEST_CHECK(cfl_array_append_uint64(array, index) == 0); + + variant = cfl_variant_create_from_int64_in(arena, (int64_t) index); + TEST_CHECK(variant != NULL); + TEST_CHECK(is_aligned(variant, TEST_OBJECT_ALIGNMENT)); + TEST_CHECK(is_aligned(variant, + offsetof(struct test_variant_probe, value))); + cfl_variant_destroy(variant); + } + + TEST_CHECK(is_aligned(list, offsetof(struct test_kvlist_probe, value))); + TEST_CHECK(is_aligned(array, offsetof(struct test_array_probe, value))); + TEST_CHECK(is_aligned(array->entries, sizeof(void *))); + + cfl_list_foreach(head, &list->list) { + pair = cfl_list_entry(head, struct cfl_kvpair, _head); + TEST_CHECK(is_aligned(pair, offsetof(struct test_kvpair_probe, value))); + TEST_CHECK(is_aligned(pair->val, + offsetof(struct test_variant_probe, value))); + TEST_CHECK(is_aligned(CFL_SDS_HEADER(pair->key), + TEST_OBJECT_ALIGNMENT)); + } + + for (index = 0; index < cfl_array_size(array); index++) { + entry = cfl_array_fetch_by_index(array, index); + TEST_CHECK(entry != NULL); + TEST_CHECK(is_aligned(entry, offsetof(struct test_variant_probe, value))); + TEST_CHECK(entry->data.as_uint64 == index); + } + + cfl_kvlist_destroy(list); + cfl_array_destroy(array); + cfl_arena_destroy(arena); +} + +static void raw_alignment_after_objects(void) +{ + struct cfl_arena *arena; + struct cfl_variant *variant; + cfl_sds_t key; + void *pointer; + size_t index; + + arena = cfl_arena_create(256); + TEST_CHECK(arena != NULL); + if (arena == NULL) { + return; + } + + /* raw allocations keep the public alignment between packed objects */ + for (index = 0; index < 64; index++) { + variant = cfl_variant_create_from_bool_in(arena, CFL_TRUE); + TEST_CHECK(variant != NULL); + key = cfl_sds_create_len_in(arena, "key", (int) (index % 5)); + TEST_CHECK(key != NULL); + + pointer = cfl_arena_malloc(arena, 1 + (index % 3)); + TEST_CHECK(pointer != NULL && is_aligned(pointer, TEST_MAX_ALIGNMENT)); + pointer = cfl_arena_calloc(arena, 3, 1); + TEST_CHECK(pointer != NULL && is_aligned(pointer, TEST_MAX_ALIGNMENT)); + pointer = cfl_arena_memdup(arena, "abc", 3); + TEST_CHECK(pointer != NULL && is_aligned(pointer, TEST_MAX_ALIGNMENT)); + pointer = cfl_arena_strndup(arena, "abc", 2); + TEST_CHECK(pointer != NULL && is_aligned(pointer, TEST_MAX_ALIGNMENT)); + } + + cfl_arena_destroy(arena); +} + +static void compact_short_keys(void) +{ + struct cfl_arena *arena; + struct cfl_kvlist *record; + struct cfl_variant *root; + static char *keys[] = {"msg", "user", "path", "status", "level", "ts"}; + size_t record_count; + size_t index; + size_t key_index; + size_t limit; + size_t reserved; + size_t object; + + record_count = 256; + arena = cfl_arena_create(16384); + TEST_CHECK(arena != NULL); + if (arena == NULL) { + return; + } + + /* worst case: every object padded by one alignment unit */ + object = TEST_OBJECT_ALIGNMENT; + limit = sizeof(struct cfl_kvlist) + object + sizeof(struct cfl_variant) + object; + for (key_index = 0; key_index < 6; key_index++) { + limit += sizeof(struct cfl_kvpair) + object + + sizeof(struct cfl_variant) + object + + 2 * sizeof(void *) + CFL_SDS_HEADER_SIZE + + strlen(keys[key_index]) + 1 + object; + } + + for (index = 0; index < record_count; index++) { + record = cfl_kvlist_create_in(arena); + TEST_CHECK(record != NULL); + if (record == NULL) { + break; + } + for (key_index = 0; key_index < 6; key_index++) { + TEST_CHECK(cfl_kvlist_insert_int64(record, keys[key_index], + (int64_t) index) == 0); + } + root = cfl_variant_create_from_kvlist_in(arena, record); + TEST_CHECK(root != NULL); + } + + /* allow one partially used chunk plus chunk headers */ + reserved = cfl_arena_bytes_reserved(arena); + TEST_CHECK(reserved <= record_count * limit + 16384 + 256); + TEST_MSG("reserved %zu limit %zu", reserved, record_count * limit + 16384 + 256); + + cfl_arena_destroy(arena); +} + +static void reset_reuses_storage_without_aliasing(void) +{ + struct cfl_arena *arena; + cfl_sds_t first; + cfl_sds_t second; + struct cfl_variant *variant; + struct cfl_variant *other; + size_t reserved; + int round; + + arena = cfl_arena_create(1024); + TEST_CHECK(arena != NULL); + if (arena == NULL) { + return; + } + + for (round = 0; round < 8; round++) { + /* released slots must not survive the reset in the free lists */ + first = cfl_sds_create_len_in(arena, "abc", 3); + variant = cfl_variant_create_from_int64_in(arena, round); + TEST_CHECK(first != NULL && variant != NULL); + cfl_sds_destroy(first); + cfl_variant_destroy(variant); + reserved = cfl_arena_bytes_reserved(arena); + cfl_arena_reset(arena); + TEST_CHECK(cfl_arena_bytes_used(arena) == 0); + TEST_CHECK(cfl_arena_bytes_reserved(arena) == reserved); + + first = cfl_sds_create_len_in(arena, "abc", 3); + second = cfl_sds_create_len_in(arena, "xyz", 3); + variant = cfl_variant_create_from_int64_in(arena, 1); + other = cfl_variant_create_from_int64_in(arena, 2); + TEST_CHECK(first != NULL && second != NULL); + TEST_CHECK(variant != NULL && other != NULL); + TEST_CHECK(first != second && variant != other); + TEST_CHECK(strcmp(first, "abc") == 0 && strcmp(second, "xyz") == 0); + TEST_CHECK(variant->data.as_int64 == 1 && other->data.as_int64 == 2); + cfl_arena_reset(arena); + } + + cfl_arena_destroy(arena); +} + +static void heap_storage_unchanged(void) +{ + cfl_sds_t value; + struct cfl_kvlist *list; + struct cfl_kvpair *pair; + + /* heap strings keep their exact capacities */ + value = cfl_sds_create_len("abc", 3); + TEST_CHECK(value != NULL); + TEST_CHECK(cfl_sds_alloc(value) == 3); + cfl_sds_destroy(value); + + value = cfl_sds_create_size(100); + TEST_CHECK(value != NULL); + TEST_CHECK(cfl_sds_alloc(value) == 100); + TEST_CHECK(cfl_sds_len(value) == 0); + cfl_sds_destroy(value); + + list = cfl_kvlist_create(); + TEST_CHECK(list != NULL); + if (list == NULL) { + return; + } + TEST_CHECK(cfl_kvlist_insert_int64(list, "status", 200) == 0); + pair = cfl_list_entry_first(&list->list, struct cfl_kvpair, _head); + TEST_CHECK(pair->arena == NULL); + TEST_CHECK(cfl_sds_alloc(pair->key) == 6); + cfl_kvlist_destroy(list); +} + +#define TEST_MUTATION_KEYS 48 + +static void randomized_kvlist_mutation(void) +{ + struct cfl_arena *arena; + struct cfl_kvlist *list; + struct cfl_variant *value; + char *payload; + char keys[TEST_MUTATION_KEYS][224]; + char names[TEST_MUTATION_KEYS][224]; + size_t lengths[TEST_MUTATION_KEYS]; + int present[TEST_MUTATION_KEYS]; + uint32_t state; + size_t key_length; + size_t index; + size_t slot; + size_t count; + int operation; + int round; + int expected; + + payload = malloc(4096); + TEST_CHECK(payload != NULL); + if (payload == NULL) { + return; + } + for (index = 0; index < 4096; index++) { + payload[index] = (char) ('a' + (index % 23)); + } + + /* small chunks and a low external threshold stress every storage path */ + arena = cfl_arena_create_ex(1024, 2048); + TEST_CHECK(arena != NULL); + if (arena == NULL) { + free(payload); + return; + } + + state = 2026; + for (round = 0; round < 4; round++) { + list = cfl_kvlist_create_in(arena); + TEST_CHECK(list != NULL); + if (list == NULL) { + break; + } + + for (slot = 0; slot < TEST_MUTATION_KEYS; slot++) { + key_length = 1 + test_random(&state) % 200; + memset(keys[slot], 'k', key_length); + snprintf(keys[slot], 8, "%02zu", slot); + keys[slot][2] = 'k'; + keys[slot][key_length] = '\0'; + strcpy(names[slot], keys[slot]); + present[slot] = 0; + lengths[slot] = 0; + } + + for (index = 0; index < 4000; index++) { + slot = test_random(&state) % TEST_MUTATION_KEYS; + operation = (int) (test_random(&state) % 4); + + if (operation == 0 && !present[slot]) { + lengths[slot] = test_random(&state) % 3000; + TEST_CHECK(cfl_kvlist_insert_string_s(list, names[slot], + strlen(names[slot]), + payload, lengths[slot], + CFL_FALSE) == 0); + present[slot] = 1; + } + else if (operation == 1 && present[slot]) { + TEST_CHECK(cfl_kvlist_remove(list, names[slot]) == CFL_TRUE); + present[slot] = 0; + } + else if (operation == 2 && present[slot]) { + /* rename between the short and the long spelling */ + if (strcmp(names[slot], keys[slot]) == 0) { + snprintf(names[slot], 8, "%02zu", slot); + } + else { + strcpy(names[slot], keys[slot]); + } + TEST_CHECK(cfl_kvlist_rename_s(list, keys[slot], strlen(keys[slot]), + names[slot], strlen(names[slot])) == 0 || + strcmp(names[slot], keys[slot]) == 0); + if (strcmp(names[slot], keys[slot]) == 0) { + snprintf(names[slot], 8, "%02zu", slot); + TEST_CHECK(cfl_kvlist_rename_s(list, names[slot], + strlen(names[slot]), + keys[slot], + strlen(keys[slot])) == 0); + strcpy(names[slot], keys[slot]); + } + } + else if (operation == 3 && present[slot]) { + TEST_CHECK(cfl_kvlist_remove(list, names[slot]) == CFL_TRUE); + lengths[slot] = test_random(&state) % 3000; + TEST_CHECK(cfl_kvlist_insert_string_s(list, names[slot], + strlen(names[slot]), + payload, lengths[slot], + CFL_FALSE) == 0); + } + } + + count = 0; + for (slot = 0; slot < TEST_MUTATION_KEYS; slot++) { + value = cfl_kvlist_fetch(list, names[slot]); + expected = present[slot]; + TEST_CHECK((value != NULL) == expected); + if (value == NULL || !expected) { + continue; + } + count++; + TEST_CHECK(value->type == CFL_VARIANT_STRING); + TEST_CHECK(cfl_sds_len(value->data.as_string) == lengths[slot]); + TEST_CHECK(memcmp(value->data.as_string, payload, + lengths[slot]) == 0); + TEST_CHECK(value->data.as_string[lengths[slot]] == '\0'); + } + TEST_CHECK((size_t) cfl_kvlist_count(list) == count); + + cfl_kvlist_destroy(list); + cfl_arena_reset(arena); + TEST_CHECK(cfl_arena_bytes_used(arena) == 0); + } + + cfl_arena_destroy(arena); + free(payload); +} + static void bound_external_rounding_and_cache(void) { struct cfl_arena *arena; @@ -523,8 +1113,7 @@ static void reclaim_failed_variant_construction(void) reused = cfl_variant_create_from_int64_in(arena, 1); TEST_CHECK(reused != NULL); - TEST_CHECK(((uintptr_t) reused % - offsetof(struct test_alignment_probe, value)) == 0); + TEST_CHECK(is_aligned(reused, offsetof(struct test_variant_probe, value))); cfl_variant_destroy(reused); cfl_arena_destroy(arena); } @@ -542,6 +1131,15 @@ TEST_LIST = { {"reuse_variant_and_kvpair_slots", reuse_variant_and_kvpair_slots}, {"create_like_and_rename", create_like_and_rename}, {"reuse_sds_size_classes", reuse_sds_size_classes}, + {"sds_class_boundaries", sds_class_boundaries}, + {"sds_capacity_is_writable", sds_capacity_is_writable}, + {"sds_grow_across_classes", sds_grow_across_classes}, + {"object_alignment", object_alignment}, + {"raw_alignment_after_objects", raw_alignment_after_objects}, + {"compact_short_keys", compact_short_keys}, + {"reset_reuses_storage_without_aliasing", reset_reuses_storage_without_aliasing}, + {"heap_storage_unchanged", heap_storage_unchanged}, + {"randomized_kvlist_mutation", randomized_kvlist_mutation}, {"bound_external_rounding_and_cache", bound_external_rounding_and_cache}, {"reclaim_failed_variant_construction", reclaim_failed_variant_construction}, {NULL, NULL}