diff --git a/ddprof-lib/src/main/cpp/codeCache.cpp b/ddprof-lib/src/main/cpp/codeCache.cpp index 9062b555b1..57842cec24 100644 --- a/ddprof-lib/src/main/cpp/codeCache.cpp +++ b/ddprof-lib/src/main/cpp/codeCache.cpp @@ -158,7 +158,7 @@ CodeCache::~CodeCache() { free(_build_id); // Free build-id memory } -long long CodeCache::memoryUsage() const { +long long CodeCache::memoryUsage(long long *overhead_out) const { // The blob array: _capacity entries of CodeBlob. long long total = (long long)_capacity * sizeof(CodeBlob); @@ -177,6 +177,25 @@ long long CodeCache::memoryUsage() const { total += (long long)NativeFunc::allocSize(_blobs[i]._name); } + // Measured allocator overhead on those name allocations, accumulated in the + // same pass. Name strings are numerous and short, so this is where the + // overhead lives: a blanket percentage derived from one workload's size mix + // cannot stand in for it. + // + // The _blobs array itself is deliberately excluded. It is one allocation per + // library, large enough that its overhead is a rounding error, and it comes + // from new CodeBlob[] -- whose returned pointer is not guaranteed to be the + // allocator's block base, so querying the allocator with it would be + // unsound. Excluding it understates by a negligible amount rather than + // risking a wrong reading. + if (overhead_out != nullptr) { + long long overhead = (long long)NativeFunc::nameOverhead(_name); + for (int i = 0; i < _count; i++) { + overhead += (long long)NativeFunc::nameOverhead(_blobs[i]._name); + } + *overhead_out += overhead; + } + // The DWARF unwind table, when present (length only — no pointer deref). total += (long long)_dwarf_table_length * sizeof(FrameDesc); diff --git a/ddprof-lib/src/main/cpp/codeCache.h b/ddprof-lib/src/main/cpp/codeCache.h index d8ac7d661e..076db50a5d 100644 --- a/ddprof-lib/src/main/cpp/codeCache.h +++ b/ddprof-lib/src/main/cpp/codeCache.h @@ -10,6 +10,7 @@ #include "common.h" #include "counters.h" #include "dwarf.h" +#include "mallocFootprint.h" #include "utils.h" #include @@ -85,6 +86,18 @@ class NativeFunc { return align_up(sizeof(NativeFunc) + 1 + strlen(name), sizeof(NativeFunc *)); } + // Allocator overhead on that allocation -- rounding to the size quantum plus + // the per-chunk header, measured rather than assumed. Lives here because only + // NativeFunc knows the real allocation base: `name` points *into* the block at + // offset sizeof(NativeFunc), so querying the allocator with `name` itself + // would be undefined. 0 if null. + static size_t nameOverhead(const char *name) { + if (name == nullptr) { + return 0; + } + return MallocFootprint::overheadOf(from(name), allocSize(name)); + } + static short libIndex(const char *name) { if (name == nullptr) { return -1; @@ -286,7 +299,9 @@ class CodeCache { // — it is mutated by the background refresher and negligible in size (see the // definition). Const and lock-free: reads only fields that are stable once the // library is published. - long long memoryUsage() const; + // overhead_out, when non-null, receives the measured allocator overhead on + // the name allocations counted here (see NativeFunc::nameOverhead). + long long memoryUsage(long long *overhead_out = nullptr) const; int count() { return _count; } CodeBlob* blob(int idx) { @@ -357,15 +372,19 @@ class CodeCacheArray { // cached at add() time. (Libraries are populated before being registered and // not grown afterwards.) The array is append-only, so iterating the published // prefix is safe alongside concurrent add()s. - size_t memoryUsage() const { + size_t memoryUsage(long long *overhead_out = nullptr) const { size_t total = 0; + long long overhead = 0; int n = count(); for (int i = 0; i < n; i++) { CodeCache *lib = at(i); if (lib != nullptr) { - total += (size_t)lib->memoryUsage(); + total += (size_t)lib->memoryUsage(overhead_out != nullptr ? &overhead : nullptr); } } + if (overhead_out != nullptr) { + *overhead_out = overhead; + } return total; } }; diff --git a/ddprof-lib/src/main/cpp/counters.h b/ddprof-lib/src/main/cpp/counters.h index 430f379c3e..20498d4640 100644 --- a/ddprof-lib/src/main/cpp/counters.h +++ b/ddprof-lib/src/main/cpp/counters.h @@ -56,6 +56,20 @@ X(NATIVE_MEM_LIVE_BYTES, "native_mem_live_bytes") \ X(NATIVE_MEM_MAX_BYTES, "native_mem_max_bytes") \ X(NATIVE_MEM_AVG_BYTES, "native_mem_avg_bytes") \ + /* Process-wide malloc arena state, from glibc's own accounting. These are \ + * NOT profiler memory and deliberately are not NM_* categories: those must \ + * partition the profiler's own allocations (see nativeMem.h), and arena \ + * slack is mostly other subsystems' chunks stranded by interleaving, so it \ + * is not attributable to any profiler allocation. Reported so that the \ + * allocator's own overhead is visible rather than folded into a "profiler \ + * cost" figure -- it differs substantially between glibc, tcmalloc and \ + * jemalloc. Sampled on the JFR flush path only: mallinfo2() walks every \ + * arena taking locks, so it is neither cheap nor async-signal-safe. */ \ + X(MALLOC_ARENA_BYTES, "malloc_arena_bytes") \ + X(MALLOC_IN_USE_BYTES, "malloc_in_use_bytes") \ + X(MALLOC_FREE_HELD_BYTES, "malloc_free_held_bytes") \ + X(MALLOC_TRIMMABLE_BYTES, "malloc_trimmable_bytes") \ + X(MALLOC_MMAP_BYTES, "malloc_mmap_bytes") \ X(THREAD_IDS_COUNT, "thread_ids_count") \ X(THREAD_NAMES_COUNT, "thread_names_count") \ X(THREAD_FILTER_PAGES, "thread_filter_pages") \ diff --git a/ddprof-lib/src/main/cpp/countingAllocator.h b/ddprof-lib/src/main/cpp/countingAllocator.h index e31aae3b85..6658b19672 100644 --- a/ddprof-lib/src/main/cpp/countingAllocator.h +++ b/ddprof-lib/src/main/cpp/countingAllocator.h @@ -25,12 +25,18 @@ class CountingAllocator { T *allocate(std::size_t n) { T *p = static_cast(::operator new(n * sizeof(T))); - NativeMem::record(Cat, (long long)(n * sizeof(T))); + // recordAlloc rather than record: STL nodes are small, so the allocator's + // rounding and per-chunk header are a large fraction of their real cost + // (a 96-byte MethodMap node occupies 112 bytes -- 16.7 %). Logical bytes + // still land in the live gauge; the extra goes to the overhead gauge. + NativeMem::recordAlloc(Cat, p, n * sizeof(T)); return p; } void deallocate(T *p, std::size_t n) noexcept { - NativeMem::record(Cat, -(long long)(n * sizeof(T))); + // Must run before operator delete: the overhead is read back off the live + // chunk. + NativeMem::recordFreeBefore(Cat, p, n * sizeof(T)); ::operator delete(p); } diff --git a/ddprof-lib/src/main/cpp/flightRecorder.cpp b/ddprof-lib/src/main/cpp/flightRecorder.cpp index 29156844cf..f20ff31218 100644 --- a/ddprof-lib/src/main/cpp/flightRecorder.cpp +++ b/ddprof-lib/src/main/cpp/flightRecorder.cpp @@ -7,6 +7,15 @@ #include #include +// mallinfo2() replaced the int-based mallinfo() in glibc 2.33; the older struct +// silently truncates past 2 GiB, so it is not a usable fallback for byte +// accounting. Absent on musl and macOS, where the arena counters stay zero. +#if defined(__linux__) && defined(__GLIBC__) && \ + (__GLIBC__ > 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 33)) + #include + #define DD_HAVE_MALLINFO2 1 +#endif + #include "buffers.h" #include "callTraceHashTable.h" #include "context.h" @@ -1809,12 +1818,46 @@ void Recording::writeLogLevels(Buffer *buf) { } } +// Snapshot the process-wide malloc arena state from glibc's own accounting. +// +// This is NOT profiler memory. Free-but-held arena pages are mostly other +// subsystems' chunks stranded by interleaving, so they cannot be attributed to +// any profiler allocation -- reporting them separately keeps them out of the +// per-category figures while still making the allocator's overhead visible. +// The numbers also differ substantially between glibc, tcmalloc and jemalloc, +// which is exactly what a reader comparing allocators needs to see. +// +// Only safe on the flush path: mallinfo2() walks every arena taking each arena +// lock, so it is neither cheap nor async-signal-safe. Must never be reached +// from the sampling signal handler. Once per JFR chunk is negligible. +void Recording::updateMallocArenaStats() { +#ifdef DD_HAVE_MALLINFO2 + struct mallinfo2 mi = mallinfo2(); + // arena: bytes obtained from the OS via brk, excluding mmap'd chunks + // uordblks: bytes currently handed out to callers + // fordblks: bytes free but retained in the arenas -- the waste term + // keepcost: the trimmable top block, i.e. what malloc_trim could return; + // separating it distinguishes trim-threshold policy from genuine + // fragmentation, which trimming cannot reclaim + // hblkhd: bytes in mmap'd chunks, which are returned to the OS on free + Counters::set(MALLOC_ARENA_BYTES, (long long)mi.arena); + Counters::set(MALLOC_IN_USE_BYTES, (long long)mi.uordblks); + Counters::set(MALLOC_FREE_HELD_BYTES, (long long)mi.fordblks); + Counters::set(MALLOC_TRIMMABLE_BYTES, (long long)mi.keepcost); + Counters::set(MALLOC_MMAP_BYTES, (long long)mi.hblkhd); +#endif +} + void Recording::updateNativeMemStats() { // Refresh the moving-window averages and the observed total peak. Per-category // peaks are maintained precisely at allocation time, so they are not sampled // here; the total peak is bracketed instead (see writeNativeMem). NativeMem::sample(); + // Process-wide allocator state, sampled at the same instant as the + // per-category figures so the two can be compared coherently. + updateMallocArenaStats(); + // Mirror the totals into the flat counter table so they flow out through the // existing counter path (JFR T_DATADOG_COUNTER events and the JNI debug // counters). NATIVE_MEM_MAX_BYTES carries the upper bound on the total peak @@ -1858,6 +1901,14 @@ void Recording::writeNativeMem(Buffer *buf) { {"native_mem_live_bytes.", NativeMem::live(cat)}, {"native_mem_avg_bytes.", NativeMem::avg(cat)}, {"native_mem_max_bytes.", NativeMem::max(cat)}, + // Measured allocator overhead on the live allocations -- rounding to + // the size quantum plus the per-chunk header. Reported separately so + // native_mem_live_bytes stays comparable to sizeof() arithmetic, and so + // that reconciliation against RSS can add a measured figure instead of + // multiplying by a factor derived from some other workload's + // allocation-size mix. Zero for categories whose call sites still use + // record() rather than recordAlloc(). + {"native_mem_chunk_overhead_bytes.", NativeMem::overhead(cat)}, }; for (const auto &m : metrics) { char label[64]; diff --git a/ddprof-lib/src/main/cpp/flightRecorder.h b/ddprof-lib/src/main/cpp/flightRecorder.h index 2922f368b2..3ed004af61 100644 --- a/ddprof-lib/src/main/cpp/flightRecorder.h +++ b/ddprof-lib/src/main/cpp/flightRecorder.h @@ -324,6 +324,9 @@ class Recording { void writeCounters(Buffer *buf); void updateNativeMemStats(); + // Process-wide malloc arena state (glibc mallinfo2). Flush path only -- + // takes every arena lock, so not async-signal-safe. No-op off glibc 2.33+. + void updateMallocArenaStats(); void writeNativeMem(Buffer *buf); void writeUnwindFailures(Buffer *buf); diff --git a/ddprof-lib/src/main/cpp/mallocFootprint.h b/ddprof-lib/src/main/cpp/mallocFootprint.h new file mode 100644 index 0000000000..5607411fb3 --- /dev/null +++ b/ddprof-lib/src/main/cpp/mallocFootprint.h @@ -0,0 +1,113 @@ +/* + * Copyright 2026, Datadog, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ +#ifndef _MALLOCFOOTPRINT_H +#define _MALLOCFOOTPRINT_H + +#include + +#if defined(__linux__) + #include + #define DD_HAVE_MALLOC_USABLE_SIZE 1 +#elif defined(__APPLE__) + #include + #define DD_HAVE_MALLOC_SIZE 1 +#endif + +// Real resident cost of a heap allocation, as opposed to the size that was +// requested. +// +// The profiler's NM_* gauges record requested (logical) bytes, which is what +// makes them comparable against sizeof() arithmetic. RSS, however, is paid in +// allocator chunks: the request is rounded up to an alignment quantum and +// carries a per-chunk header. Reconciliation previously multiplied logical bytes +// by a single blanket factor measured on one workload, which is wrong whenever +// the allocation-size mix differs -- overhead is a function of *per-allocation* +// size, not of total bytes. A 512 KB chunk pays ~0.003 %; a 96-byte tree node +// pays 16.7 %. +// +// This measures it instead: usable size comes from the allocator, and the +// per-chunk header is probed once at first use rather than assumed. +class MallocFootprint { +private: + // Determined empirically: allocate several same-size blocks and take the + // smallest positive address stride between any two. That stride is + // usable + header, so header = stride - usable. + // + // Probed rather than hardcoded because the allocator in force at runtime is + // not knowable at compile time -- an LD_PRELOAD'd tcmalloc or jemalloc leaves + // __GLIBC__ defined while adding no per-object header at all (their metadata + // is out of band). Assuming glibc's 8 bytes would then invent overhead that + // does not exist, at one allocation's worth per allocation. + static size_t probeHeaderBytes() { +#ifdef DD_HAVE_MALLOC_USABLE_SIZE + const int N = 16; + const size_t SZ = 48; + void *p[N]; + for (int i = 0; i < N; i++) { + p[i] = malloc(SZ); + if (p[i] == NULL) { // give up cleanly rather than guess + for (int j = 0; j < i; j++) free(p[j]); + return 0; + } + } + size_t usable = malloc_usable_size(p[0]); + long best = 0; + for (int i = 0; i < N; i++) { + for (int j = 0; j < N; j++) { + long d = (char *)p[j] - (char *)p[i]; + if (d > 0 && (best == 0 || d < best)) { + best = d; + } + } + } + for (int i = 0; i < N; i++) free(p[i]); + long header = best - (long)usable; + // Sanity-bound the result. A negative or implausibly large value means the + // blocks were not laid out contiguously (a size-class allocator, or an + // arena boundary landed mid-probe), in which case 0 is the honest answer: + // report only the rounding we can see, and understate rather than invent. + if (header < 0 || header > 64) { + return 0; + } + return (size_t)header; +#else + return 0; +#endif + } + +public: + // Per-chunk header size for the allocator actually in force. Probed once; + // the C++11 function-local static makes initialisation thread-safe. NOT + // async-signal-safe (it allocates), so first use must not be from a signal + // handler -- every current call site is on a normal thread. + static size_t headerBytes() { + static const size_t header = probeHeaderBytes(); + return header; + } + + // Bytes this allocation actually costs: allocator-reported usable size plus + // the per-chunk header. Page rounding for large mmap'd chunks is already + // inside the usable size, so it must not be added again. + static size_t of(void *p, size_t requested) { + if (p == NULL) { + return 0; + } +#ifdef DD_HAVE_MALLOC_USABLE_SIZE + return malloc_usable_size(p) + headerBytes(); +#elif defined(DD_HAVE_MALLOC_SIZE) + return malloc_size(p) + headerBytes(); +#else + return requested; // no introspection available: report no overhead +#endif + } + + // Overhead alone -- the part RSS pays for that the logical counters miss. + static size_t overheadOf(void *p, size_t requested) { + size_t total = of(p, requested); + return total > requested ? total - requested : 0; + } +}; + +#endif // _MALLOCFOOTPRINT_H diff --git a/ddprof-lib/src/main/cpp/nativeMem.cpp b/ddprof-lib/src/main/cpp/nativeMem.cpp index df8cf3da58..3c39387959 100644 --- a/ddprof-lib/src/main/cpp/nativeMem.cpp +++ b/ddprof-lib/src/main/cpp/nativeMem.cpp @@ -5,6 +5,7 @@ #include "nativeMem.h" volatile long long NativeMem::_live[NM_NUM_CATEGORIES] = {}; +volatile long long NativeMem::_overhead[NM_NUM_CATEGORIES] = {}; volatile long long NativeMem::_max[NM_NUM_CATEGORIES] = {}; long long NativeMem::_window[NM_NUM_CATEGORIES][NativeMem::WINDOW] = {}; long long NativeMem::_total_window[NativeMem::WINDOW] = {}; @@ -14,6 +15,17 @@ long long NativeMem::_avg[NM_NUM_CATEGORIES] = {}; long long NativeMem::_total_avg = 0; long long NativeMem::_total_max_observed = 0; +long long NativeMem::overheadTotal() { + long long total = 0; + for (int c = 0; c < NM_NUM_CATEGORIES; c++) { + long long v = load(_overhead[c]); + if (v > 0) { + total += v; + } + } + return total; +} + long long NativeMem::liveTotal() { long long total = 0; for (int c = 0; c < NM_NUM_CATEGORIES; c++) { @@ -84,6 +96,7 @@ void NativeMem::reset() { for (int c = 0; c < NM_NUM_CATEGORIES; c++) { store(_live[c], (long long)0); store(_max[c], (long long)0); + store(_overhead[c], (long long)0); _avg[c] = 0; for (int i = 0; i < WINDOW; i++) { _window[c][i] = 0; diff --git a/ddprof-lib/src/main/cpp/nativeMem.h b/ddprof-lib/src/main/cpp/nativeMem.h index 2706f56297..26a33c3255 100644 --- a/ddprof-lib/src/main/cpp/nativeMem.h +++ b/ddprof-lib/src/main/cpp/nativeMem.h @@ -6,6 +6,7 @@ #define _NATIVEMEM_H #include "arch.h" +#include "mallocFootprint.h" #include // Physical native-memory categories used by the profiler's own allocations. @@ -60,6 +61,12 @@ class NativeMem { // Precise per-category high-water mark, maintained at allocation time by // record() so peaks that rise and fall between sample() ticks are still seen. static volatile long long _max[NM_NUM_CATEGORIES]; + // Allocator overhead on the live allocations: rounding to the size quantum + // plus the per-chunk header. Kept SEPARATE from _live rather than folded in, + // so _live stays directly comparable to sizeof() arithmetic while the amount + // RSS additionally pays stays visible. Maintained only by recordAlloc/ + // recordFreeBefore; call sites using plain record() contribute nothing here. + static volatile long long _overhead[NM_NUM_CATEGORIES]; // sample()-owned state; touched only from the single-threaded sampling path. static long long _window[NM_NUM_CATEGORIES][WINDOW]; @@ -94,6 +101,47 @@ class NativeMem { } } + // Record an allocation together with its measured allocator overhead. + // + // Prefer this over record() wherever the pointer is in hand: it keeps the + // logical byte count in _live (comparable to sizeof()) while accumulating the + // rounding-plus-header cost that RSS actually pays. Overhead is a function of + // per-allocation size, so it cannot be recovered later from a byte total -- + // a 512 KB chunk pays ~0.003 %, a 96-byte node 16.7 %. + // + // Not async-signal-safe on first call (the header probe allocates); every + // current call site runs on a normal thread. + static void recordAlloc(NativeMemCategory category, void *ptr, + size_t requested) { + record(category, (long long)requested); + if (ptr != NULL) { + atomicIncRelaxed(_overhead[category], + (long long)MallocFootprint::overheadOf(ptr, requested)); + } + } + + // Counterpart to recordAlloc. MUST be called before the pointer is freed -- + // the overhead is read back off the live chunk, which is unreadable afterwards. + static void recordFreeBefore(NativeMemCategory category, void *ptr, + size_t requested) { + if (ptr != NULL) { + atomicIncRelaxed(_overhead[category], + -(long long)MallocFootprint::overheadOf(ptr, requested)); + } + record(category, -(long long)requested); + } + + // Gauge-style overhead setter, for categories whose size is recomputed as an + // absolute rather than tracked via alloc/free deltas (see setLive). + static void setOverhead(NativeMemCategory category, long long value) { + store(_overhead[category], value); + } + + static long long overhead(NativeMemCategory category) { + return load(_overhead[category]); + } + static long long overheadTotal(); + // Set a category's live value directly, for gauge-style subsystems whose size // is recomputed as an absolute (rather than tracked via alloc/free deltas). // Also advances the peak. Not for use from a signal handler. diff --git a/ddprof-lib/src/main/cpp/profiler.cpp b/ddprof-lib/src/main/cpp/profiler.cpp index 7f9f208bc0..63f9a667fe 100644 --- a/ddprof-lib/src/main/cpp/profiler.cpp +++ b/ddprof-lib/src/main/cpp/profiler.cpp @@ -1892,7 +1892,8 @@ void Profiler::updateNativeLibMemStats() { // cache). memoryUsage() is a recomputed gauge; read it once and publish the // counters plus the NativeMem gauge (NATIVE_SYMBOLS) as absolutes. const CodeCacheArray& native_libs = _libs->native_libs(); - long long usage = (long long)native_libs.memoryUsage(); + long long symbols_overhead = 0; + long long usage = (long long)native_libs.memoryUsage(&symbols_overhead); Counters::set(CODECACHE_NATIVE_COUNT, native_libs.count()); Counters::set(CODECACHE_NATIVE_SIZE_BYTES, usage); // The runtime-stubs cache is a distinct HotSpot cache, not the native-symbol @@ -1901,6 +1902,11 @@ void Profiler::updateNativeLibMemStats() { Counters::set(CODECACHE_RUNTIME_STUBS_SIZE_BYTES, JVMSupport::runtimeStubsMemoryUsage()); NativeMem::setLive(NM_NATIVE_SYMBOLS, usage); + // Measured, not assumed: the symbol tables are many short name strings, so + // the allocator's rounding and per-chunk header are a material fraction of + // their real cost. Gauge-style to match setLive above -- memoryUsage() + // recomputes an absolute rather than tracking deltas. + NativeMem::setOverhead(NM_NATIVE_SYMBOLS, symbols_overhead); } Error Profiler::dump(const char *path, const int length) { diff --git a/ddprof-lib/src/main/cpp/stringDictionary.h b/ddprof-lib/src/main/cpp/stringDictionary.h index b5572b6237..3295a95acb 100644 --- a/ddprof-lib/src/main/cpp/stringDictionary.h +++ b/ddprof-lib/src/main/cpp/stringDictionary.h @@ -86,7 +86,11 @@ class StringArena { // gates the diagnostic DICTIONARY_BYTES counter). Keys are bump-allocated // inside these chunks, so they must not be counted separately. if (c != nullptr) { - NativeMem::record(NM_DICTIONARY, (long long)sizeof(Chunk)); + // recordAlloc measures the allocator overhead rather than assuming + // it. These chunks are ~512 KB, so glibc serves them by mmap and the + // overhead is ~16 bytes -- about 0.003 %, not the double-digit + // percentage a small-allocation-derived blanket factor would imply. + NativeMem::recordAlloc(NM_DICTIONARY, c, sizeof(Chunk)); } return c; } @@ -143,8 +147,9 @@ class StringArena { Chunk* c = _first; while (c) { Chunk* n = c->next; + // Before free(): the overhead is read back off the live chunk. + NativeMem::recordFreeBefore(NM_DICTIONARY, c, sizeof(Chunk)); free(c); - NativeMem::record(NM_DICTIONARY, -(long long)sizeof(Chunk)); c = n; } } @@ -182,8 +187,9 @@ class StringArena { int freed = 0; while (c) { Chunk* n = c->next; + // Before free(): the overhead is read back off the live chunk. + NativeMem::recordFreeBefore(NM_DICTIONARY, c, sizeof(Chunk)); free(c); - NativeMem::record(NM_DICTIONARY, -(long long)sizeof(Chunk)); c = n; ++freed; } diff --git a/ddprof-lib/src/test/cpp/mallocArenaStats_ut.cpp b/ddprof-lib/src/test/cpp/mallocArenaStats_ut.cpp new file mode 100644 index 0000000000..f6931e520e --- /dev/null +++ b/ddprof-lib/src/test/cpp/mallocArenaStats_ut.cpp @@ -0,0 +1,127 @@ +/* + * Copyright 2026, Datadog, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include + +#if defined(__linux__) && defined(__GLIBC__) && \ + (__GLIBC__ > 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 33)) + #include + #define DD_HAVE_MALLINFO2 1 +#endif + +// These tests validate the *premise* of the MALLOC_FREE_HELD_BYTES counter +// rather than its plumbing: that allocation churn of the shape the profiler +// actually produces leaves memory stranded inside glibc's arenas, retained +// rather than returned to the OS. +// +// The shape is taken from a measured JFR chunk flush: +// StringDictionaryBuffer::insert_with_id allocates 26,450 blocks of exactly +// 6,144 bytes (sizeof(SBTable)) during a single flush, then frees them. If that +// churn strands a significant number of free chunks, it is a candidate +// explanation for memory the profiler's own per-category counters cannot see -- +// and, crucially, it is process-wide allocator behaviour rather than profiler +// memory, which is why it is reported as a separate counter and not as an NM_* +// category (see nativeMem.h). + +#ifdef DD_HAVE_MALLINFO2 + +static const size_t SBTABLE_SIZE = 6144; // sizeof(SBTable), measured +static const int FLUSH_BLOCKS = 26450; // allocations in one observed flush + +// Churn alone does NOT strand memory. 26,450 blocks allocated contiguously and +// then all freed coalesce into one large free region at the arena top, which +// glibc returns to the OS -- measured: arena 163 MB back down to 905 KB. So a +// flush burst is not, by itself, an explanation for retained memory. The next +// test shows what is actually required. +TEST(MallocArenaStats, ContiguousChurnIsReturnedToTheOS) { + { + std::vector warm; + for (int i = 0; i < 1000; i++) warm.push_back(malloc(SBTABLE_SIZE)); + for (void *p : warm) free(p); + } + + struct mallinfo2 before = mallinfo2(); + + std::vector blocks; + blocks.reserve(FLUSH_BLOCKS); + for (int i = 0; i < FLUSH_BLOCKS; i++) { + void *p = malloc(SBTABLE_SIZE); + ASSERT_NE(nullptr, p); + *(volatile char *)p = 1; // touch it, as the real code would + blocks.push_back(p); + } + struct mallinfo2 peak = mallinfo2(); + for (void *p : blocks) free(p); + struct mallinfo2 after = mallinfo2(); + + EXPECT_GE(peak.uordblks - before.uordblks, + (size_t)FLUSH_BLOCKS * SBTABLE_SIZE) + << "in-use accounting did not reflect the held blocks"; + EXPECT_LT(after.uordblks, peak.uordblks); + + // The arena shrinks back: the coalesced free region is returned, so this + // pattern leaves nothing meaningful in the free-but-held term. + EXPECT_LT((long long)after.arena, (long long)peak.arena / 2) + << "expected the coalesced top region to be returned to the OS"; + + printf("[ARENA] before: arena=%zu in_use=%zu free_held=%zu trimmable=%zu\n", + before.arena, before.uordblks, before.fordblks, before.keepcost); + printf("[ARENA] peak: arena=%zu in_use=%zu free_held=%zu trimmable=%zu\n", + peak.arena, peak.uordblks, peak.fordblks, peak.keepcost); + printf("[ARENA] after: arena=%zu in_use=%zu free_held=%zu trimmable=%zu\n", + after.arena, after.uordblks, after.fordblks, after.keepcost); + printf("[ARENA] free_held change: %+.2f MiB (signed)\n", + (double)((long long)after.fordblks - (long long)before.fordblks) / + (1024 * 1024)); +} + +// Stranding requires LIVE allocations interleaved among the freed ones: they pin +// the region so the free chunks cannot coalesce to the top and be returned. +// Measured here: 117.5 MiB retained, of which malloc_trim reclaims 0.03 MiB. +// +// This is why the counter separates `keepcost` (the trimmable top block) from +// total free-but-held, and why arena slack is a property of the whole process's +// allocation *pattern* rather than of any one subsystem's allocations. +TEST(MallocArenaStats, InterleavedSurvivorsStrandFreeChunksBeyondTrim) { + std::vector survivors, churn; + for (int i = 0; i < 20000; i++) { + void *a = malloc(SBTABLE_SIZE); + void *b = malloc(SBTABLE_SIZE); + ASSERT_NE(nullptr, a); + ASSERT_NE(nullptr, b); + *(volatile char *)a = 1; + *(volatile char *)b = 1; + survivors.push_back(a); // kept live, so it pins the region + churn.push_back(b); // freed below + } + for (void *p : churn) free(p); + + struct mallinfo2 pre_trim = mallinfo2(); + malloc_trim(0); + struct mallinfo2 post_trim = mallinfo2(); + + EXPECT_GT(post_trim.fordblks, 0u) + << "trim reclaimed everything; interleaving failed to strand chunks"; + + printf("[TRIM] free_held before trim: %.2f MiB, after trim: %.2f MiB " + "(reclaimed %.2f MiB)\n", + (double)pre_trim.fordblks / (1024 * 1024), + (double)post_trim.fordblks / (1024 * 1024), + (double)((long long)pre_trim.fordblks - (long long)post_trim.fordblks) / + (1024 * 1024)); + + for (void *p : survivors) free(p); +} + +#else + +TEST(MallocArenaStats, SkippedWithoutMallinfo2) { + GTEST_SKIP() << "mallinfo2 requires glibc 2.33+; arena counters report zero " + "on this platform"; +} + +#endif diff --git a/ddprof-lib/src/test/cpp/mallocFootprint_ut.cpp b/ddprof-lib/src/test/cpp/mallocFootprint_ut.cpp new file mode 100644 index 0000000000..3d6295c936 --- /dev/null +++ b/ddprof-lib/src/test/cpp/mallocFootprint_ut.cpp @@ -0,0 +1,184 @@ +/* + * Copyright 2026, Datadog, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include +#include "mallocFootprint.h" +#include "nativeMem.h" +#include "countingAllocator.h" + +// Allocator overhead is a function of PER-ALLOCATION size, not of total bytes. +// That is the whole reason a single blanket multiplier cannot be correct across +// categories: the same total can be one large chunk paying ~0 % or thousands of +// small nodes paying ~17 %. These tests pin that, and pin that the profiler now +// measures it rather than assuming a factor. + +// True when the allocator in force exposes enough for overhead to be observed +// at all. Sanitizer builds substitute their own allocator, whose usable size is +// the requested size and which lays blocks out so the header probe yields 0 -- +// so it correctly reports no overhead, and the magnitude assertions below have +// nothing to measure. Reporting zero there is the honest answer, not a bug. +static bool allocatorExposesOverhead() { + void *p = malloc(96); + if (p == NULL) return false; + size_t oh = MallocFootprint::overheadOf(p, 96); + free(p); + return oh > 0; +} + +class MallocFootprintTest : public ::testing::Test { +protected: + long long _baseline[NM_NUM_CATEGORIES]; + void SetUp() override { + for (int c = 0; c < NM_NUM_CATEGORIES; c++) { + _baseline[c] = NativeMem::live((NativeMemCategory)c); + } + NativeMem::reset(); + } + void TearDown() override { + NativeMem::reset(); + for (int c = 0; c < NM_NUM_CATEGORIES; c++) { + NativeMem::setLive((NativeMemCategory)c, _baseline[c]); + } + } +}; + +// The header must be discovered, not hardcoded: an LD_PRELOAD'd tcmalloc or +// jemalloc adds none while __GLIBC__ stays defined. On glibc the probe is +// expected to find 8; anywhere else it must at least be sane. +TEST_F(MallocFootprintTest, HeaderProbeReturnsSaneValue) { + size_t h = MallocFootprint::headerBytes(); + EXPECT_LE(h, 64u) << "probe returned an implausible header size"; + printf("[FOOTPRINT] probed per-chunk header = %zu bytes\n", h); + + // Cross-check against an independently measured case: a 96-byte request + // occupies 112 bytes on glibc (104 usable + 8 header). + void *p = malloc(96); + ASSERT_NE(nullptr, p); + size_t fp = MallocFootprint::of(p, 96); + printf("[FOOTPRINT] request=96 usable=%zu footprint=%zu\n", + malloc_usable_size(p), fp); + EXPECT_GE(fp, 96u) << "footprint cannot be below the requested size"; + free(p); +} + +// The central claim: overhead as a FRACTION collapses as allocation size grows. +// A blanket multiplier calibrated on small allocations therefore over-charges +// large ones badly -- which is precisely the case for the 512 KB string-arena +// chunks and the JFR buffers. +TEST_F(MallocFootprintTest, OverheadFractionCollapsesWithAllocationSize) { + if (!allocatorExposesOverhead()) { + GTEST_SKIP() << "allocator exposes no overhead (sanitizer build)"; + } + struct { size_t size; const char *what; } cases[] = { + {96, "MethodMap node"}, + {6 * 1024, "SBTable"}, + {512 * 1024, "string-arena chunk"}, + {2 * 1024 * 1024, "large buffer"}, + }; + double small_pct = -1, large_pct = -1; + for (auto &c : cases) { + void *p = malloc(c.size); + ASSERT_NE(nullptr, p); + size_t oh = MallocFootprint::overheadOf(p, c.size); + double pct = 100.0 * (double)oh / (double)c.size; + printf("[FOOTPRINT] %-20s request=%-8zu overhead=%-4zu (%.4f %%)\n", + c.what, c.size, oh, pct); + if (c.size == 96) small_pct = pct; + if (c.size == 512 * 1024) large_pct = pct; + free(p); + } + ASSERT_GE(small_pct, 0); + ASSERT_GE(large_pct, 0); + // A small node pays a double-digit percentage... + EXPECT_GT(small_pct, 10.0) + << "a 96-byte allocation should pay a double-digit overhead percentage"; + // ...while a 512 KB chunk pays well under one percent. A blanket factor + // calibrated on the former over-charges the latter by more than an order of + // magnitude, which is the quantitative reason one factor cannot serve both. + EXPECT_LT(large_pct, 1.0) + << "a 512 KB chunk must not be charged a double-digit percentage"; + EXPECT_GT(small_pct, large_pct * 10); + + // Large-allocation overhead is not even a function of the request size. + // glibc serves large requests either from an arena or by mmap, and the + // crossover is a DYNAMIC threshold that rises as large blocks are freed. So + // the identical 512 KB request measures 4088 bytes of overhead (mmap: page + // rounding on the final page) in one allocator state and 16 bytes (arena: + // header only) in another, purely as a function of allocation history. + // + // This is the decisive argument for measuring rather than computing: no + // arithmetic formula over the requested size -- align16(S + 8) or anything + // else -- can be right for large allocations, because the answer depends on + // state the caller cannot see. + size_t seen_min = (size_t)-1, seen_max = 0; + for (int i = 0; i < 8; i++) { + void *p = malloc(512 * 1024); + ASSERT_NE(nullptr, p); + size_t oh = MallocFootprint::overheadOf(p, 512 * 1024); + if (oh < seen_min) seen_min = oh; + if (oh > seen_max) seen_max = oh; + free(p); + } + printf("[FOOTPRINT] identical 512 KB request, 8 attempts: overhead %zu..%zu\n", + seen_min, seen_max); + // Whatever the allocator chose, it must stay far below the double-digit + // percentage a small-allocation-derived factor would charge. + EXPECT_LT(100.0 * (double)seen_max / (double)(512 * 1024), 1.0); +} + +// CountingAllocator must populate the overhead gauge while leaving the live +// gauge as logical bytes, and both must return to zero on release. +TEST_F(MallocFootprintTest, CountingAllocatorTracksOverheadSeparately) { + if (!allocatorExposesOverhead()) { + GTEST_SKIP() << "allocator exposes no overhead (sanitizer build)"; + } + using Map = std::map, + CountingAllocator, + NM_METHOD_MAP>>; + { + Map m; + for (long i = 0; i < 2000; i++) { + m[i * 7919] = i; + } + long long live = NativeMem::live(NM_METHOD_MAP); + long long oh = NativeMem::overhead(NM_METHOD_MAP); + printf("[FOOTPRINT] map of 2000 nodes: live=%lld overhead=%lld (%.1f %%)\n", + live, oh, 100.0 * (double)oh / (double)live); + EXPECT_GT(live, 0); + EXPECT_GT(oh, 0) << "overhead gauge was not populated"; + // Nodes are small, so overhead is a substantial fraction -- not a rounding + // error, and not something a logical-bytes-only counter can see. + EXPECT_GT(100.0 * (double)oh / (double)live, 1.0); + } + EXPECT_EQ(0, NativeMem::live(NM_METHOD_MAP)) + << "live gauge did not return to zero"; + EXPECT_EQ(0, NativeMem::overhead(NM_METHOD_MAP)) + << "overhead gauge did not return to zero; recordFreeBefore is unbalanced"; +} + +// recordAlloc/recordFreeBefore must balance exactly, including for a size whose +// rounding is non-trivial. +TEST_F(MallocFootprintTest, RecordAllocAndFreeBalance) { + const size_t SZ = 100; // not a multiple of the alignment quantum + void *ptrs[500]; + for (int i = 0; i < 500; i++) { + ptrs[i] = malloc(SZ); + ASSERT_NE(nullptr, ptrs[i]); + NativeMem::recordAlloc(NM_MISC, ptrs[i], SZ); + } + EXPECT_EQ((long long)(500 * SZ), NativeMem::live(NM_MISC)); + if (allocatorExposesOverhead()) { + EXPECT_GT(NativeMem::overhead(NM_MISC), 0); + } + + for (int i = 0; i < 500; i++) { + NativeMem::recordFreeBefore(NM_MISC, ptrs[i], SZ); + free(ptrs[i]); + } + EXPECT_EQ(0, NativeMem::live(NM_MISC)); + EXPECT_EQ(0, NativeMem::overhead(NM_MISC)); +}