From 172a8896058a2365077816ecd29e21741fa0eb83 Mon Sep 17 00:00:00 2001 From: Roman Kennke Date: Thu, 27 Aug 2026 18:12:05 +0000 Subject: [PATCH 1/3] feat(nativemem): report process-wide malloc arena state from glibc's accounting Adds five counters sampled from mallinfo2() on the JFR flush path: malloc_arena_bytes, malloc_in_use_bytes, malloc_free_held_bytes, malloc_trimmable_bytes, malloc_mmap_bytes. Motivation: memory reconciliation against process RSS currently applies a x1.17 "chunk overhead" factor and treats free-but-held arena pages as an unmeasured residual term, both borrowed from a different workload. Both are properties of the ALLOCATOR, not of the profiler -- under tcmalloc or jemalloc they behave differently -- so they belong in reported counters rather than folded into a "profiler cost" figure or estimated by analogy. These are deliberately NOT NM_* categories. nativeMem.h documents that the per-category gauges partition the profiler's own allocations (sum(category) == total, no double counting); free-but-held arena pages are mostly other subsystems' chunks stranded by interleaving and cannot be attributed to any profiler allocation, so adding them as a category would break that invariant. Reported separately, they bracket the profiler's contribution: lower bound = the per-category counters, upper bound = those plus process-wide free-but-held. keepcost is reported separately from fordblks because it distinguishes glibc's trim-threshold retention (reclaimable by malloc_trim) from genuine fragmentation (not reclaimable). Flush path only: mallinfo2() walks every arena taking each arena lock, so it is neither cheap nor async-signal-safe and must never be reached from the sampling signal handler. Once per JFR chunk is negligible. Guarded to glibc 2.33+; the older int-based mallinfo() silently truncates past 2 GiB so it is not a usable fallback, and the counters simply read zero on musl and macOS. The tests validate the premise rather than the plumbing, and correct an assumption I held while writing them: - Contiguous churn of the shape a real flush produces (26,450 blocks of 6,144 B = sizeof(SBTable), from StringDictionaryBuffer::insert_with_id) allocated and then all freed is FULLY RETURNED to the OS -- arena 163 MB back down to 905 KB. Churn alone strands nothing. - Stranding requires live allocations INTERLEAVED among the freed ones, which pin the region so free chunks cannot coalesce to the arena top. Measured: 117.5 MiB retained, of which malloc_trim reclaims 0.03 MiB. That distinction matters for interpreting flush-time memory steps: a burst is not self-evidently a stranding mechanism, and whether it strands depends on what else is live at the time. Co-Authored-By: Claude Opus 5 (1M context) --- ddprof-lib/src/main/cpp/counters.h | 14 ++ ddprof-lib/src/main/cpp/flightRecorder.cpp | 43 ++++++ ddprof-lib/src/main/cpp/flightRecorder.h | 3 + .../src/test/cpp/mallocArenaStats_ut.cpp | 127 ++++++++++++++++++ 4 files changed, 187 insertions(+) create mode 100644 ddprof-lib/src/test/cpp/mallocArenaStats_ut.cpp diff --git a/ddprof-lib/src/main/cpp/counters.h b/ddprof-lib/src/main/cpp/counters.h index 430f379c3e..20498d4640 100644 --- a/ddprof-lib/src/main/cpp/counters.h +++ b/ddprof-lib/src/main/cpp/counters.h @@ -56,6 +56,20 @@ X(NATIVE_MEM_LIVE_BYTES, "native_mem_live_bytes") \ X(NATIVE_MEM_MAX_BYTES, "native_mem_max_bytes") \ X(NATIVE_MEM_AVG_BYTES, "native_mem_avg_bytes") \ + /* Process-wide malloc arena state, from glibc's own accounting. These are \ + * NOT profiler memory and deliberately are not NM_* categories: those must \ + * partition the profiler's own allocations (see nativeMem.h), and arena \ + * slack is mostly other subsystems' chunks stranded by interleaving, so it \ + * is not attributable to any profiler allocation. Reported so that the \ + * allocator's own overhead is visible rather than folded into a "profiler \ + * cost" figure -- it differs substantially between glibc, tcmalloc and \ + * jemalloc. Sampled on the JFR flush path only: mallinfo2() walks every \ + * arena taking locks, so it is neither cheap nor async-signal-safe. */ \ + X(MALLOC_ARENA_BYTES, "malloc_arena_bytes") \ + X(MALLOC_IN_USE_BYTES, "malloc_in_use_bytes") \ + X(MALLOC_FREE_HELD_BYTES, "malloc_free_held_bytes") \ + X(MALLOC_TRIMMABLE_BYTES, "malloc_trimmable_bytes") \ + X(MALLOC_MMAP_BYTES, "malloc_mmap_bytes") \ X(THREAD_IDS_COUNT, "thread_ids_count") \ X(THREAD_NAMES_COUNT, "thread_names_count") \ X(THREAD_FILTER_PAGES, "thread_filter_pages") \ diff --git a/ddprof-lib/src/main/cpp/flightRecorder.cpp b/ddprof-lib/src/main/cpp/flightRecorder.cpp index 29156844cf..cbef027f62 100644 --- a/ddprof-lib/src/main/cpp/flightRecorder.cpp +++ b/ddprof-lib/src/main/cpp/flightRecorder.cpp @@ -7,6 +7,15 @@ #include #include +// mallinfo2() replaced the int-based mallinfo() in glibc 2.33; the older struct +// silently truncates past 2 GiB, so it is not a usable fallback for byte +// accounting. Absent on musl and macOS, where the arena counters stay zero. +#if defined(__linux__) && defined(__GLIBC__) && \ + (__GLIBC__ > 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 33)) + #include + #define DD_HAVE_MALLINFO2 1 +#endif + #include "buffers.h" #include "callTraceHashTable.h" #include "context.h" @@ -1809,12 +1818,46 @@ void Recording::writeLogLevels(Buffer *buf) { } } +// Snapshot the process-wide malloc arena state from glibc's own accounting. +// +// This is NOT profiler memory. Free-but-held arena pages are mostly other +// subsystems' chunks stranded by interleaving, so they cannot be attributed to +// any profiler allocation -- reporting them separately keeps them out of the +// per-category figures while still making the allocator's overhead visible. +// The numbers also differ substantially between glibc, tcmalloc and jemalloc, +// which is exactly what a reader comparing allocators needs to see. +// +// Only safe on the flush path: mallinfo2() walks every arena taking each arena +// lock, so it is neither cheap nor async-signal-safe. Must never be reached +// from the sampling signal handler. Once per JFR chunk is negligible. +void Recording::updateMallocArenaStats() { +#ifdef DD_HAVE_MALLINFO2 + struct mallinfo2 mi = mallinfo2(); + // arena: bytes obtained from the OS via brk, excluding mmap'd chunks + // uordblks: bytes currently handed out to callers + // fordblks: bytes free but retained in the arenas -- the waste term + // keepcost: the trimmable top block, i.e. what malloc_trim could return; + // separating it distinguishes trim-threshold policy from genuine + // fragmentation, which trimming cannot reclaim + // hblkhd: bytes in mmap'd chunks, which are returned to the OS on free + Counters::set(MALLOC_ARENA_BYTES, (long long)mi.arena); + Counters::set(MALLOC_IN_USE_BYTES, (long long)mi.uordblks); + Counters::set(MALLOC_FREE_HELD_BYTES, (long long)mi.fordblks); + Counters::set(MALLOC_TRIMMABLE_BYTES, (long long)mi.keepcost); + Counters::set(MALLOC_MMAP_BYTES, (long long)mi.hblkhd); +#endif +} + void Recording::updateNativeMemStats() { // Refresh the moving-window averages and the observed total peak. Per-category // peaks are maintained precisely at allocation time, so they are not sampled // here; the total peak is bracketed instead (see writeNativeMem). NativeMem::sample(); + // Process-wide allocator state, sampled at the same instant as the + // per-category figures so the two can be compared coherently. + updateMallocArenaStats(); + // Mirror the totals into the flat counter table so they flow out through the // existing counter path (JFR T_DATADOG_COUNTER events and the JNI debug // counters). NATIVE_MEM_MAX_BYTES carries the upper bound on the total peak diff --git a/ddprof-lib/src/main/cpp/flightRecorder.h b/ddprof-lib/src/main/cpp/flightRecorder.h index 2922f368b2..3ed004af61 100644 --- a/ddprof-lib/src/main/cpp/flightRecorder.h +++ b/ddprof-lib/src/main/cpp/flightRecorder.h @@ -324,6 +324,9 @@ class Recording { void writeCounters(Buffer *buf); void updateNativeMemStats(); + // Process-wide malloc arena state (glibc mallinfo2). Flush path only -- + // takes every arena lock, so not async-signal-safe. No-op off glibc 2.33+. + void updateMallocArenaStats(); void writeNativeMem(Buffer *buf); void writeUnwindFailures(Buffer *buf); diff --git a/ddprof-lib/src/test/cpp/mallocArenaStats_ut.cpp b/ddprof-lib/src/test/cpp/mallocArenaStats_ut.cpp new file mode 100644 index 0000000000..f6931e520e --- /dev/null +++ b/ddprof-lib/src/test/cpp/mallocArenaStats_ut.cpp @@ -0,0 +1,127 @@ +/* + * Copyright 2026, Datadog, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include + +#if defined(__linux__) && defined(__GLIBC__) && \ + (__GLIBC__ > 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 33)) + #include + #define DD_HAVE_MALLINFO2 1 +#endif + +// These tests validate the *premise* of the MALLOC_FREE_HELD_BYTES counter +// rather than its plumbing: that allocation churn of the shape the profiler +// actually produces leaves memory stranded inside glibc's arenas, retained +// rather than returned to the OS. +// +// The shape is taken from a measured JFR chunk flush: +// StringDictionaryBuffer::insert_with_id allocates 26,450 blocks of exactly +// 6,144 bytes (sizeof(SBTable)) during a single flush, then frees them. If that +// churn strands a significant number of free chunks, it is a candidate +// explanation for memory the profiler's own per-category counters cannot see -- +// and, crucially, it is process-wide allocator behaviour rather than profiler +// memory, which is why it is reported as a separate counter and not as an NM_* +// category (see nativeMem.h). + +#ifdef DD_HAVE_MALLINFO2 + +static const size_t SBTABLE_SIZE = 6144; // sizeof(SBTable), measured +static const int FLUSH_BLOCKS = 26450; // allocations in one observed flush + +// Churn alone does NOT strand memory. 26,450 blocks allocated contiguously and +// then all freed coalesce into one large free region at the arena top, which +// glibc returns to the OS -- measured: arena 163 MB back down to 905 KB. So a +// flush burst is not, by itself, an explanation for retained memory. The next +// test shows what is actually required. +TEST(MallocArenaStats, ContiguousChurnIsReturnedToTheOS) { + { + std::vector warm; + for (int i = 0; i < 1000; i++) warm.push_back(malloc(SBTABLE_SIZE)); + for (void *p : warm) free(p); + } + + struct mallinfo2 before = mallinfo2(); + + std::vector blocks; + blocks.reserve(FLUSH_BLOCKS); + for (int i = 0; i < FLUSH_BLOCKS; i++) { + void *p = malloc(SBTABLE_SIZE); + ASSERT_NE(nullptr, p); + *(volatile char *)p = 1; // touch it, as the real code would + blocks.push_back(p); + } + struct mallinfo2 peak = mallinfo2(); + for (void *p : blocks) free(p); + struct mallinfo2 after = mallinfo2(); + + EXPECT_GE(peak.uordblks - before.uordblks, + (size_t)FLUSH_BLOCKS * SBTABLE_SIZE) + << "in-use accounting did not reflect the held blocks"; + EXPECT_LT(after.uordblks, peak.uordblks); + + // The arena shrinks back: the coalesced free region is returned, so this + // pattern leaves nothing meaningful in the free-but-held term. + EXPECT_LT((long long)after.arena, (long long)peak.arena / 2) + << "expected the coalesced top region to be returned to the OS"; + + printf("[ARENA] before: arena=%zu in_use=%zu free_held=%zu trimmable=%zu\n", + before.arena, before.uordblks, before.fordblks, before.keepcost); + printf("[ARENA] peak: arena=%zu in_use=%zu free_held=%zu trimmable=%zu\n", + peak.arena, peak.uordblks, peak.fordblks, peak.keepcost); + printf("[ARENA] after: arena=%zu in_use=%zu free_held=%zu trimmable=%zu\n", + after.arena, after.uordblks, after.fordblks, after.keepcost); + printf("[ARENA] free_held change: %+.2f MiB (signed)\n", + (double)((long long)after.fordblks - (long long)before.fordblks) / + (1024 * 1024)); +} + +// Stranding requires LIVE allocations interleaved among the freed ones: they pin +// the region so the free chunks cannot coalesce to the top and be returned. +// Measured here: 117.5 MiB retained, of which malloc_trim reclaims 0.03 MiB. +// +// This is why the counter separates `keepcost` (the trimmable top block) from +// total free-but-held, and why arena slack is a property of the whole process's +// allocation *pattern* rather than of any one subsystem's allocations. +TEST(MallocArenaStats, InterleavedSurvivorsStrandFreeChunksBeyondTrim) { + std::vector survivors, churn; + for (int i = 0; i < 20000; i++) { + void *a = malloc(SBTABLE_SIZE); + void *b = malloc(SBTABLE_SIZE); + ASSERT_NE(nullptr, a); + ASSERT_NE(nullptr, b); + *(volatile char *)a = 1; + *(volatile char *)b = 1; + survivors.push_back(a); // kept live, so it pins the region + churn.push_back(b); // freed below + } + for (void *p : churn) free(p); + + struct mallinfo2 pre_trim = mallinfo2(); + malloc_trim(0); + struct mallinfo2 post_trim = mallinfo2(); + + EXPECT_GT(post_trim.fordblks, 0u) + << "trim reclaimed everything; interleaving failed to strand chunks"; + + printf("[TRIM] free_held before trim: %.2f MiB, after trim: %.2f MiB " + "(reclaimed %.2f MiB)\n", + (double)pre_trim.fordblks / (1024 * 1024), + (double)post_trim.fordblks / (1024 * 1024), + (double)((long long)pre_trim.fordblks - (long long)post_trim.fordblks) / + (1024 * 1024)); + + for (void *p : survivors) free(p); +} + +#else + +TEST(MallocArenaStats, SkippedWithoutMallinfo2) { + GTEST_SKIP() << "mallinfo2 requires glibc 2.33+; arena counters report zero " + "on this platform"; +} + +#endif From 265227dc5542a0c78ce20bf97150ebb97f63422c Mon Sep 17 00:00:00 2001 From: Roman Kennke Date: Fri, 28 Aug 2026 09:56:46 +0000 Subject: [PATCH 2/3] feat(nativemem): measure allocator overhead per allocation instead of assuming it Adds native_mem_chunk_overhead_bytes.: the rounding-plus-header cost that RSS pays on top of the logical bytes the NM_* gauges record. Measured from malloc_usable_size() at each allocation, kept in a separate gauge so native_mem_live_bytes stays directly comparable to sizeof() arithmetic. This replaces a blanket x1.17 multiplier applied during RSS reconciliation. That factor was measured on one workload (~650,000 live allocations averaging ~83 B) and then applied to categories with completely different size mixes. Overhead is a function of PER-ALLOCATION size, so no single factor can serve them: 96 B (MethodMap node) overhead 16 B 16.67 % <- x1.17 is right here 6 KB (SBTable) overhead 16 B 0.26 % 512 KB (string arena) overhead 16 B 0.003 % (arena-served) 2 MB (large buffer) overhead 4088 B 0.19 % Charging the 512 KB string-arena chunks and the JFR buffers 17 % over-credits the explained total by over 1 MiB against a residual of only a few MiB. The header size is probed at first use rather than hardcoded, because the allocator in force is not knowable at compile time: an LD_PRELOAD'd tcmalloc or jemalloc adds no per-object header while __GLIBC__ stays defined, so assuming glibc's 8 bytes would invent overhead that does not exist. The probe allocates same-size blocks and takes the smallest positive address stride (stride = usable + header); implausible results fall back to 0, reporting only the rounding we can see rather than guessing. Wired into CountingAllocator (all STL-node categories) and StringArena's chunk alloc/free. Call sites still using plain record() contribute zero here, which is why the gauge is additive rather than a correction to the live figure -- notably NM_NATIVE_SYMBOLS, which is a computed sum of logical sizes with no pointer available at record time and needs a different approach. Two findings from the tests, both correcting assumptions I had made: - The 512 KB chunk does NOT pay a negligible fixed 16 bytes as predicted. The identical request measures 4088 bytes of overhead in one allocator state and 16 in another, because glibc's mmap threshold is DYNAMIC and rises as large blocks are freed. Large-allocation overhead is therefore not a function of request size at all, which rules out any arithmetic formula -- including the align16(S + 8) approach originally proposed -- and settles that it must be measured. - Sanitizer builds substitute an allocator that reports no usable-size slack and defeats the header probe, so overhead reads 0. That is the honest answer for such an allocator, so the magnitude assertions skip rather than fail. Release, debug, ASan and TSan all pass. Co-Authored-By: Claude Opus 5 (1M context) --- ddprof-lib/src/main/cpp/countingAllocator.h | 10 +- ddprof-lib/src/main/cpp/flightRecorder.cpp | 8 + ddprof-lib/src/main/cpp/mallocFootprint.h | 113 +++++++++++ ddprof-lib/src/main/cpp/nativeMem.cpp | 13 ++ ddprof-lib/src/main/cpp/nativeMem.h | 42 ++++ ddprof-lib/src/main/cpp/stringDictionary.h | 12 +- .../src/test/cpp/mallocFootprint_ut.cpp | 184 ++++++++++++++++++ 7 files changed, 377 insertions(+), 5 deletions(-) create mode 100644 ddprof-lib/src/main/cpp/mallocFootprint.h create mode 100644 ddprof-lib/src/test/cpp/mallocFootprint_ut.cpp diff --git a/ddprof-lib/src/main/cpp/countingAllocator.h b/ddprof-lib/src/main/cpp/countingAllocator.h index e31aae3b85..6658b19672 100644 --- a/ddprof-lib/src/main/cpp/countingAllocator.h +++ b/ddprof-lib/src/main/cpp/countingAllocator.h @@ -25,12 +25,18 @@ class CountingAllocator { T *allocate(std::size_t n) { T *p = static_cast(::operator new(n * sizeof(T))); - NativeMem::record(Cat, (long long)(n * sizeof(T))); + // recordAlloc rather than record: STL nodes are small, so the allocator's + // rounding and per-chunk header are a large fraction of their real cost + // (a 96-byte MethodMap node occupies 112 bytes -- 16.7 %). Logical bytes + // still land in the live gauge; the extra goes to the overhead gauge. + NativeMem::recordAlloc(Cat, p, n * sizeof(T)); return p; } void deallocate(T *p, std::size_t n) noexcept { - NativeMem::record(Cat, -(long long)(n * sizeof(T))); + // Must run before operator delete: the overhead is read back off the live + // chunk. + NativeMem::recordFreeBefore(Cat, p, n * sizeof(T)); ::operator delete(p); } diff --git a/ddprof-lib/src/main/cpp/flightRecorder.cpp b/ddprof-lib/src/main/cpp/flightRecorder.cpp index cbef027f62..f20ff31218 100644 --- a/ddprof-lib/src/main/cpp/flightRecorder.cpp +++ b/ddprof-lib/src/main/cpp/flightRecorder.cpp @@ -1901,6 +1901,14 @@ void Recording::writeNativeMem(Buffer *buf) { {"native_mem_live_bytes.", NativeMem::live(cat)}, {"native_mem_avg_bytes.", NativeMem::avg(cat)}, {"native_mem_max_bytes.", NativeMem::max(cat)}, + // Measured allocator overhead on the live allocations -- rounding to + // the size quantum plus the per-chunk header. Reported separately so + // native_mem_live_bytes stays comparable to sizeof() arithmetic, and so + // that reconciliation against RSS can add a measured figure instead of + // multiplying by a factor derived from some other workload's + // allocation-size mix. Zero for categories whose call sites still use + // record() rather than recordAlloc(). + {"native_mem_chunk_overhead_bytes.", NativeMem::overhead(cat)}, }; for (const auto &m : metrics) { char label[64]; diff --git a/ddprof-lib/src/main/cpp/mallocFootprint.h b/ddprof-lib/src/main/cpp/mallocFootprint.h new file mode 100644 index 0000000000..5607411fb3 --- /dev/null +++ b/ddprof-lib/src/main/cpp/mallocFootprint.h @@ -0,0 +1,113 @@ +/* + * Copyright 2026, Datadog, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ +#ifndef _MALLOCFOOTPRINT_H +#define _MALLOCFOOTPRINT_H + +#include + +#if defined(__linux__) + #include + #define DD_HAVE_MALLOC_USABLE_SIZE 1 +#elif defined(__APPLE__) + #include + #define DD_HAVE_MALLOC_SIZE 1 +#endif + +// Real resident cost of a heap allocation, as opposed to the size that was +// requested. +// +// The profiler's NM_* gauges record requested (logical) bytes, which is what +// makes them comparable against sizeof() arithmetic. RSS, however, is paid in +// allocator chunks: the request is rounded up to an alignment quantum and +// carries a per-chunk header. Reconciliation previously multiplied logical bytes +// by a single blanket factor measured on one workload, which is wrong whenever +// the allocation-size mix differs -- overhead is a function of *per-allocation* +// size, not of total bytes. A 512 KB chunk pays ~0.003 %; a 96-byte tree node +// pays 16.7 %. +// +// This measures it instead: usable size comes from the allocator, and the +// per-chunk header is probed once at first use rather than assumed. +class MallocFootprint { +private: + // Determined empirically: allocate several same-size blocks and take the + // smallest positive address stride between any two. That stride is + // usable + header, so header = stride - usable. + // + // Probed rather than hardcoded because the allocator in force at runtime is + // not knowable at compile time -- an LD_PRELOAD'd tcmalloc or jemalloc leaves + // __GLIBC__ defined while adding no per-object header at all (their metadata + // is out of band). Assuming glibc's 8 bytes would then invent overhead that + // does not exist, at one allocation's worth per allocation. + static size_t probeHeaderBytes() { +#ifdef DD_HAVE_MALLOC_USABLE_SIZE + const int N = 16; + const size_t SZ = 48; + void *p[N]; + for (int i = 0; i < N; i++) { + p[i] = malloc(SZ); + if (p[i] == NULL) { // give up cleanly rather than guess + for (int j = 0; j < i; j++) free(p[j]); + return 0; + } + } + size_t usable = malloc_usable_size(p[0]); + long best = 0; + for (int i = 0; i < N; i++) { + for (int j = 0; j < N; j++) { + long d = (char *)p[j] - (char *)p[i]; + if (d > 0 && (best == 0 || d < best)) { + best = d; + } + } + } + for (int i = 0; i < N; i++) free(p[i]); + long header = best - (long)usable; + // Sanity-bound the result. A negative or implausibly large value means the + // blocks were not laid out contiguously (a size-class allocator, or an + // arena boundary landed mid-probe), in which case 0 is the honest answer: + // report only the rounding we can see, and understate rather than invent. + if (header < 0 || header > 64) { + return 0; + } + return (size_t)header; +#else + return 0; +#endif + } + +public: + // Per-chunk header size for the allocator actually in force. Probed once; + // the C++11 function-local static makes initialisation thread-safe. NOT + // async-signal-safe (it allocates), so first use must not be from a signal + // handler -- every current call site is on a normal thread. + static size_t headerBytes() { + static const size_t header = probeHeaderBytes(); + return header; + } + + // Bytes this allocation actually costs: allocator-reported usable size plus + // the per-chunk header. Page rounding for large mmap'd chunks is already + // inside the usable size, so it must not be added again. + static size_t of(void *p, size_t requested) { + if (p == NULL) { + return 0; + } +#ifdef DD_HAVE_MALLOC_USABLE_SIZE + return malloc_usable_size(p) + headerBytes(); +#elif defined(DD_HAVE_MALLOC_SIZE) + return malloc_size(p) + headerBytes(); +#else + return requested; // no introspection available: report no overhead +#endif + } + + // Overhead alone -- the part RSS pays for that the logical counters miss. + static size_t overheadOf(void *p, size_t requested) { + size_t total = of(p, requested); + return total > requested ? total - requested : 0; + } +}; + +#endif // _MALLOCFOOTPRINT_H diff --git a/ddprof-lib/src/main/cpp/nativeMem.cpp b/ddprof-lib/src/main/cpp/nativeMem.cpp index df8cf3da58..3c39387959 100644 --- a/ddprof-lib/src/main/cpp/nativeMem.cpp +++ b/ddprof-lib/src/main/cpp/nativeMem.cpp @@ -5,6 +5,7 @@ #include "nativeMem.h" volatile long long NativeMem::_live[NM_NUM_CATEGORIES] = {}; +volatile long long NativeMem::_overhead[NM_NUM_CATEGORIES] = {}; volatile long long NativeMem::_max[NM_NUM_CATEGORIES] = {}; long long NativeMem::_window[NM_NUM_CATEGORIES][NativeMem::WINDOW] = {}; long long NativeMem::_total_window[NativeMem::WINDOW] = {}; @@ -14,6 +15,17 @@ long long NativeMem::_avg[NM_NUM_CATEGORIES] = {}; long long NativeMem::_total_avg = 0; long long NativeMem::_total_max_observed = 0; +long long NativeMem::overheadTotal() { + long long total = 0; + for (int c = 0; c < NM_NUM_CATEGORIES; c++) { + long long v = load(_overhead[c]); + if (v > 0) { + total += v; + } + } + return total; +} + long long NativeMem::liveTotal() { long long total = 0; for (int c = 0; c < NM_NUM_CATEGORIES; c++) { @@ -84,6 +96,7 @@ void NativeMem::reset() { for (int c = 0; c < NM_NUM_CATEGORIES; c++) { store(_live[c], (long long)0); store(_max[c], (long long)0); + store(_overhead[c], (long long)0); _avg[c] = 0; for (int i = 0; i < WINDOW; i++) { _window[c][i] = 0; diff --git a/ddprof-lib/src/main/cpp/nativeMem.h b/ddprof-lib/src/main/cpp/nativeMem.h index 2706f56297..af71f8fdaa 100644 --- a/ddprof-lib/src/main/cpp/nativeMem.h +++ b/ddprof-lib/src/main/cpp/nativeMem.h @@ -6,6 +6,7 @@ #define _NATIVEMEM_H #include "arch.h" +#include "mallocFootprint.h" #include // Physical native-memory categories used by the profiler's own allocations. @@ -60,6 +61,12 @@ class NativeMem { // Precise per-category high-water mark, maintained at allocation time by // record() so peaks that rise and fall between sample() ticks are still seen. static volatile long long _max[NM_NUM_CATEGORIES]; + // Allocator overhead on the live allocations: rounding to the size quantum + // plus the per-chunk header. Kept SEPARATE from _live rather than folded in, + // so _live stays directly comparable to sizeof() arithmetic while the amount + // RSS additionally pays stays visible. Maintained only by recordAlloc/ + // recordFreeBefore; call sites using plain record() contribute nothing here. + static volatile long long _overhead[NM_NUM_CATEGORIES]; // sample()-owned state; touched only from the single-threaded sampling path. static long long _window[NM_NUM_CATEGORIES][WINDOW]; @@ -94,6 +101,41 @@ class NativeMem { } } + // Record an allocation together with its measured allocator overhead. + // + // Prefer this over record() wherever the pointer is in hand: it keeps the + // logical byte count in _live (comparable to sizeof()) while accumulating the + // rounding-plus-header cost that RSS actually pays. Overhead is a function of + // per-allocation size, so it cannot be recovered later from a byte total -- + // a 512 KB chunk pays ~0.003 %, a 96-byte node 16.7 %. + // + // Not async-signal-safe on first call (the header probe allocates); every + // current call site runs on a normal thread. + static void recordAlloc(NativeMemCategory category, void *ptr, + size_t requested) { + record(category, (long long)requested); + if (ptr != NULL) { + atomicIncRelaxed(_overhead[category], + (long long)MallocFootprint::overheadOf(ptr, requested)); + } + } + + // Counterpart to recordAlloc. MUST be called before the pointer is freed -- + // the overhead is read back off the live chunk, which is unreadable afterwards. + static void recordFreeBefore(NativeMemCategory category, void *ptr, + size_t requested) { + if (ptr != NULL) { + atomicIncRelaxed(_overhead[category], + -(long long)MallocFootprint::overheadOf(ptr, requested)); + } + record(category, -(long long)requested); + } + + static long long overhead(NativeMemCategory category) { + return load(_overhead[category]); + } + static long long overheadTotal(); + // Set a category's live value directly, for gauge-style subsystems whose size // is recomputed as an absolute (rather than tracked via alloc/free deltas). // Also advances the peak. Not for use from a signal handler. diff --git a/ddprof-lib/src/main/cpp/stringDictionary.h b/ddprof-lib/src/main/cpp/stringDictionary.h index b5572b6237..3295a95acb 100644 --- a/ddprof-lib/src/main/cpp/stringDictionary.h +++ b/ddprof-lib/src/main/cpp/stringDictionary.h @@ -86,7 +86,11 @@ class StringArena { // gates the diagnostic DICTIONARY_BYTES counter). Keys are bump-allocated // inside these chunks, so they must not be counted separately. if (c != nullptr) { - NativeMem::record(NM_DICTIONARY, (long long)sizeof(Chunk)); + // recordAlloc measures the allocator overhead rather than assuming + // it. These chunks are ~512 KB, so glibc serves them by mmap and the + // overhead is ~16 bytes -- about 0.003 %, not the double-digit + // percentage a small-allocation-derived blanket factor would imply. + NativeMem::recordAlloc(NM_DICTIONARY, c, sizeof(Chunk)); } return c; } @@ -143,8 +147,9 @@ class StringArena { Chunk* c = _first; while (c) { Chunk* n = c->next; + // Before free(): the overhead is read back off the live chunk. + NativeMem::recordFreeBefore(NM_DICTIONARY, c, sizeof(Chunk)); free(c); - NativeMem::record(NM_DICTIONARY, -(long long)sizeof(Chunk)); c = n; } } @@ -182,8 +187,9 @@ class StringArena { int freed = 0; while (c) { Chunk* n = c->next; + // Before free(): the overhead is read back off the live chunk. + NativeMem::recordFreeBefore(NM_DICTIONARY, c, sizeof(Chunk)); free(c); - NativeMem::record(NM_DICTIONARY, -(long long)sizeof(Chunk)); c = n; ++freed; } diff --git a/ddprof-lib/src/test/cpp/mallocFootprint_ut.cpp b/ddprof-lib/src/test/cpp/mallocFootprint_ut.cpp new file mode 100644 index 0000000000..3d6295c936 --- /dev/null +++ b/ddprof-lib/src/test/cpp/mallocFootprint_ut.cpp @@ -0,0 +1,184 @@ +/* + * Copyright 2026, Datadog, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include +#include "mallocFootprint.h" +#include "nativeMem.h" +#include "countingAllocator.h" + +// Allocator overhead is a function of PER-ALLOCATION size, not of total bytes. +// That is the whole reason a single blanket multiplier cannot be correct across +// categories: the same total can be one large chunk paying ~0 % or thousands of +// small nodes paying ~17 %. These tests pin that, and pin that the profiler now +// measures it rather than assuming a factor. + +// True when the allocator in force exposes enough for overhead to be observed +// at all. Sanitizer builds substitute their own allocator, whose usable size is +// the requested size and which lays blocks out so the header probe yields 0 -- +// so it correctly reports no overhead, and the magnitude assertions below have +// nothing to measure. Reporting zero there is the honest answer, not a bug. +static bool allocatorExposesOverhead() { + void *p = malloc(96); + if (p == NULL) return false; + size_t oh = MallocFootprint::overheadOf(p, 96); + free(p); + return oh > 0; +} + +class MallocFootprintTest : public ::testing::Test { +protected: + long long _baseline[NM_NUM_CATEGORIES]; + void SetUp() override { + for (int c = 0; c < NM_NUM_CATEGORIES; c++) { + _baseline[c] = NativeMem::live((NativeMemCategory)c); + } + NativeMem::reset(); + } + void TearDown() override { + NativeMem::reset(); + for (int c = 0; c < NM_NUM_CATEGORIES; c++) { + NativeMem::setLive((NativeMemCategory)c, _baseline[c]); + } + } +}; + +// The header must be discovered, not hardcoded: an LD_PRELOAD'd tcmalloc or +// jemalloc adds none while __GLIBC__ stays defined. On glibc the probe is +// expected to find 8; anywhere else it must at least be sane. +TEST_F(MallocFootprintTest, HeaderProbeReturnsSaneValue) { + size_t h = MallocFootprint::headerBytes(); + EXPECT_LE(h, 64u) << "probe returned an implausible header size"; + printf("[FOOTPRINT] probed per-chunk header = %zu bytes\n", h); + + // Cross-check against an independently measured case: a 96-byte request + // occupies 112 bytes on glibc (104 usable + 8 header). + void *p = malloc(96); + ASSERT_NE(nullptr, p); + size_t fp = MallocFootprint::of(p, 96); + printf("[FOOTPRINT] request=96 usable=%zu footprint=%zu\n", + malloc_usable_size(p), fp); + EXPECT_GE(fp, 96u) << "footprint cannot be below the requested size"; + free(p); +} + +// The central claim: overhead as a FRACTION collapses as allocation size grows. +// A blanket multiplier calibrated on small allocations therefore over-charges +// large ones badly -- which is precisely the case for the 512 KB string-arena +// chunks and the JFR buffers. +TEST_F(MallocFootprintTest, OverheadFractionCollapsesWithAllocationSize) { + if (!allocatorExposesOverhead()) { + GTEST_SKIP() << "allocator exposes no overhead (sanitizer build)"; + } + struct { size_t size; const char *what; } cases[] = { + {96, "MethodMap node"}, + {6 * 1024, "SBTable"}, + {512 * 1024, "string-arena chunk"}, + {2 * 1024 * 1024, "large buffer"}, + }; + double small_pct = -1, large_pct = -1; + for (auto &c : cases) { + void *p = malloc(c.size); + ASSERT_NE(nullptr, p); + size_t oh = MallocFootprint::overheadOf(p, c.size); + double pct = 100.0 * (double)oh / (double)c.size; + printf("[FOOTPRINT] %-20s request=%-8zu overhead=%-4zu (%.4f %%)\n", + c.what, c.size, oh, pct); + if (c.size == 96) small_pct = pct; + if (c.size == 512 * 1024) large_pct = pct; + free(p); + } + ASSERT_GE(small_pct, 0); + ASSERT_GE(large_pct, 0); + // A small node pays a double-digit percentage... + EXPECT_GT(small_pct, 10.0) + << "a 96-byte allocation should pay a double-digit overhead percentage"; + // ...while a 512 KB chunk pays well under one percent. A blanket factor + // calibrated on the former over-charges the latter by more than an order of + // magnitude, which is the quantitative reason one factor cannot serve both. + EXPECT_LT(large_pct, 1.0) + << "a 512 KB chunk must not be charged a double-digit percentage"; + EXPECT_GT(small_pct, large_pct * 10); + + // Large-allocation overhead is not even a function of the request size. + // glibc serves large requests either from an arena or by mmap, and the + // crossover is a DYNAMIC threshold that rises as large blocks are freed. So + // the identical 512 KB request measures 4088 bytes of overhead (mmap: page + // rounding on the final page) in one allocator state and 16 bytes (arena: + // header only) in another, purely as a function of allocation history. + // + // This is the decisive argument for measuring rather than computing: no + // arithmetic formula over the requested size -- align16(S + 8) or anything + // else -- can be right for large allocations, because the answer depends on + // state the caller cannot see. + size_t seen_min = (size_t)-1, seen_max = 0; + for (int i = 0; i < 8; i++) { + void *p = malloc(512 * 1024); + ASSERT_NE(nullptr, p); + size_t oh = MallocFootprint::overheadOf(p, 512 * 1024); + if (oh < seen_min) seen_min = oh; + if (oh > seen_max) seen_max = oh; + free(p); + } + printf("[FOOTPRINT] identical 512 KB request, 8 attempts: overhead %zu..%zu\n", + seen_min, seen_max); + // Whatever the allocator chose, it must stay far below the double-digit + // percentage a small-allocation-derived factor would charge. + EXPECT_LT(100.0 * (double)seen_max / (double)(512 * 1024), 1.0); +} + +// CountingAllocator must populate the overhead gauge while leaving the live +// gauge as logical bytes, and both must return to zero on release. +TEST_F(MallocFootprintTest, CountingAllocatorTracksOverheadSeparately) { + if (!allocatorExposesOverhead()) { + GTEST_SKIP() << "allocator exposes no overhead (sanitizer build)"; + } + using Map = std::map, + CountingAllocator, + NM_METHOD_MAP>>; + { + Map m; + for (long i = 0; i < 2000; i++) { + m[i * 7919] = i; + } + long long live = NativeMem::live(NM_METHOD_MAP); + long long oh = NativeMem::overhead(NM_METHOD_MAP); + printf("[FOOTPRINT] map of 2000 nodes: live=%lld overhead=%lld (%.1f %%)\n", + live, oh, 100.0 * (double)oh / (double)live); + EXPECT_GT(live, 0); + EXPECT_GT(oh, 0) << "overhead gauge was not populated"; + // Nodes are small, so overhead is a substantial fraction -- not a rounding + // error, and not something a logical-bytes-only counter can see. + EXPECT_GT(100.0 * (double)oh / (double)live, 1.0); + } + EXPECT_EQ(0, NativeMem::live(NM_METHOD_MAP)) + << "live gauge did not return to zero"; + EXPECT_EQ(0, NativeMem::overhead(NM_METHOD_MAP)) + << "overhead gauge did not return to zero; recordFreeBefore is unbalanced"; +} + +// recordAlloc/recordFreeBefore must balance exactly, including for a size whose +// rounding is non-trivial. +TEST_F(MallocFootprintTest, RecordAllocAndFreeBalance) { + const size_t SZ = 100; // not a multiple of the alignment quantum + void *ptrs[500]; + for (int i = 0; i < 500; i++) { + ptrs[i] = malloc(SZ); + ASSERT_NE(nullptr, ptrs[i]); + NativeMem::recordAlloc(NM_MISC, ptrs[i], SZ); + } + EXPECT_EQ((long long)(500 * SZ), NativeMem::live(NM_MISC)); + if (allocatorExposesOverhead()) { + EXPECT_GT(NativeMem::overhead(NM_MISC), 0); + } + + for (int i = 0; i < 500; i++) { + NativeMem::recordFreeBefore(NM_MISC, ptrs[i], SZ); + free(ptrs[i]); + } + EXPECT_EQ(0, NativeMem::live(NM_MISC)); + EXPECT_EQ(0, NativeMem::overhead(NM_MISC)); +} From ec7bfbd2b93ca4210a0f30c8ba2aa9591cbc08a7 Mon Sep 17 00:00:00 2001 From: Roman Kennke Date: Fri, 28 Aug 2026 11:55:23 +0000 Subject: [PATCH 3/3] feat(nativemem): measure allocator overhead for NM_NATIVE_SYMBOLS Extends the measured chunk-overhead accounting to the largest category, which the previous change could not cover. NM_NATIVE_SYMBOLS is a gauge -- setLive(native_libs.memoryUsage()) -- computed as a sum of logical sizes, so there is no pointer in hand at record time and recordAlloc() does not apply. CodeCache::memoryUsage() already walks every symbol name, so it now optionally accumulates the measured overhead in that same pass via an out-param, and Profiler publishes it with a new gauge-style NativeMem::setOverhead(). Single pass, no extra walk. NativeFunc::nameOverhead() is where the measurement lives because only NativeFunc knows the real allocation base: `name` points *into* the block at offset sizeof(NativeFunc), so querying the allocator with `name` itself would be undefined. It sits next to allocSize(), which the header already documents as the single source of truth for that allocation's size. The _blobs array is deliberately excluded: one allocation per library, large enough that its overhead is a rounding error, and it comes from new CodeBlob[] whose returned pointer is not guaranteed to be the allocator's block base. Excluding it understates by a negligible amount rather than risking an unsound reading. Measured on a real profiled JVM (classes sweep, 60k classes): category live MiB overhead MiB overhead % calltrace 24.534 0.0000 0.00 (mmap-backed) native_symbols 11.882 0.9796 8.24 dictionary 4.553 0.0350 0.77 thread_info 0.003 0.0004 13.97 native_symbols measures 8.24 %, roughly half the 17 % the blanket factor assumed, and it is the largest category -- so this is where that factor did the most damage. Across the malloc-backed categories the blanket x1.17 would charge about 3.0 MiB where 1.0 MiB is measured, over-crediting the explained total by ~2 MiB against a residual of only a few MiB. Correcting it therefore WIDENS the reconciliation gap, which is the direction predicted. Coverage is still partial and the measured figure is a lower bound: jfr_buffers, liveness and line_tables record through plain record() and contribute zero here. calltrace's zero is correct rather than missing -- it is mmap-backed and pays no malloc chunk overhead at all. Release, debug, ASan and TSan pass, including codeCache_ut and libraries_ut. Co-Authored-By: Claude Opus 5 (1M context) --- ddprof-lib/src/main/cpp/codeCache.cpp | 21 ++++++++++++++++++++- ddprof-lib/src/main/cpp/codeCache.h | 25 ++++++++++++++++++++++--- ddprof-lib/src/main/cpp/nativeMem.h | 6 ++++++ ddprof-lib/src/main/cpp/profiler.cpp | 8 +++++++- 4 files changed, 55 insertions(+), 5 deletions(-) diff --git a/ddprof-lib/src/main/cpp/codeCache.cpp b/ddprof-lib/src/main/cpp/codeCache.cpp index 9062b555b1..57842cec24 100644 --- a/ddprof-lib/src/main/cpp/codeCache.cpp +++ b/ddprof-lib/src/main/cpp/codeCache.cpp @@ -158,7 +158,7 @@ CodeCache::~CodeCache() { free(_build_id); // Free build-id memory } -long long CodeCache::memoryUsage() const { +long long CodeCache::memoryUsage(long long *overhead_out) const { // The blob array: _capacity entries of CodeBlob. long long total = (long long)_capacity * sizeof(CodeBlob); @@ -177,6 +177,25 @@ long long CodeCache::memoryUsage() const { total += (long long)NativeFunc::allocSize(_blobs[i]._name); } + // Measured allocator overhead on those name allocations, accumulated in the + // same pass. Name strings are numerous and short, so this is where the + // overhead lives: a blanket percentage derived from one workload's size mix + // cannot stand in for it. + // + // The _blobs array itself is deliberately excluded. It is one allocation per + // library, large enough that its overhead is a rounding error, and it comes + // from new CodeBlob[] -- whose returned pointer is not guaranteed to be the + // allocator's block base, so querying the allocator with it would be + // unsound. Excluding it understates by a negligible amount rather than + // risking a wrong reading. + if (overhead_out != nullptr) { + long long overhead = (long long)NativeFunc::nameOverhead(_name); + for (int i = 0; i < _count; i++) { + overhead += (long long)NativeFunc::nameOverhead(_blobs[i]._name); + } + *overhead_out += overhead; + } + // The DWARF unwind table, when present (length only — no pointer deref). total += (long long)_dwarf_table_length * sizeof(FrameDesc); diff --git a/ddprof-lib/src/main/cpp/codeCache.h b/ddprof-lib/src/main/cpp/codeCache.h index d8ac7d661e..076db50a5d 100644 --- a/ddprof-lib/src/main/cpp/codeCache.h +++ b/ddprof-lib/src/main/cpp/codeCache.h @@ -10,6 +10,7 @@ #include "common.h" #include "counters.h" #include "dwarf.h" +#include "mallocFootprint.h" #include "utils.h" #include @@ -85,6 +86,18 @@ class NativeFunc { return align_up(sizeof(NativeFunc) + 1 + strlen(name), sizeof(NativeFunc *)); } + // Allocator overhead on that allocation -- rounding to the size quantum plus + // the per-chunk header, measured rather than assumed. Lives here because only + // NativeFunc knows the real allocation base: `name` points *into* the block at + // offset sizeof(NativeFunc), so querying the allocator with `name` itself + // would be undefined. 0 if null. + static size_t nameOverhead(const char *name) { + if (name == nullptr) { + return 0; + } + return MallocFootprint::overheadOf(from(name), allocSize(name)); + } + static short libIndex(const char *name) { if (name == nullptr) { return -1; @@ -286,7 +299,9 @@ class CodeCache { // — it is mutated by the background refresher and negligible in size (see the // definition). Const and lock-free: reads only fields that are stable once the // library is published. - long long memoryUsage() const; + // overhead_out, when non-null, receives the measured allocator overhead on + // the name allocations counted here (see NativeFunc::nameOverhead). + long long memoryUsage(long long *overhead_out = nullptr) const; int count() { return _count; } CodeBlob* blob(int idx) { @@ -357,15 +372,19 @@ class CodeCacheArray { // cached at add() time. (Libraries are populated before being registered and // not grown afterwards.) The array is append-only, so iterating the published // prefix is safe alongside concurrent add()s. - size_t memoryUsage() const { + size_t memoryUsage(long long *overhead_out = nullptr) const { size_t total = 0; + long long overhead = 0; int n = count(); for (int i = 0; i < n; i++) { CodeCache *lib = at(i); if (lib != nullptr) { - total += (size_t)lib->memoryUsage(); + total += (size_t)lib->memoryUsage(overhead_out != nullptr ? &overhead : nullptr); } } + if (overhead_out != nullptr) { + *overhead_out = overhead; + } return total; } }; diff --git a/ddprof-lib/src/main/cpp/nativeMem.h b/ddprof-lib/src/main/cpp/nativeMem.h index af71f8fdaa..26a33c3255 100644 --- a/ddprof-lib/src/main/cpp/nativeMem.h +++ b/ddprof-lib/src/main/cpp/nativeMem.h @@ -131,6 +131,12 @@ class NativeMem { record(category, -(long long)requested); } + // Gauge-style overhead setter, for categories whose size is recomputed as an + // absolute rather than tracked via alloc/free deltas (see setLive). + static void setOverhead(NativeMemCategory category, long long value) { + store(_overhead[category], value); + } + static long long overhead(NativeMemCategory category) { return load(_overhead[category]); } diff --git a/ddprof-lib/src/main/cpp/profiler.cpp b/ddprof-lib/src/main/cpp/profiler.cpp index 7f9f208bc0..63f9a667fe 100644 --- a/ddprof-lib/src/main/cpp/profiler.cpp +++ b/ddprof-lib/src/main/cpp/profiler.cpp @@ -1892,7 +1892,8 @@ void Profiler::updateNativeLibMemStats() { // cache). memoryUsage() is a recomputed gauge; read it once and publish the // counters plus the NativeMem gauge (NATIVE_SYMBOLS) as absolutes. const CodeCacheArray& native_libs = _libs->native_libs(); - long long usage = (long long)native_libs.memoryUsage(); + long long symbols_overhead = 0; + long long usage = (long long)native_libs.memoryUsage(&symbols_overhead); Counters::set(CODECACHE_NATIVE_COUNT, native_libs.count()); Counters::set(CODECACHE_NATIVE_SIZE_BYTES, usage); // The runtime-stubs cache is a distinct HotSpot cache, not the native-symbol @@ -1901,6 +1902,11 @@ void Profiler::updateNativeLibMemStats() { Counters::set(CODECACHE_RUNTIME_STUBS_SIZE_BYTES, JVMSupport::runtimeStubsMemoryUsage()); NativeMem::setLive(NM_NATIVE_SYMBOLS, usage); + // Measured, not assumed: the symbol tables are many short name strings, so + // the allocator's rounding and per-chunk header are a material fraction of + // their real cost. Gauge-style to match setLive above -- memoryUsage() + // recomputes an absolute rather than tracking deltas. + NativeMem::setOverhead(NM_NATIVE_SYMBOLS, symbols_overhead); } Error Profiler::dump(const char *path, const int length) {