feat(nativemem): measure allocator overhead instead of assuming a blanket factor

ddprof-lib/src/main/cpp/codeCache.cpp

@@ -158,7 +158,7 @@ CodeCache::~CodeCache() {
   free(_build_id);  // Free build-id memory
 }
 
-long long CodeCache::memoryUsage() const {
+long long CodeCache::memoryUsage(long long *overhead_out) const {
   // The blob array: _capacity entries of CodeBlob.
   long long total = (long long)_capacity * sizeof(CodeBlob);
 
@@ -177,6 +177,25 @@ long long CodeCache::memoryUsage() const {
     total += (long long)NativeFunc::allocSize(_blobs[i]._name);
   }
 
+  // Measured allocator overhead on those name allocations, accumulated in the
+  // same pass. Name strings are numerous and short, so this is where the
+  // overhead lives: a blanket percentage derived from one workload's size mix
+  // cannot stand in for it.
+  //
+  // The _blobs array itself is deliberately excluded. It is one allocation per
+  // library, large enough that its overhead is a rounding error, and it comes
+  // from new CodeBlob[] -- whose returned pointer is not guaranteed to be the
+  // allocator's block base, so querying the allocator with it would be
+  // unsound. Excluding it understates by a negligible amount rather than
+  // risking a wrong reading.
+  if (overhead_out != nullptr) {
+    long long overhead = (long long)NativeFunc::nameOverhead(_name);
+    for (int i = 0; i < _count; i++) {
+      overhead += (long long)NativeFunc::nameOverhead(_blobs[i]._name);
+    }
+    *overhead_out += overhead;
+  }
+
   // The DWARF unwind table, when present (length only — no pointer deref).
   total += (long long)_dwarf_table_length * sizeof(FrameDesc);
 

ddprof-lib/src/main/cpp/codeCache.h

@@ -10,6 +10,7 @@
 #include "common.h"
 #include "counters.h"
 #include "dwarf.h"
+#include "mallocFootprint.h"
 #include "utils.h"
 
 #include <atomic>
@@ -85,6 +86,18 @@ class NativeFunc {
     return align_up(sizeof(NativeFunc) + 1 + strlen(name), sizeof(NativeFunc *));
   }
 
+  // Allocator overhead on that allocation -- rounding to the size quantum plus
+  // the per-chunk header, measured rather than assumed. Lives here because only
+  // NativeFunc knows the real allocation base: `name` points *into* the block at
+  // offset sizeof(NativeFunc), so querying the allocator with `name` itself
+  // would be undefined. 0 if null.
+  static size_t nameOverhead(const char *name) {
+    if (name == nullptr) {
+      return 0;
+    }
+    return MallocFootprint::overheadOf(from(name), allocSize(name));
+  }
+
   static short libIndex(const char *name) {
     if (name == nullptr) {
       return -1;
@@ -286,7 +299,9 @@ class CodeCache {
   // — it is mutated by the background refresher and negligible in size (see the
   // definition). Const and lock-free: reads only fields that are stable once the
   // library is published.
-  long long memoryUsage() const;
+  // overhead_out, when non-null, receives the measured allocator overhead on
+  // the name allocations counted here (see NativeFunc::nameOverhead).
+  long long memoryUsage(long long *overhead_out = nullptr) const;
 
   int count() { return _count; }
   CodeBlob* blob(int idx) {
@@ -357,15 +372,19 @@ class CodeCacheArray {
   // cached at add() time. (Libraries are populated before being registered and
   // not grown afterwards.) The array is append-only, so iterating the published
   // prefix is safe alongside concurrent add()s.
-  size_t memoryUsage() const {
+  size_t memoryUsage(long long *overhead_out = nullptr) const {
     size_t total = 0;
+    long long overhead = 0;
     int n = count();
     for (int i = 0; i < n; i++) {
       CodeCache *lib = at(i);
       if (lib != nullptr) {
-        total += (size_t)lib->memoryUsage();
+        total += (size_t)lib->memoryUsage(overhead_out != nullptr ? &overhead : nullptr);
       }
     }
+    if (overhead_out != nullptr) {
+      *overhead_out = overhead;
+    }
     return total;
   }
 };

ddprof-lib/src/main/cpp/counters.h

@@ -56,6 +56,20 @@
   X(NATIVE_MEM_LIVE_BYTES, "native_mem_live_bytes")                            \
   X(NATIVE_MEM_MAX_BYTES, "native_mem_max_bytes")                              \
   X(NATIVE_MEM_AVG_BYTES, "native_mem_avg_bytes")                              \
+  /* Process-wide malloc arena state, from glibc's own accounting. These are   \
+   * NOT profiler memory and deliberately are not NM_* categories: those must  \
+   * partition the profiler's own allocations (see nativeMem.h), and arena     \
+   * slack is mostly other subsystems' chunks stranded by interleaving, so it  \
+   * is not attributable to any profiler allocation. Reported so that the      \
+   * allocator's own overhead is visible rather than folded into a "profiler   \
+   * cost" figure -- it differs substantially between glibc, tcmalloc and      \
+   * jemalloc. Sampled on the JFR flush path only: mallinfo2() walks every     \
+   * arena taking locks, so it is neither cheap nor async-signal-safe. */      \
+  X(MALLOC_ARENA_BYTES, "malloc_arena_bytes")                                  \
+  X(MALLOC_IN_USE_BYTES, "malloc_in_use_bytes")                                \
+  X(MALLOC_FREE_HELD_BYTES, "malloc_free_held_bytes")                          \
+  X(MALLOC_TRIMMABLE_BYTES, "malloc_trimmable_bytes")                          \
+  X(MALLOC_MMAP_BYTES, "malloc_mmap_bytes")                                    \
   X(THREAD_IDS_COUNT, "thread_ids_count")                                      \
   X(THREAD_NAMES_COUNT, "thread_names_count")                                  \
   X(THREAD_FILTER_PAGES, "thread_filter_pages")                                \

ddprof-lib/src/main/cpp/countingAllocator.h

@@ -25,12 +25,18 @@ class CountingAllocator {
 
   T *allocate(std::size_t n) {
     T *p = static_cast<T *>(::operator new(n * sizeof(T)));
-    NativeMem::record(Cat, (long long)(n * sizeof(T)));
+    // recordAlloc rather than record: STL nodes are small, so the allocator's
+    // rounding and per-chunk header are a large fraction of their real cost
+    // (a 96-byte MethodMap node occupies 112 bytes -- 16.7 %). Logical bytes
+    // still land in the live gauge; the extra goes to the overhead gauge.
+    NativeMem::recordAlloc(Cat, p, n * sizeof(T));
     return p;
   }
 
   void deallocate(T *p, std::size_t n) noexcept {
-    NativeMem::record(Cat, -(long long)(n * sizeof(T)));
+    // Must run before operator delete: the overhead is read back off the live
+    // chunk.
+    NativeMem::recordFreeBefore(Cat, p, n * sizeof(T));
     ::operator delete(p);
   }
 

ddprof-lib/src/main/cpp/flightRecorder.cpp

@@ -7,6 +7,15 @@
 #include <assert.h>
 #include <inttypes.h>
 
+// mallinfo2() replaced the int-based mallinfo() in glibc 2.33; the older struct
+// silently truncates past 2 GiB, so it is not a usable fallback for byte
+// accounting. Absent on musl and macOS, where the arena counters stay zero.
+#if defined(__linux__) && defined(__GLIBC__) &&                                \
+    (__GLIBC__ > 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 33))
+  #include <malloc.h>
+  #define DD_HAVE_MALLINFO2 1
+#endif
+
 #include "buffers.h"
 #include "callTraceHashTable.h"
 #include "context.h"
@@ -1809,12 +1818,46 @@ void Recording::writeLogLevels(Buffer *buf) {
   }
 }
 
+// Snapshot the process-wide malloc arena state from glibc's own accounting.
+//
+// This is NOT profiler memory. Free-but-held arena pages are mostly other
+// subsystems' chunks stranded by interleaving, so they cannot be attributed to
+// any profiler allocation -- reporting them separately keeps them out of the
+// per-category figures while still making the allocator's overhead visible.
+// The numbers also differ substantially between glibc, tcmalloc and jemalloc,
+// which is exactly what a reader comparing allocators needs to see.
+//
+// Only safe on the flush path: mallinfo2() walks every arena taking each arena
+// lock, so it is neither cheap nor async-signal-safe. Must never be reached
+// from the sampling signal handler. Once per JFR chunk is negligible.
+void Recording::updateMallocArenaStats() {
+#ifdef DD_HAVE_MALLINFO2
+  struct mallinfo2 mi = mallinfo2();
+  // arena:    bytes obtained from the OS via brk, excluding mmap'd chunks
+  // uordblks: bytes currently handed out to callers
+  // fordblks: bytes free but retained in the arenas -- the waste term
+  // keepcost: the trimmable top block, i.e. what malloc_trim could return;
+  //           separating it distinguishes trim-threshold policy from genuine
+  //           fragmentation, which trimming cannot reclaim
+  // hblkhd:   bytes in mmap'd chunks, which are returned to the OS on free
+  Counters::set(MALLOC_ARENA_BYTES, (long long)mi.arena);
+  Counters::set(MALLOC_IN_USE_BYTES, (long long)mi.uordblks);
+  Counters::set(MALLOC_FREE_HELD_BYTES, (long long)mi.fordblks);
+  Counters::set(MALLOC_TRIMMABLE_BYTES, (long long)mi.keepcost);
+  Counters::set(MALLOC_MMAP_BYTES, (long long)mi.hblkhd);
+#endif
+}
+
 void Recording::updateNativeMemStats() {
   // Refresh the moving-window averages and the observed total peak. Per-category
   // peaks are maintained precisely at allocation time, so they are not sampled
   // here; the total peak is bracketed instead (see writeNativeMem).
   NativeMem::sample();
 
+  // Process-wide allocator state, sampled at the same instant as the
+  // per-category figures so the two can be compared coherently.
+  updateMallocArenaStats();
+
   // Mirror the totals into the flat counter table so they flow out through the
   // existing counter path (JFR T_DATADOG_COUNTER events and the JNI debug
   // counters). NATIVE_MEM_MAX_BYTES carries the upper bound on the total peak
@@ -1858,6 +1901,14 @@ void Recording::writeNativeMem(Buffer *buf) {
         {"native_mem_live_bytes.", NativeMem::live(cat)},
         {"native_mem_avg_bytes.", NativeMem::avg(cat)},
         {"native_mem_max_bytes.", NativeMem::max(cat)},
+        // Measured allocator overhead on the live allocations -- rounding to
+        // the size quantum plus the per-chunk header. Reported separately so
+        // native_mem_live_bytes stays comparable to sizeof() arithmetic, and so
+        // that reconciliation against RSS can add a measured figure instead of
+        // multiplying by a factor derived from some other workload's
+        // allocation-size mix. Zero for categories whose call sites still use
+        // record() rather than recordAlloc().
+        {"native_mem_chunk_overhead_bytes.", NativeMem::overhead(cat)},
     };
     for (const auto &m : metrics) {
       char label[64];

ddprof-lib/src/main/cpp/flightRecorder.h

@@ -324,6 +324,9 @@ class Recording {
   void writeCounters(Buffer *buf);
 
   void updateNativeMemStats();
+  // Process-wide malloc arena state (glibc mallinfo2). Flush path only --
+  // takes every arena lock, so not async-signal-safe. No-op off glibc 2.33+.
+  void updateMallocArenaStats();
   void writeNativeMem(Buffer *buf);
 
   void writeUnwindFailures(Buffer *buf);

ddprof-lib/src/main/cpp/mallocFootprint.h

@@ -0,0 +1,113 @@
+/*
+ * Copyright 2026, Datadog, Inc.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+#ifndef _MALLOCFOOTPRINT_H
+#define _MALLOCFOOTPRINT_H
+
+#include <cstddef>
+
+#if defined(__linux__)
+  #include <malloc.h>
+  #define DD_HAVE_MALLOC_USABLE_SIZE 1
+#elif defined(__APPLE__)
+  #include <malloc/malloc.h>
+  #define DD_HAVE_MALLOC_SIZE 1
+#endif
+
+// Real resident cost of a heap allocation, as opposed to the size that was
+// requested.
+//
+// The profiler's NM_* gauges record requested (logical) bytes, which is what
+// makes them comparable against sizeof() arithmetic. RSS, however, is paid in
+// allocator chunks: the request is rounded up to an alignment quantum and
+// carries a per-chunk header. Reconciliation previously multiplied logical bytes
+// by a single blanket factor measured on one workload, which is wrong whenever
+// the allocation-size mix differs -- overhead is a function of *per-allocation*
+// size, not of total bytes. A 512 KB chunk pays ~0.003 %; a 96-byte tree node
+// pays 16.7 %.
+//
+// This measures it instead: usable size comes from the allocator, and the
+// per-chunk header is probed once at first use rather than assumed.
+class MallocFootprint {
+private:
+  // Determined empirically: allocate several same-size blocks and take the
+  // smallest positive address stride between any two. That stride is
+  // usable + header, so header = stride - usable.
+  //
+  // Probed rather than hardcoded because the allocator in force at runtime is
+  // not knowable at compile time -- an LD_PRELOAD'd tcmalloc or jemalloc leaves
+  // __GLIBC__ defined while adding no per-object header at all (their metadata
+  // is out of band). Assuming glibc's 8 bytes would then invent overhead that
+  // does not exist, at one allocation's worth per allocation.
+  static size_t probeHeaderBytes() {
+#ifdef DD_HAVE_MALLOC_USABLE_SIZE
+    const int N = 16;
+    const size_t SZ = 48;
+    void *p[N];
+    for (int i = 0; i < N; i++) {
+      p[i] = malloc(SZ);
+      if (p[i] == NULL) {  // give up cleanly rather than guess
+        for (int j = 0; j < i; j++) free(p[j]);
+        return 0;
+      }
+    }
+    size_t usable = malloc_usable_size(p[0]);
+    long best = 0;
+    for (int i = 0; i < N; i++) {
+      for (int j = 0; j < N; j++) {
+        long d = (char *)p[j] - (char *)p[i];
+        if (d > 0 && (best == 0 || d < best)) {
+          best = d;
+        }
+      }
+    }
+    for (int i = 0; i < N; i++) free(p[i]);
+    long header = best - (long)usable;
+    // Sanity-bound the result. A negative or implausibly large value means the
+    // blocks were not laid out contiguously (a size-class allocator, or an
+    // arena boundary landed mid-probe), in which case 0 is the honest answer:
+    // report only the rounding we can see, and understate rather than invent.
+    if (header < 0 || header > 64) {
+      return 0;
+    }
+    return (size_t)header;
+#else
+    return 0;
+#endif
+  }
+
+public:
+  // Per-chunk header size for the allocator actually in force. Probed once;
+  // the C++11 function-local static makes initialisation thread-safe. NOT
+  // async-signal-safe (it allocates), so first use must not be from a signal
+  // handler -- every current call site is on a normal thread.
+  static size_t headerBytes() {
+    static const size_t header = probeHeaderBytes();
+    return header;
+  }
+
+  // Bytes this allocation actually costs: allocator-reported usable size plus
+  // the per-chunk header. Page rounding for large mmap'd chunks is already
+  // inside the usable size, so it must not be added again.
+  static size_t of(void *p, size_t requested) {
+    if (p == NULL) {
+      return 0;
+    }
+#ifdef DD_HAVE_MALLOC_USABLE_SIZE
+    return malloc_usable_size(p) + headerBytes();
+#elif defined(DD_HAVE_MALLOC_SIZE)
+    return malloc_size(p) + headerBytes();
+#else
+    return requested;  // no introspection available: report no overhead
+#endif
+  }
+
+  // Overhead alone -- the part RSS pays for that the logical counters miss.
+  static size_t overheadOf(void *p, size_t requested) {
+    size_t total = of(p, requested);
+    return total > requested ? total - requested : 0;
+  }
+};
+
+#endif // _MALLOCFOOTPRINT_H

ddprof-lib/src/main/cpp/nativeMem.cpp

@@ -5,6 +5,7 @@
 #include "nativeMem.h"
 
 volatile long long NativeMem::_live[NM_NUM_CATEGORIES] = {};
+volatile long long NativeMem::_overhead[NM_NUM_CATEGORIES] = {};
 volatile long long NativeMem::_max[NM_NUM_CATEGORIES] = {};
 long long NativeMem::_window[NM_NUM_CATEGORIES][NativeMem::WINDOW] = {};
 long long NativeMem::_total_window[NativeMem::WINDOW] = {};
@@ -14,6 +15,17 @@ long long NativeMem::_avg[NM_NUM_CATEGORIES] = {};
 long long NativeMem::_total_avg = 0;
 long long NativeMem::_total_max_observed = 0;
 
+long long NativeMem::overheadTotal() {
+  long long total = 0;
+  for (int c = 0; c < NM_NUM_CATEGORIES; c++) {
+    long long v = load(_overhead[c]);
+    if (v > 0) {
+      total += v;
+    }
+  }
+  return total;
+}
+
 long long NativeMem::liveTotal() {
   long long total = 0;
   for (int c = 0; c < NM_NUM_CATEGORIES; c++) {
@@ -84,6 +96,7 @@ void NativeMem::reset() {
   for (int c = 0; c < NM_NUM_CATEGORIES; c++) {
     store(_live[c], (long long)0);
     store(_max[c], (long long)0);
+    store(_overhead[c], (long long)0);
     _avg[c] = 0;
     for (int i = 0; i < WINDOW; i++) {
       _window[c][i] = 0;