Give the memory of freed large buffers back to the system with glibc

Changes.md

@@ -2,6 +2,7 @@
 libhxcpp 6.0.0 (unreleased)
 ------------------------------------------------------------
 
+* Fixed a program on Linux with glibc staying as large as its largest burst of large allocations, such as the files a game reads while it loads: glibc keeps a freed buffer's memory in its heap unless it had mapped the buffer on its own, and once a buffer mapped on its own has been freed it stops mapping those of that size or less, up to 32 MB. The memory of the large buffers a collection freed then stayed the process's. The collector now trims the C heap once 16 MB has been freed to it, on the thread that frees the buffers. A program that dropped 384 MB of 4 MB buffers stayed at 396 MB resident and now returns to 12 MB
 * Fixed the 32-bit x86 collector missing live objects held only in EBP when frame pointers are omitted. It now captures EBP with the other callee-saved registers; the lost strings in the Linux 32 CFFI tests were one consequence (libhxcpp #430)
 * Fixed GCC TSan builds reporting allocator races because a Clang sanitizer annotation also disabled instrumentation of the block-claim atomics on newer GCC. GCC keeps those atomics instrumented, without adding suppressions, and the Linux 32 and TSan jobs run on ubuntu-latest again (libhxcpp #430)
 * Fixed the fuzz job failing to find libFuzzer on Ubuntu 26.04 with Clang 21, whose reported runtime directory differs from the compiler-rt package's lib/linux directory. Setup and the fuzz build now check both locations for the native archive

src/hx/gc/Immix.cpp

@@ -32,6 +32,16 @@
 #include <string>
 #include <stdlib.h>
 
+// glibc keeps the memory of a freed buffer in its heap, unless the buffer was
+//  mapped on its own, and it stops mapping buffers on their own up to the size
+//  of the largest one freed so far, 32 MB at most on 64 bits. The memory of the
+//  large buffers a collection frees then stays the process's, for the next
+//  allocation of a size that fits, which may never come
+#if defined(__GLIBC__) && !defined(__EMSCRIPTEN__)
+   #include <malloc.h>
+   #define HX_GC_TRIM_FREED
+#endif
+
 #if defined(__SANITIZE_ADDRESS__)
    #define HX_ADDRESS_SANITIZER
 #elif defined(__has_feature)
@@ -3840,6 +3850,11 @@ class AllocCounter : public hx::VisitContext
 
 
 static const size_t sLargeAllocSpaceFloor = 40 << 20;
+#ifdef HX_GC_TRIM_FREED
+// The heap is trimmed once this much has been freed to it: a trim walks every
+//  free chunk of the heap, which is not worth it for less
+static const size_t sTrimFreedBytes = 16 << 20;
+#endif
 
 class GlobalAllocator *sGlobalAlloc = 0;
 
@@ -4223,6 +4238,37 @@ class GlobalAllocator
       return bytes;
    }
 
+   // Counts memory handed to free(), for trimFreed
+   void noteFreed(size_t inBytes)
+   {
+      #ifdef HX_GC_TRIM_FREED
+      mFreedSinceTrim.fetch_add(inBytes, std::memory_order_relaxed);
+      #endif
+   }
+
+   bool trimDue()
+   {
+      #ifdef HX_GC_TRIM_FREED
+      return mFreedSinceTrim.load(std::memory_order_relaxed) >= sTrimFreedBytes;
+      #else
+      return false;
+      #endif
+   }
+
+   // Hands the free memory of the C heap back to the system once enough has
+   //  been freed to it. Not called while the other threads wait: the trim
+   //  takes time in the memory it gives back
+   void trimFreed()
+   {
+      #ifdef HX_GC_TRIM_FREED
+      if (trimDue())
+      {
+         mFreedSinceTrim.store(0, std::memory_order_relaxed);
+         malloc_trim(0);
+      }
+      #endif
+   }
+
    // A collection drops the recycle cache it found, and frees it once every
    //  thread runs again: freeing tens of thousands of buffers took tens of
    //  milliseconds while they all waited (libhxcpp #332). A thread of its
@@ -4231,7 +4277,8 @@ class GlobalAllocator
    //  before it returns
    void releaseDropped(RecycleLists &ioDropped, bool inNow)
    {
-      if (ioDropped.empty() && !inNow)
+      // With nothing to free, the releaser still trims what the sweep freed
+      if (ioDropped.empty() && !inNow && !trimDue())
          return;
       #if !defined(HXCPP_SINGLE_THREADED_APP) && !defined(__EMSCRIPTEN__)
       if (!inNow)
@@ -4264,21 +4311,31 @@ class GlobalAllocator
       }
       #endif
 
-      mDroppedBytes.fetch_sub(freeRecycle(ioDropped), std::memory_order_relaxed);
+      releaseBatch(ioDropped);
 
       #if !defined(HXCPP_SINGLE_THREADED_APP) && !defined(__EMSCRIPTEN__)
-      std::unique_lock<std::mutex> lock(mReleaseLock);
-      while(!mReleaseQueue.empty())
       {
-         RecycleLists waiting;
-         waiting.swap(mReleaseQueue.back());
-         mReleaseQueue.pop_back();
-         lock.unlock();
-         mDroppedBytes.fetch_sub(freeRecycle(waiting), std::memory_order_relaxed);
-         lock.lock();
+         std::unique_lock<std::mutex> lock(mReleaseLock);
+         while(!mReleaseQueue.empty())
+         {
+            RecycleLists waiting;
+            waiting.swap(mReleaseQueue.back());
+            mReleaseQueue.pop_back();
+            lock.unlock();
+            releaseBatch(waiting);
+            lock.lock();
+         }
+         mReleaseIdle.wait(lock, [this]() { return !mReleaseBusy; });
       }
-      mReleaseIdle.wait(lock, [this]() { return !mReleaseBusy; });
       #endif
+      trimFreed();
+   }
+
+   void releaseBatch(RecycleLists &ioBatch)
+   {
+      size_t bytes = freeRecycle(ioBatch);
+      mDroppedBytes.fetch_sub(bytes, std::memory_order_relaxed);
+      noteFreed(bytes);
    }
 
    #if !defined(HXCPP_SINGLE_THREADED_APP) && !defined(__EMSCRIPTEN__)
@@ -4295,7 +4352,8 @@ class GlobalAllocator
          mReleaseQueue.pop_back();
          mReleaseBusy = true;
          lock.unlock();
-         mDroppedBytes.fetch_sub(freeRecycle(batch), std::memory_order_relaxed);
+         releaseBatch(batch);
+         trimFreed();
          lock.lock();
          mReleaseBusy = false;
          mReleaseIdle.notify_all();
@@ -5168,6 +5226,7 @@ class GlobalAllocator
             #endif
 
             HxFreeGCBlock(g.alloc);
+            noteFreed(groupBytes);
             g.alloc = 0;
 
             groups++;
@@ -6754,6 +6813,7 @@ class GlobalAllocator
             else
             {
                HxFree(blob);
+               noteFreed(size);
             }
 
             qeraseLarge(idx);
@@ -7628,6 +7688,10 @@ class GlobalAllocator
    std::atomic_size_t mRecycleBytes;
    // The data bytes of dropped caches not freed yet
    std::atomic_size_t mDroppedBytes;
+   #ifdef HX_GC_TRIM_FREED
+   // Bytes freed to the C heap since it was last trimmed
+   std::atomic_size_t mFreedSinceTrim{0};
+   #endif
    #if !defined(HXCPP_SINGLE_THREADED_APP) && !defined(__EMSCRIPTEN__)
    std::mutex mReleaseLock;
    std::condition_variable mReleaseWake;

test/regression/LargeFreedMemoryReturned/Main.hx

@@ -0,0 +1,67 @@
+import cpp.vm.Gc;
+import haxe.io.Bytes;
+
+// glibc keeps a freed buffer's memory in its heap unless the buffer was mapped
+// on its own, and it stops mapping buffers on their own once one that size has
+// been freed. The collector hands the memory of the large buffers it frees back
+// to the system.
+class Main {
+	static inline var MB = 1024 * 1024;
+
+	static var kept:Array<Bytes> = [];
+	static var temporary:Array<Bytes>;
+
+	// Megabytes of the process in memory, or -1 where /proc does not tell
+	static function resident():Float {
+		if (Sys.systemName() != "Linux" || !sys.FileSystem.exists("/proc/self/statm"))
+			return -1;
+		var pages = Std.parseFloat(sys.io.File.getContent("/proc/self/statm").split(" ")[1]);
+		return pages * 4096 / MB;
+	}
+
+	// A small buffer that lives follows each large one, so that none of the
+	// freed memory is at the end of the heap, which glibc gives back by itself
+	static function burst(count:Int, size:Int) {
+		temporary = [];
+		for (i in 0...count) {
+			var bytes = Bytes.alloc(size);
+			bytes.fill(0, size, 1);
+			temporary.push(bytes);
+			kept.push(Bytes.alloc(64 * 1024));
+		}
+	}
+
+	static function release() {
+		temporary = null;
+		// The first collection keeps the dead buffers for reuse, the second frees them
+		Gc.run(true);
+		Gc.run(true);
+	}
+
+	static function main() {
+		if (resident() < 0) {
+			Sys.println("the burst was in memory: true");
+			Sys.println("freed large buffers go back to the system: true");
+			return;
+		}
+		// Freeing buffers mapped on their own raises the size from which glibc maps them
+		burst(4, 8 * MB);
+		release();
+		var before = resident();
+		burst(96, 4 * MB);
+		var during = resident();
+		release();
+		// A thread of the collector's frees them
+		var after = resident();
+		var waited = 0.0;
+		while (after - before > 96 && waited < 10) {
+			Sys.sleep(0.05);
+			waited += 0.05;
+			after = resident();
+		}
+		Sys.println("the burst was in memory: " + (during - before > 300));
+		Sys.println("freed large buffers go back to the system: " + (after - before <= 96));
+		if (after - before > 96)
+			Sys.println("resident before " + Math.round(before) + " MB, during " + Math.round(during) + " MB, after " + Math.round(after) + " MB");
+	}
+}

test/regression/LargeFreedMemoryReturned/build.hxml

@@ -0,0 +1,2 @@
+-m Main
+--cpp bin

test/regression/LargeFreedMemoryReturned/stdout.txt

@@ -0,0 +1,2 @@
+the burst was in memory: true
+freed large buffers go back to the system: true