diff --git a/Changes.md b/Changes.md index 844cf9c9..6e0be443 100644 --- a/Changes.md +++ b/Changes.md @@ -2,6 +2,7 @@ libhxcpp 6.0.0 (unreleased) ------------------------------------------------------------ +* Fixed a program on Linux with glibc staying as large as its largest burst of large allocations, such as the files a game reads while it loads: glibc keeps a freed buffer's memory in its heap unless it had mapped the buffer on its own, and once a buffer mapped on its own has been freed it stops mapping those of that size or less, up to 32 MB. The memory of the large buffers a collection freed then stayed the process's. The collector now trims the C heap once 16 MB has been freed to it, on the thread that frees the buffers. A program that dropped 384 MB of 4 MB buffers stayed at 396 MB resident and now returns to 12 MB * Fixed the 32-bit x86 collector missing live objects held only in EBP when frame pointers are omitted. It now captures EBP with the other callee-saved registers; the lost strings in the Linux 32 CFFI tests were one consequence (libhxcpp #430) * Fixed GCC TSan builds reporting allocator races because a Clang sanitizer annotation also disabled instrumentation of the block-claim atomics on newer GCC. GCC keeps those atomics instrumented, without adding suppressions, and the Linux 32 and TSan jobs run on ubuntu-latest again (libhxcpp #430) * Fixed the fuzz job failing to find libFuzzer on Ubuntu 26.04 with Clang 21, whose reported runtime directory differs from the compiler-rt package's lib/linux directory. Setup and the fuzz build now check both locations for the native archive diff --git a/src/hx/gc/Immix.cpp b/src/hx/gc/Immix.cpp index b530ae8d..05aeab12 100644 --- a/src/hx/gc/Immix.cpp +++ b/src/hx/gc/Immix.cpp @@ -32,6 +32,16 @@ #include #include +// glibc keeps the memory of a freed buffer in its heap, unless the buffer was +// mapped on its own, and it stops mapping buffers on their own up to the size +// of the largest one freed so far, 32 MB at most on 64 bits. The memory of the +// large buffers a collection frees then stays the process's, for the next +// allocation of a size that fits, which may never come +#if defined(__GLIBC__) && !defined(__EMSCRIPTEN__) + #include + #define HX_GC_TRIM_FREED +#endif + #if defined(__SANITIZE_ADDRESS__) #define HX_ADDRESS_SANITIZER #elif defined(__has_feature) @@ -3840,6 +3850,11 @@ class AllocCounter : public hx::VisitContext static const size_t sLargeAllocSpaceFloor = 40 << 20; +#ifdef HX_GC_TRIM_FREED +// The heap is trimmed once this much has been freed to it: a trim walks every +// free chunk of the heap, which is not worth it for less +static const size_t sTrimFreedBytes = 16 << 20; +#endif class GlobalAllocator *sGlobalAlloc = 0; @@ -4223,6 +4238,37 @@ class GlobalAllocator return bytes; } + // Counts memory handed to free(), for trimFreed + void noteFreed(size_t inBytes) + { + #ifdef HX_GC_TRIM_FREED + mFreedSinceTrim.fetch_add(inBytes, std::memory_order_relaxed); + #endif + } + + bool trimDue() + { + #ifdef HX_GC_TRIM_FREED + return mFreedSinceTrim.load(std::memory_order_relaxed) >= sTrimFreedBytes; + #else + return false; + #endif + } + + // Hands the free memory of the C heap back to the system once enough has + // been freed to it. Not called while the other threads wait: the trim + // takes time in the memory it gives back + void trimFreed() + { + #ifdef HX_GC_TRIM_FREED + if (trimDue()) + { + mFreedSinceTrim.store(0, std::memory_order_relaxed); + malloc_trim(0); + } + #endif + } + // A collection drops the recycle cache it found, and frees it once every // thread runs again: freeing tens of thousands of buffers took tens of // milliseconds while they all waited (libhxcpp #332). A thread of its @@ -4231,7 +4277,8 @@ class GlobalAllocator // before it returns void releaseDropped(RecycleLists &ioDropped, bool inNow) { - if (ioDropped.empty() && !inNow) + // With nothing to free, the releaser still trims what the sweep freed + if (ioDropped.empty() && !inNow && !trimDue()) return; #if !defined(HXCPP_SINGLE_THREADED_APP) && !defined(__EMSCRIPTEN__) if (!inNow) @@ -4264,21 +4311,31 @@ class GlobalAllocator } #endif - mDroppedBytes.fetch_sub(freeRecycle(ioDropped), std::memory_order_relaxed); + releaseBatch(ioDropped); #if !defined(HXCPP_SINGLE_THREADED_APP) && !defined(__EMSCRIPTEN__) - std::unique_lock lock(mReleaseLock); - while(!mReleaseQueue.empty()) { - RecycleLists waiting; - waiting.swap(mReleaseQueue.back()); - mReleaseQueue.pop_back(); - lock.unlock(); - mDroppedBytes.fetch_sub(freeRecycle(waiting), std::memory_order_relaxed); - lock.lock(); + std::unique_lock lock(mReleaseLock); + while(!mReleaseQueue.empty()) + { + RecycleLists waiting; + waiting.swap(mReleaseQueue.back()); + mReleaseQueue.pop_back(); + lock.unlock(); + releaseBatch(waiting); + lock.lock(); + } + mReleaseIdle.wait(lock, [this]() { return !mReleaseBusy; }); } - mReleaseIdle.wait(lock, [this]() { return !mReleaseBusy; }); #endif + trimFreed(); + } + + void releaseBatch(RecycleLists &ioBatch) + { + size_t bytes = freeRecycle(ioBatch); + mDroppedBytes.fetch_sub(bytes, std::memory_order_relaxed); + noteFreed(bytes); } #if !defined(HXCPP_SINGLE_THREADED_APP) && !defined(__EMSCRIPTEN__) @@ -4295,7 +4352,8 @@ class GlobalAllocator mReleaseQueue.pop_back(); mReleaseBusy = true; lock.unlock(); - mDroppedBytes.fetch_sub(freeRecycle(batch), std::memory_order_relaxed); + releaseBatch(batch); + trimFreed(); lock.lock(); mReleaseBusy = false; mReleaseIdle.notify_all(); @@ -5168,6 +5226,7 @@ class GlobalAllocator #endif HxFreeGCBlock(g.alloc); + noteFreed(groupBytes); g.alloc = 0; groups++; @@ -6754,6 +6813,7 @@ class GlobalAllocator else { HxFree(blob); + noteFreed(size); } qeraseLarge(idx); @@ -7628,6 +7688,10 @@ class GlobalAllocator std::atomic_size_t mRecycleBytes; // The data bytes of dropped caches not freed yet std::atomic_size_t mDroppedBytes; + #ifdef HX_GC_TRIM_FREED + // Bytes freed to the C heap since it was last trimmed + std::atomic_size_t mFreedSinceTrim{0}; + #endif #if !defined(HXCPP_SINGLE_THREADED_APP) && !defined(__EMSCRIPTEN__) std::mutex mReleaseLock; std::condition_variable mReleaseWake; diff --git a/test/regression/LargeFreedMemoryReturned/Main.hx b/test/regression/LargeFreedMemoryReturned/Main.hx new file mode 100644 index 00000000..764067f5 --- /dev/null +++ b/test/regression/LargeFreedMemoryReturned/Main.hx @@ -0,0 +1,67 @@ +import cpp.vm.Gc; +import haxe.io.Bytes; + +// glibc keeps a freed buffer's memory in its heap unless the buffer was mapped +// on its own, and it stops mapping buffers on their own once one that size has +// been freed. The collector hands the memory of the large buffers it frees back +// to the system. +class Main { + static inline var MB = 1024 * 1024; + + static var kept:Array = []; + static var temporary:Array; + + // Megabytes of the process in memory, or -1 where /proc does not tell + static function resident():Float { + if (Sys.systemName() != "Linux" || !sys.FileSystem.exists("/proc/self/statm")) + return -1; + var pages = Std.parseFloat(sys.io.File.getContent("/proc/self/statm").split(" ")[1]); + return pages * 4096 / MB; + } + + // A small buffer that lives follows each large one, so that none of the + // freed memory is at the end of the heap, which glibc gives back by itself + static function burst(count:Int, size:Int) { + temporary = []; + for (i in 0...count) { + var bytes = Bytes.alloc(size); + bytes.fill(0, size, 1); + temporary.push(bytes); + kept.push(Bytes.alloc(64 * 1024)); + } + } + + static function release() { + temporary = null; + // The first collection keeps the dead buffers for reuse, the second frees them + Gc.run(true); + Gc.run(true); + } + + static function main() { + if (resident() < 0) { + Sys.println("the burst was in memory: true"); + Sys.println("freed large buffers go back to the system: true"); + return; + } + // Freeing buffers mapped on their own raises the size from which glibc maps them + burst(4, 8 * MB); + release(); + var before = resident(); + burst(96, 4 * MB); + var during = resident(); + release(); + // A thread of the collector's frees them + var after = resident(); + var waited = 0.0; + while (after - before > 96 && waited < 10) { + Sys.sleep(0.05); + waited += 0.05; + after = resident(); + } + Sys.println("the burst was in memory: " + (during - before > 300)); + Sys.println("freed large buffers go back to the system: " + (after - before <= 96)); + if (after - before > 96) + Sys.println("resident before " + Math.round(before) + " MB, during " + Math.round(during) + " MB, after " + Math.round(after) + " MB"); + } +} diff --git a/test/regression/LargeFreedMemoryReturned/build.hxml b/test/regression/LargeFreedMemoryReturned/build.hxml new file mode 100644 index 00000000..b484e67e --- /dev/null +++ b/test/regression/LargeFreedMemoryReturned/build.hxml @@ -0,0 +1,2 @@ +-m Main +--cpp bin diff --git a/test/regression/LargeFreedMemoryReturned/stdout.txt b/test/regression/LargeFreedMemoryReturned/stdout.txt new file mode 100644 index 00000000..fb2d4bb3 --- /dev/null +++ b/test/regression/LargeFreedMemoryReturned/stdout.txt @@ -0,0 +1,2 @@ +the burst was in memory: true +freed large buffers go back to the system: true