Bug 24328 – Very poor GC memory utilization due to fragmentation

This program seems to be able to utilize only 2% of memory available to it - the rest seems to be wasted by fragmentation: ///////////////////////////////// test.d //////////////////////////////// import core.stdc.stdio; import core.stdc.stdlib; enum size_t memoryLimit = 1UL * 1024 * 1024 * 1024; enum size_t allocationSize = 64; void main() { { import core.sys.posix.sys.resource : rlimit, RLIMIT_AS, getrlimit, setrlimit; rlimit lim; getrlimit(RLIMIT_AS, &lim); lim.rlim_cur = memoryLimit; setrlimit(RLIMIT_AS, &lim); } // Size of reachable data created by this function size_t utilizedSize; scope(exit) printf("Utilized size: %zu (%zd%%)\n", utilizedSize, utilizedSize * 100 / memoryLimit); ubyte[][] pinned; int n; while (true) { ubyte[] arr; { scope(failure) printf("Failed to allocate %zu\n", allocationSize); arr = new ubyte[allocationSize]; } bool keep = n++ % 2 == 0; if (keep) { { scope(failure) printf("Failed to reallocate %zu bytes\n", (pinned.length + 1) * pinned[0].sizeof); pinned ~= arr; } utilizedSize += allocationSize + pinned[0].sizeof; } } } /////////////////////////////////////////////////////////////////////////

If I modify the program to use C malloc/free instead of the D GC, I get 81% utilization: ////////////////////////// test_c.d ////////////////////////// import core.stdc.stdio; import core.stdc.stdlib; enum size_t memoryLimit = 1UL * 1024 * 1024 * 1024; enum size_t allocationSize = 64; @nogc void main() { { import core.sys.posix.sys.resource : rlimit, RLIMIT_AS, getrlimit, setrlimit; rlimit lim; getrlimit(RLIMIT_AS, &lim); lim.rlim_cur = memoryLimit; setrlimit(RLIMIT_AS, &lim); } // Size of reachable data created by this function size_t utilizedSize; scope(exit) printf("Utilized size: %zu (%zd%%)\n", utilizedSize, utilizedSize * 100 / memoryLimit); void** pinned; size_t numPinned; int n; while (true) { void* arr = malloc(allocationSize); if (!arr) { printf("Failed to allocate %zu\n", allocationSize); return; } bool keep = n++ % 2 == 0; if (keep) { auto newSize = (numPinned + 1) * pinned[0].sizeof; pinned = cast(void**)realloc(pinned, newSize); if (!pinned) { printf("Failed to reallocate %zu bytes\n", (numPinned + 1) * pinned[0].sizeof); return; } pinned[numPinned++] = arr; utilizedSize += allocationSize + pinned[0].sizeof; } else free(arr); } } //////////////////////////////////////////////////////////////

THIS ISSUE HAS BEEN MOVED TO GITHUB https://github.com/dlang/dmd/issues/17471 DO NOT COMMENT HERE ANYMORE, NOBODY WILL SEE IT, THIS ISSUE HAS BEEN MOVED TO GITHUB