head 1.1; branch 1.1.1; access ; symbols jemalloc-5-4-0:1.1.1.1 JASONE:1.1.1; locks ; strict; comment @ * @; 1.1 date 2026.09.24.18.28.52; author christos; state Exp; branches 1.1.1.1; next ; commitid GeR2ar6mKPT47UWG; 1.1.1.1 date 2026.09.24.18.28.52; author christos; state Exp; branches ; next ; commitid GeR2ar6mKPT47UWG; desc @@ 1.1 log @Initial revision @ text @#include "test/jemalloc_test.h" /* * Tests that HPA cleanly packs allocations into a single pageslab when they * collectively fit, instead of growing a new one. */ const char *malloc_conf = "disable_large_size_classes:true," "cache_oblivious:false," "hpa_sec_nshards:0," "hpa_slab_max_alloc:2097152," "hpa_dirty_mult:-1," "hpa_hugify_delay_ms:1000000000"; #define HPDATA_PAGES (HUGEPAGE / PAGE) /* * Every allocation must be >= SC_LARGE_MINCLASS so it goes through * arena_malloc_large -> pa_alloc -> hpa_alloc with the exact size we ask * for. Anything <= SC_SMALL_MAXCLASS would route through the bin/slab * path, which sizes its own slab independent of our request. */ #define LARGE_MIN_PAGES (SC_LARGE_MINCLASS / PAGE) /* * Sample sizes via Fibonacci numbers, filtered at runtime to fit the current * page geometry. This spans a useful range (covers each pszind group, hits * both pszind-boundary sizes like 5/8 and non-boundary sizes like * 13/21/34/...) without sweeping every page count, which would balloon the * 3-allocation cross product. */ static const size_t fib_pages[] = { 5, 8, 13, 21, 34, 55, 89, 144, 233, 377, }; #define NFIB (sizeof(fib_pages) / sizeof(fib_pages[0])) static bool pages_testable(size_t pages) { return pages >= LARGE_MIN_PAGES && pages < HPDATA_PAGES; } /* Shared per-test state, populated by setup_arena. */ static int g_alloc_flags; static size_t g_epoch_mib[1]; static size_t g_epoch_miblen; static size_t g_npageslabs_mib[5]; static size_t g_npageslabs_miblen; static void setup_arena(void) { unsigned arena_ind; size_t sz = sizeof(arena_ind); expect_d_eq(mallctl("arenas.create", &arena_ind, &sz, NULL, 0), 0, "arenas.create failed"); g_alloc_flags = MALLOCX_ARENA(arena_ind) | MALLOCX_TCACHE_NONE; g_epoch_miblen = sizeof(g_epoch_mib) / sizeof(g_epoch_mib[0]); expect_d_eq(mallctlnametomib("epoch", g_epoch_mib, &g_epoch_miblen), 0, "epoch mib lookup failed"); g_npageslabs_miblen = sizeof(g_npageslabs_mib) / sizeof(g_npageslabs_mib[0]); expect_d_eq(mallctlnametomib("stats.arenas.0.hpa_shard.npageslabs", g_npageslabs_mib, &g_npageslabs_miblen), 0, "npageslabs mib lookup failed"); g_npageslabs_mib[2] = arena_ind; } static size_t get_npageslabs(void) { uint64_t epoch = 1; size_t esz = sizeof(epoch); expect_d_eq(mallctlbymib(g_epoch_mib, g_epoch_miblen, &epoch, &esz, &epoch, sizeof(epoch)), 0, "epoch refresh failed"); size_t n; size_t nsz = sizeof(n); expect_d_eq(mallctlbymib(g_npageslabs_mib, g_npageslabs_miblen, &n, &nsz, NULL, 0), 0, "npageslabs read failed"); return n; } TEST_BEGIN(test_hpa_pageslab_packing_two_allocs) { test_skip_if(!config_stats); test_skip_if(!hpa_supported()); test_skip_if(opt_cache_oblivious); test_skip_if(!sz_large_size_classes_disabled()); setup_arena(); for (size_t i = 0; i < NFIB; i++) { size_t b_pages = fib_pages[i]; if (!pages_testable(b_pages)) { continue; } size_t a_pages = HPDATA_PAGES - b_pages; if (a_pages < LARGE_MIN_PAGES) { continue; } void *a = mallocx(a_pages * PAGE, g_alloc_flags); expect_ptr_not_null(a, "a mallocx(%zu PAGE) failed", a_pages); void *b = mallocx(b_pages * PAGE, g_alloc_flags); expect_ptr_not_null(b, "b mallocx(%zu PAGE) failed", b_pages); expect_zu_eq(get_npageslabs(), 1, "two allocs (a=%zu b=%zu pages) should pack into one " "pageslab", a_pages, b_pages); dallocx(b, g_alloc_flags); dallocx(a, g_alloc_flags); } } TEST_END TEST_BEGIN(test_hpa_pageslab_packing_three_allocs) { test_skip_if(!config_stats); test_skip_if(!hpa_supported()); test_skip_if(opt_cache_oblivious); test_skip_if(!sz_large_size_classes_disabled()); setup_arena(); /* * Sample c (the third allocation) and a (the first allocation) from * the Fibonacci table, with a up to rest - LARGE_MIN_PAGES so that * b = rest - a is always at least LARGE_MIN_PAGES. b is * derived to fill the rest of the pageslab. * * After packing, free the middle allocation (b) first, then a, then * c. This exercises hpdata_unreserve's coalescing of free ranges * with both neighbors as they appear: dealloc b leaves a single * b-sized hole; dealloc a should grow that hole to a+b; dealloc c * should leave the whole pageslab empty. As a final coalescing * check, allocate the entire pageslab in one call afterwards — that * only succeeds if the hpdata's longest_free_range was correctly * recomputed back to HPDATA_PAGES. */ for (size_t i = 0; i < NFIB; i++) { size_t c_pages = fib_pages[i]; if (!pages_testable(c_pages)) { continue; } size_t rest = HPDATA_PAGES - c_pages; if (rest < 2 * LARGE_MIN_PAGES) { continue; } for (size_t j = 0; j < NFIB; j++) { size_t a_pages = fib_pages[j]; if (!pages_testable(a_pages)) { continue; } if (a_pages > rest - LARGE_MIN_PAGES) { continue; } size_t b_pages = rest - a_pages; if (b_pages < LARGE_MIN_PAGES) { continue; } void *a = mallocx(a_pages * PAGE, g_alloc_flags); expect_ptr_not_null(a, "a mallocx(%zu PAGE) failed", a_pages); void *b = mallocx(b_pages * PAGE, g_alloc_flags); expect_ptr_not_null(b, "b mallocx(%zu PAGE) failed", b_pages); void *c = mallocx(c_pages * PAGE, g_alloc_flags); expect_ptr_not_null(c, "c mallocx(%zu PAGE) failed", c_pages); expect_zu_eq(get_npageslabs(), 1, "three allocs (a=%zu b=%zu c=%zu pages) should " "pack into one pageslab", a_pages, b_pages, c_pages); /* Free middle, then first, then last. */ dallocx(b, g_alloc_flags); dallocx(a, g_alloc_flags); dallocx(c, g_alloc_flags); /* * After freeing all three the hpdata should be empty * with longest_free_range == HPDATA_PAGES. The only * way this allocation succeeds in one pageslab is if * the coalesce on dalloc rebuilt that range. */ void *full = mallocx(HPDATA_PAGES * PAGE, g_alloc_flags); expect_ptr_not_null(full, "full mallocx(HPDATA_PAGES) after free of " "(a=%zu b=%zu c=%zu) failed — coalesce broken?", a_pages, b_pages, c_pages); expect_zu_eq(get_npageslabs(), 1, "full mallocx after free of (a=%zu b=%zu c=%zu) " "should fit the same pageslab", a_pages, b_pages, c_pages); dallocx(full, g_alloc_flags); } } } TEST_END int main(void) { if (config_stats && hpa_supported()) { opt_hpa = true; } return test( test_hpa_pageslab_packing_two_allocs, test_hpa_pageslab_packing_three_allocs); } @ 1.1.1.1 log @Import jemalloc 5.4.0 (previous was 5.3.1) * 5.4.0 (Sep 17, 2026) This release contains over 160 commits, focusing on the technical debts cleaning including refactorings, bug fixes, test coverage improvement, and option cleanups. The release also includes portability improvements per upstream issues report. New features: - Add `EXTENT_ALLOC_FLAG_PINNED` so custom extent-allocation hooks can mark non-reclaimable mappings, such as HugeTLB pages, for preferential reuse outside the decay and purge pipeline. Add the mallctl interfaces `stats.pinned`, `stats.arenas..pinned`, `stats.arenas..extents..npinned`, `stats.arenas..extents..pinned_bytes`, and `stats.arenas..mutexes.extents_pinned.{counter}` to report pinned-memory usage and mutex statistics. (@@binliu19: be2de8cc) - Allow resuming per-CPU arena selection via `thread.arena`. (@@algunenano: 68c35f65) - Better align the contents of human-readable and JSON malloc statistics. (@@spredolac: 68fe1be1, 30b1a410, 04aad97f) - Replace the runtime `experimental_infallible_new` option with the compile-time `--enable-cxx-infallible-new` option, enabling compiler-level optimizations and optimization in move constructors, and fix the `new(std::nothrow)` contract (@@spredolac: 160ab9d7, fe336672) Incompatible changes: - Adapt tcache fill and retention targets per bin to demand observed between GC events, replacing the fixed refill/flush policy. Remove seven legacy non-experimental controls: `lg_tcache_nslots_mul`, `tcache_nslots_small_min`, `tcache_nslots_small_max`, `tcache_nslots_large`, `tcache_gc_delay_bytes`, `lg_tcache_flush_small_div`, and `lg_tcache_flush_large_div`. Corresponding `malloc_conf` settings are silently ignored, matching `opt.*` mallctls return ENOENT, and `tcache_ncached_max` remains supported. (@@spredolac: d13fe91a) Bug fixes: - Preserve `errno` across `free`, `free_sized`, and `free_aligned_sized`, and across `process_madvise`-based page purging. (@@spredolac: 3a779661, 86f05828) - Fix numeric overflow checks in size classes. (@@spredolac: 6b245225) - Accept NULL in `free_sized()` and `free_aligned_sized()` (C23 correctness). (@@bigbruno: 7ce8b916) - Fix TSD lifecycle edge cases by 1) initializing thread-cache bins before marking the cache enabled, preventing reentrant bootstrap allocations from using uninitialized state, and 2) avoiding TSD recreation for late deallocations after thread teardown on generic-TSD platforms. (@@fzakaria: 54f22c83, @@spredolac: fb5499aa) - Use `O_CLOEXEC` when opening the THP sysfs file in `init_thp_state`. (@@ibookstein: d0705543) - Fix duplicate `opt.stats_print` and `opt.stats_print_opts` fields in `malloc_stats_print` output. (@@spredolac: dfe3a2ed) - Fix a potential deadlock during `arena_reset`. (@@guangli-dai: 6957341f) - Fix a prof-sampling / guard-page interaction bug in the SAN. (@@gctony: e36a0fa5) Optimizations and refactors: - Modularize jemalloc's front end by extracting arena management, initialization, fork orchestration, and allocation dispatch from `jemalloc.c`; untangle tcache/arena ownership; and consolidate the internal header graph to eliminate circular dependencies. (@@spredolac: ba1e2fe4, 88745978, 9d757223, de9ad145, ...) - Simplify ctl dispatch by refactoring arena helpers, internalizing an implementation-only control, replacing control-flow macros with typed helpers, and organizing `ctl.c` by subsystem. (@@spredolac: 9e143468, 901365d4, 4e903a0a, 5bc8d6e9) - Cap the base-block growth heuristic to avoid virtual-memory exhaustion under rare racy conditions. (@@guangli-dai: 2f4db8cf) - Move background-thread lifecycle and state operations into the background-thread module, clarifying ownership independently of PAC/HPA callers. (@@guangli-dai: f1f07923) - Simplify the page-allocation boundary by replacing PAI vtable dispatch with direct PAC/HPA calls, removing the obsolete `pai_t`/`pai.h` abstraction, and moving deferred-work and decay orchestration out of the arena. (@@guangli-dai: 1dfa6f7a, 8edd1012, d410f43e) - Refactor statistics collection and rendering into separate gather/emission stages and descriptor-driven tables. (@@spredolac: 184f304d, 80c8fcb4) - Introduce an OS abstraction layer and move platform-dependent file/process I/O, time, synchronization, CPU, virtual-memory, atfork, error-handling, profiling, thread-yield, and configuration-access operations out of allocator core code. (@@guangli-dai: c4158aca, c8e2e013, ...) Portability improvements: - Replace the `std::__throw_bad_alloc` call with standard C++ (#2900). (@@lexprfuncall: 1a15fe33) - Make `arena_s` use a flexible array member (`bin_t all_bins[]`) for C99 or newer. (@@grueninger: 300b58b4) - Fix rdtscp detection with `--with-lg-vaddr`. (@@xinydev: e8a0d2b4) - Fix `malloc_getcpu` on macOS to read the current CPU number correctly. (@@algunenano: 5aabbc87) - Use `CLOCK_MONOTONIC` for background-thread sleep to prevent clock-rollback stalls, and detect monotonic-condvar support at configure time. (@@antonio2368: ebacec3a, 8361239b) - Fix compilation warnings on macOS. (@@gctony: abb0a8a0) - Fix thread-exit TSD cleanup on MinGW builds. (@@gctony: 1e923170) - Fix GCC 16 build warnings by removing `-Wpedantic` violations in macro and function syntax and explicitly NUL-terminating profiling thread-name copies to resolve `-Wstringop-truncation`. (@@grueninger: 5acdcee6, c22d929a, @@jasonangelov: 6a245e02) - Parse PID-namespace symlinks without glibc-dependent `strtok`/`atol`, and return identifiers as `uint64_t`. (@@guangli-dai: 278d90a5) @ text @@