head 1.1; branch 1.1.1; access ; symbols jemalloc-5-4-0:1.1.1.1 JASONE:1.1.1; locks ; strict; comment @ * @; 1.1 date 2026.09.24.18.28.52; author christos; state Exp; branches 1.1.1.1; next ; commitid GeR2ar6mKPT47UWG; 1.1.1.1 date 2026.09.24.18.28.52; author christos; state Exp; branches ; next ; commitid GeR2ar6mKPT47UWG; desc @@ 1.1 log @Initial revision @ text @#include "test/jemalloc_test.h" #include "test/hpa.h" #include "jemalloc/internal/hpa.h" #include "jemalloc/internal/hpa_central.h" #define HPA_TEST_ARENA_IND 125 #define HPA_TEST_EDEN_SIZE (128 * HUGEPAGE) static bool hpa_test_map_fail; static unsigned hpa_test_map_calls; static unsigned hpa_test_unmap_calls; static unsigned hpa_test_hugify_calls; static void *hpa_test_last_map; static size_t hpa_test_last_map_size; static void *hpa_test_last_unmap; static size_t hpa_test_last_unmap_size; static void *hpa_test_last_hugify; static size_t hpa_test_last_hugify_size; static void hpa_test_hooks_reset(void) { hpa_test_map_fail = false; hpa_test_map_calls = 0; hpa_test_unmap_calls = 0; hpa_test_hugify_calls = 0; hpa_test_last_map = NULL; hpa_test_last_map_size = 0; hpa_test_last_unmap = NULL; hpa_test_last_unmap_size = 0; hpa_test_last_hugify = NULL; hpa_test_last_hugify_size = 0; } static void * hpa_test_map(size_t size) { hpa_test_map_calls++; hpa_test_last_map_size = size; if (hpa_test_map_fail) { hpa_test_last_map = NULL; return NULL; } bool commit = true; void *ret = pages_map(NULL, size, HUGEPAGE, &commit); assert_true(commit, "HPA test mappings should be committed"); hpa_test_last_map = ret; return ret; } static void hpa_test_unmap(void *ptr, size_t size) { hpa_test_unmap_calls++; hpa_test_last_unmap = ptr; hpa_test_last_unmap_size = size; pages_unmap(ptr, size); } static void hpa_test_purge(void *ptr, size_t size) { } static bool hpa_test_hugify(void *ptr, size_t size, bool sync) { hpa_test_hugify_calls++; hpa_test_last_hugify = ptr; hpa_test_last_hugify_size = size; return false; } static void hpa_test_dehugify(void *ptr, size_t size) { } static void hpa_test_curtime(nstime_t *r_time, bool first_reading) { nstime_init(r_time, 0); } static uint64_t hpa_test_ms_since(nstime_t *r_time) { return 0; } static bool hpa_test_vectorized_purge(void *vec, size_t vlen, size_t nbytes) { return true; } static hpa_hooks_t hpa_test_hooks = { hpa_test_map, hpa_test_unmap, hpa_test_purge, hpa_test_hugify, hpa_test_dehugify, hpa_test_curtime, hpa_test_ms_since, hpa_test_vectorized_purge }; static bool hpa_base_fail_after_new; static unsigned hpa_base_alloc_calls; static void hpa_base_hooks_reset(void) { hpa_base_fail_after_new = false; hpa_base_alloc_calls = 0; } static void * hpa_base_alloc(extent_hooks_t *extent_hooks, void *new_addr, size_t size, size_t alignment, bool *zero, bool *commit, unsigned arena_ind) { if (hpa_base_fail_after_new && hpa_base_alloc_calls > 0) { hpa_base_alloc_calls++; return NULL; } hpa_base_alloc_calls++; return pages_map(new_addr, size, alignment, commit); } static bool hpa_base_dalloc(extent_hooks_t *extent_hooks, void *addr, size_t size, bool committed, unsigned arena_ind) { pages_unmap(addr, size); return false; } static void hpa_base_destroy(extent_hooks_t *extent_hooks, void *addr, size_t size, bool committed, unsigned arena_ind) { pages_unmap(addr, size); } static extent_hooks_t hpa_base_hooks = { hpa_base_alloc, hpa_base_dalloc, hpa_base_destroy, NULL, /* commit */ NULL, /* decommit */ NULL, /* purge_lazy */ NULL, /* purge_forced */ NULL, /* split */ NULL /* merge */ }; static hpdata_t * hpa_central_extract_with_lock(hpa_central_t *central, malloc_mutex_t *mtx, uint64_t age, bool hugify_eager, bool *oom) { tsdn_t *tsdn = tsdn_fetch(); malloc_mutex_lock(tsdn, mtx); hpdata_t *ret = hpa_central_extract(tsdn, central, PAGE, age, hugify_eager, oom); malloc_mutex_unlock(tsdn, mtx); return ret; } static void hpa_test_shard_grow_mtx_init(malloc_mutex_t *mtx) { assert_false(malloc_mutex_init(mtx, "hpa_test_shard_grow", WITNESS_RANK_HPA_SHARD_GROW, malloc_mutex_rank_exclusive), "Unexpected mutex initialization failure"); } TEST_BEGIN(test_hpa_central_extract_eden) { test_skip_if(!hpa_supported() || hpa_hugepage_size_exceeds_limit()); hpa_test_hooks_reset(); tsdn_t *tsdn = tsdn_fetch(); base_t *base = base_new(tsdn, HPA_TEST_ARENA_IND, &ehooks_default_extent_hooks, /* metadata_use_hooks */ true); assert_ptr_not_null(base, "Unexpected base_new failure"); hpa_central_t central; assert_false(hpa_central_init(¢ral, base, &hpa_test_hooks), "Unexpected hpa_central_init failure"); malloc_mutex_t shard_grow_mtx; hpa_test_shard_grow_mtx_init(&shard_grow_mtx); void *eden = NULL; for (unsigned i = 0; i < 128; i++) { bool oom = true; hpdata_t *ps = hpa_central_extract_with_lock(¢ral, &shard_grow_mtx, 1000 + i, /* hugify_eager */ true, &oom); expect_false(oom, "Unexpected HPA central OOM"); expect_ptr_not_null(ps, "Unexpected HPA central extraction failure"); if (i == 0) { eden = hpa_test_last_map; expect_u_eq(1, hpa_test_map_calls, "First extraction should map eden"); expect_zu_eq(HPA_TEST_EDEN_SIZE, hpa_test_last_map_size, "Unexpected eden mapping size"); expect_u_eq(1, hpa_test_hugify_calls, "Eager extraction should hugify the whole eden"); expect_ptr_eq(eden, hpa_test_last_hugify, "Hugify should apply to eden"); expect_zu_eq(HPA_TEST_EDEN_SIZE, hpa_test_last_hugify_size, "Hugify should cover the whole eden"); } expect_ptr_eq((void *)((byte_t *)eden + i * HUGEPAGE), hpdata_addr_get(ps), "Unexpected extracted pageslab addr"); expect_u64_eq(1000 + i, hpdata_age_get(ps), "Unexpected hpdata age"); expect_true(hpdata_huge_get(ps), "Eager extraction should mark hpdata huge"); } expect_ptr_null(central.eden, "Exact final extraction should empty eden"); expect_zu_eq(0, central.eden_len, "Exact final extraction should clear eden length"); expect_u_eq(1, hpa_test_map_calls, "All pageslabs should come from one eden mapping"); expect_u_eq(0, hpa_test_unmap_calls, "Successful extraction should not unmap eden"); pages_unmap(eden, HPA_TEST_EDEN_SIZE); base_delete(tsdn, base); } TEST_END TEST_BEGIN(test_hpa_central_failure_paths) { test_skip_if(!hpa_supported() || hpa_hugepage_size_exceeds_limit()); hpa_test_hooks_reset(); tsdn_t *tsdn = tsdn_fetch(); base_t *base = base_new(tsdn, HPA_TEST_ARENA_IND + 1, &ehooks_default_extent_hooks, /* metadata_use_hooks */ true); assert_ptr_not_null(base, "Unexpected base_new failure"); hpa_central_t central; assert_false(hpa_central_init(¢ral, base, &hpa_test_hooks), "Unexpected hpa_central_init failure"); malloc_mutex_t shard_grow_mtx; hpa_test_shard_grow_mtx_init(&shard_grow_mtx); hpa_test_map_fail = true; bool oom = false; hpdata_t *ps = hpa_central_extract_with_lock(¢ral, &shard_grow_mtx, 1, /* hugify_eager */ false, &oom); expect_ptr_null(ps, "Map failure should not return hpdata"); expect_true(oom, "Map failure should report OOM"); expect_u_eq(1, hpa_test_map_calls, "Expected one map attempt"); expect_u_eq(0, hpa_test_unmap_calls, "Map failure should not call unmap"); base_delete(tsdn, base); hpa_base_hooks_reset(); hpa_test_hooks_reset(); base = base_new(tsdn, HPA_TEST_ARENA_IND + 2, &hpa_base_hooks, /* metadata_use_hooks */ true); assert_ptr_not_null(base, "Unexpected base_new failure"); hpa_base_fail_after_new = true; while (base_alloc(tsdn, base, sizeof(hpdata_t), CACHELINE) != NULL) { } assert_false(hpa_central_init(¢ral, base, &hpa_test_hooks), "Unexpected hpa_central_init failure"); malloc_mutex_t shard_grow_mtx2; hpa_test_shard_grow_mtx_init(&shard_grow_mtx2); oom = false; ps = hpa_central_extract_with_lock(¢ral, &shard_grow_mtx2, 2, /* hugify_eager */ false, &oom); expect_ptr_null(ps, "Metadata OOM should not return hpdata"); expect_true(oom, "Metadata allocation failure should report OOM"); expect_u_eq(1, hpa_test_map_calls, "Metadata OOM should happen after mapping eden once"); expect_u_eq(1, hpa_test_unmap_calls, "Metadata OOM should unmap the freshly mapped eden"); expect_ptr_eq(hpa_test_last_map, hpa_test_last_unmap, "Metadata OOM should unmap the eden it just mapped"); expect_zu_eq(HPA_TEST_EDEN_SIZE, hpa_test_last_unmap_size, "Metadata OOM should unmap the full eden"); base_delete(tsdn, base); } TEST_END int main(void) { return test_no_reentrancy(test_hpa_central_extract_eden, test_hpa_central_failure_paths); } @ 1.1.1.1 log @Import jemalloc 5.4.0 (previous was 5.3.1) * 5.4.0 (Sep 17, 2026) This release contains over 160 commits, focusing on the technical debts cleaning including refactorings, bug fixes, test coverage improvement, and option cleanups. The release also includes portability improvements per upstream issues report. New features: - Add `EXTENT_ALLOC_FLAG_PINNED` so custom extent-allocation hooks can mark non-reclaimable mappings, such as HugeTLB pages, for preferential reuse outside the decay and purge pipeline. Add the mallctl interfaces `stats.pinned`, `stats.arenas..pinned`, `stats.arenas..extents..npinned`, `stats.arenas..extents..pinned_bytes`, and `stats.arenas..mutexes.extents_pinned.{counter}` to report pinned-memory usage and mutex statistics. (@@binliu19: be2de8cc) - Allow resuming per-CPU arena selection via `thread.arena`. (@@algunenano: 68c35f65) - Better align the contents of human-readable and JSON malloc statistics. (@@spredolac: 68fe1be1, 30b1a410, 04aad97f) - Replace the runtime `experimental_infallible_new` option with the compile-time `--enable-cxx-infallible-new` option, enabling compiler-level optimizations and optimization in move constructors, and fix the `new(std::nothrow)` contract (@@spredolac: 160ab9d7, fe336672) Incompatible changes: - Adapt tcache fill and retention targets per bin to demand observed between GC events, replacing the fixed refill/flush policy. Remove seven legacy non-experimental controls: `lg_tcache_nslots_mul`, `tcache_nslots_small_min`, `tcache_nslots_small_max`, `tcache_nslots_large`, `tcache_gc_delay_bytes`, `lg_tcache_flush_small_div`, and `lg_tcache_flush_large_div`. Corresponding `malloc_conf` settings are silently ignored, matching `opt.*` mallctls return ENOENT, and `tcache_ncached_max` remains supported. (@@spredolac: d13fe91a) Bug fixes: - Preserve `errno` across `free`, `free_sized`, and `free_aligned_sized`, and across `process_madvise`-based page purging. (@@spredolac: 3a779661, 86f05828) - Fix numeric overflow checks in size classes. (@@spredolac: 6b245225) - Accept NULL in `free_sized()` and `free_aligned_sized()` (C23 correctness). (@@bigbruno: 7ce8b916) - Fix TSD lifecycle edge cases by 1) initializing thread-cache bins before marking the cache enabled, preventing reentrant bootstrap allocations from using uninitialized state, and 2) avoiding TSD recreation for late deallocations after thread teardown on generic-TSD platforms. (@@fzakaria: 54f22c83, @@spredolac: fb5499aa) - Use `O_CLOEXEC` when opening the THP sysfs file in `init_thp_state`. (@@ibookstein: d0705543) - Fix duplicate `opt.stats_print` and `opt.stats_print_opts` fields in `malloc_stats_print` output. (@@spredolac: dfe3a2ed) - Fix a potential deadlock during `arena_reset`. (@@guangli-dai: 6957341f) - Fix a prof-sampling / guard-page interaction bug in the SAN. (@@gctony: e36a0fa5) Optimizations and refactors: - Modularize jemalloc's front end by extracting arena management, initialization, fork orchestration, and allocation dispatch from `jemalloc.c`; untangle tcache/arena ownership; and consolidate the internal header graph to eliminate circular dependencies. (@@spredolac: ba1e2fe4, 88745978, 9d757223, de9ad145, ...) - Simplify ctl dispatch by refactoring arena helpers, internalizing an implementation-only control, replacing control-flow macros with typed helpers, and organizing `ctl.c` by subsystem. (@@spredolac: 9e143468, 901365d4, 4e903a0a, 5bc8d6e9) - Cap the base-block growth heuristic to avoid virtual-memory exhaustion under rare racy conditions. (@@guangli-dai: 2f4db8cf) - Move background-thread lifecycle and state operations into the background-thread module, clarifying ownership independently of PAC/HPA callers. (@@guangli-dai: f1f07923) - Simplify the page-allocation boundary by replacing PAI vtable dispatch with direct PAC/HPA calls, removing the obsolete `pai_t`/`pai.h` abstraction, and moving deferred-work and decay orchestration out of the arena. (@@guangli-dai: 1dfa6f7a, 8edd1012, d410f43e) - Refactor statistics collection and rendering into separate gather/emission stages and descriptor-driven tables. (@@spredolac: 184f304d, 80c8fcb4) - Introduce an OS abstraction layer and move platform-dependent file/process I/O, time, synchronization, CPU, virtual-memory, atfork, error-handling, profiling, thread-yield, and configuration-access operations out of allocator core code. (@@guangli-dai: c4158aca, c8e2e013, ...) Portability improvements: - Replace the `std::__throw_bad_alloc` call with standard C++ (#2900). (@@lexprfuncall: 1a15fe33) - Make `arena_s` use a flexible array member (`bin_t all_bins[]`) for C99 or newer. (@@grueninger: 300b58b4) - Fix rdtscp detection with `--with-lg-vaddr`. (@@xinydev: e8a0d2b4) - Fix `malloc_getcpu` on macOS to read the current CPU number correctly. (@@algunenano: 5aabbc87) - Use `CLOCK_MONOTONIC` for background-thread sleep to prevent clock-rollback stalls, and detect monotonic-condvar support at configure time. (@@antonio2368: ebacec3a, 8361239b) - Fix compilation warnings on macOS. (@@gctony: abb0a8a0) - Fix thread-exit TSD cleanup on MinGW builds. (@@gctony: 1e923170) - Fix GCC 16 build warnings by removing `-Wpedantic` violations in macro and function syntax and explicitly NUL-terminating profiling thread-name copies to resolve `-Wstringop-truncation`. (@@grueninger: 5acdcee6, c22d929a, @@jasonangelov: 6a245e02) - Parse PID-namespace symlinks without glibc-dependent `strtok`/`atol`, and return identifiers as `uint64_t`. (@@guangli-dai: 278d90a5) @ text @@