head 1.1; branch 1.1.1; access ; symbols jemalloc-5-4-0:1.1.1.1 JASONE:1.1.1; locks ; strict; comment @ * @; 1.1 date 2026.09.24.18.28.52; author christos; state Exp; branches 1.1.1.1; next ; commitid GeR2ar6mKPT47UWG; 1.1.1.1 date 2026.09.24.18.28.52; author christos; state Exp; branches ; next ; commitid GeR2ar6mKPT47UWG; desc @@ 1.1 log @Initial revision @ text @#include "test/jemalloc_test.h" #include "jemalloc/internal/pa.h" /* * pac_decay_deferred is JET_EXTERN (test-only; see src/pac.c) and has no public * header declaration, so declare it locally for the deterministic test below. */ extern void pac_decay_deferred(tsdn_t *tsdn, pac_t *pac, pac_purge_eagerness_t eagerness, pac_deferred_work_result_t *result); #define PAC_TEST_ARENA_IND 124 static bool pac_test_dalloc_fail; static bool pac_test_purge_lazy_fail; static unsigned pac_test_alloc_calls; static unsigned pac_test_dalloc_calls; static unsigned pac_test_destroy_calls; static unsigned pac_test_purge_lazy_calls; static void pac_test_hooks_reset(void) { pac_test_dalloc_fail = false; pac_test_purge_lazy_fail = false; pac_test_alloc_calls = 0; pac_test_dalloc_calls = 0; pac_test_destroy_calls = 0; pac_test_purge_lazy_calls = 0; } static void * pac_test_alloc(extent_hooks_t *extent_hooks, void *new_addr, size_t size, size_t alignment, bool *zero, bool *commit, unsigned arena_ind) { pac_test_alloc_calls++; void *ret = pages_map(new_addr, size, alignment, commit); return ret; } static bool pac_test_dalloc(extent_hooks_t *extent_hooks, void *addr, size_t size, bool committed, unsigned arena_ind) { pac_test_dalloc_calls++; if (pac_test_dalloc_fail) { return true; } pages_unmap(addr, size); return false; } static void pac_test_destroy(extent_hooks_t *extent_hooks, void *addr, size_t size, bool committed, unsigned arena_ind) { pac_test_destroy_calls++; pages_unmap(addr, size); } static bool pac_test_purge_lazy(extent_hooks_t *extent_hooks, void *addr, size_t size, size_t offset, size_t length, unsigned arena_ind) { pac_test_purge_lazy_calls++; return pac_test_purge_lazy_fail; } static bool pac_test_split(extent_hooks_t *extent_hooks, void *addr, size_t size, size_t size_a, size_t size_b, bool committed, unsigned arena_ind) { return !maps_coalesce && !opt_retain; } static bool pac_test_merge(extent_hooks_t *extent_hooks, void *addr_a, size_t size_a, void *addr_b, size_t size_b, bool committed, unsigned arena_ind) { return !maps_coalesce && !opt_retain; } static extent_hooks_t pac_test_hooks = { pac_test_alloc, pac_test_dalloc, pac_test_destroy, NULL, /* commit */ NULL, /* decommit */ pac_test_purge_lazy, NULL, /* purge_forced */ pac_test_split, pac_test_merge }; typedef struct pac_test_data_s pac_test_data_t; struct pac_test_data_s { pa_shard_t shard; pa_central_t central; base_t *base; emap_t emap; pa_shard_stats_t stats; malloc_mutex_t stats_mtx; extent_hooks_t hooks; }; static pac_test_data_t * pac_test_data_init(bool custom_hooks, ssize_t dirty_decay_ms, ssize_t muzzy_decay_ms) { tsdn_t *tsdn = tsdn_fetch(); pac_test_data_t *data = calloc(1, sizeof(*data)); assert_ptr_not_null(data, "Unexpected calloc failure"); if (custom_hooks) { memcpy(&data->hooks, &pac_test_hooks, sizeof(extent_hooks_t)); } else { memcpy(&data->hooks, &ehooks_default_extent_hooks, sizeof(extent_hooks_t)); } data->base = base_new(tsdn, PAC_TEST_ARENA_IND, &data->hooks, /* metadata_use_hooks */ true); assert_ptr_not_null(data->base, "Unexpected base_new failure"); assert_false(emap_init(&data->emap, data->base, /* zeroed */ true), "Unexpected emap_init failure"); assert_false(malloc_mutex_init(&data->stats_mtx, "pac_test_stats", WITNESS_RANK_ARENA_STATS, malloc_mutex_rank_exclusive), "Unexpected stats mutex initialization failure"); nstime_t time; nstime_init(&time, 0); assert_false(pa_central_init(&data->central, data->base, /* hpa */ false, &hpa_hooks_default), "Unexpected pa_central_init failure"); assert_false(pa_shard_init(tsdn, &data->shard, &data->central, &data->emap, data->base, PAC_TEST_ARENA_IND, &data->stats, &data->stats_mtx, &time, SC_LARGE_MAXCLASS + PAGE, dirty_decay_ms, muzzy_decay_ms), "Unexpected pa_shard_init failure"); return data; } static void pac_decay_all_locked(pac_t *pac, extent_state_t state, bool fully_decay) { decay_t *decay; pac_decay_stats_t *decay_stats; ecache_t *ecache; if (state == extent_state_dirty) { decay = &pac->decay_dirty; decay_stats = &pac->stats->decay_dirty; ecache = &pac->ecache_dirty; } else { assert_d_eq(extent_state_muzzy, state, "Only dirty and muzzy decay are supported"); decay = &pac->decay_muzzy; decay_stats = &pac->stats->decay_muzzy; ecache = &pac->ecache_muzzy; } tsdn_t *tsdn = tsdn_fetch(); malloc_mutex_lock(tsdn, &decay->mtx); pac_decay_all(tsdn, pac, decay, decay_stats, ecache, fully_decay); malloc_mutex_unlock(tsdn, &decay->mtx); } static void pac_test_data_destroy(pac_test_data_t *data) { /* * Decay below calls back into the test hooks; reset all hook state * (including the fail flags) so teardown is unaffected by anything the * preceding test toggled. */ pac_test_hooks_reset(); pac_decay_all_locked(&data->shard.pac, extent_state_dirty, /* fully_decay */ true); pac_decay_all_locked(&data->shard.pac, extent_state_muzzy, /* fully_decay */ true); pa_shard_destroy(tsdn_fetch(), &data->shard); base_delete(tsdn_fetch(), data->base); /* * pac operations populated the tsd rtree_ctx with leaf-node pointers * from the private emap we just destroyed. Invalidate the cache so * the next test's fresh emap doesn't follow stale entries. */ rtree_ctx_data_init(tsd_rtree_ctx(tsd_fetch())); free(data); } static edata_t * pac_alloc_expect(pac_test_data_t *data, size_t size, bool guarded) { bool deferred_work_generated = false; edata_t *edata = pac_alloc(tsdn_fetch(), &data->shard.pac, size, PAGE, /* zero */ false, guarded, /* frequent_reuse */ false, &deferred_work_generated); expect_ptr_not_null(edata, "Unexpected pac_alloc failure"); expect_zu_eq(size, edata_size_get(edata), "Unexpected allocation size"); return edata; } TEST_BEGIN(test_pac_dirty_muzzy_alloc_priority) { pac_test_hooks_reset(); pac_test_data_t *data = pac_test_data_init( /* custom_hooks */ true, -1, -1); pac_t *pac = &data->shard.pac; size_t alloc_size = HUGEPAGE; size_t alloc_npages = alloc_size >> LG_PAGE; edata_t *muzzy = pac_alloc_expect( data, alloc_size, /* guarded */ false); void *muzzy_addr = edata_base_get(muzzy); edata_t *dirty = pac_alloc_expect( data, alloc_size, /* guarded */ false); void *dirty_addr = edata_base_get(dirty); bool deferred_work_generated = false; pac_dalloc(tsdn_fetch(), pac, muzzy, &deferred_work_generated); pac_decay_all_locked(pac, extent_state_dirty, /* fully_decay */ false); expect_zu_eq(alloc_npages, ecache_npages_get(&pac->ecache_muzzy), "Expected one muzzy page after dirty decay"); deferred_work_generated = false; pac_dalloc(tsdn_fetch(), pac, dirty, &deferred_work_generated); expect_zu_eq(alloc_npages, ecache_npages_get(&pac->ecache_dirty), "Expected one dirty page"); edata_t *from_dirty = pac_alloc_expect( data, alloc_size, /* guarded */ false); expect_ptr_eq(dirty_addr, edata_base_get(from_dirty), "Dirty cache should be preferred over muzzy"); edata_t *from_muzzy = pac_alloc_expect( data, alloc_size, /* guarded */ false); expect_ptr_eq(muzzy_addr, edata_base_get(from_muzzy), "Muzzy cache should be used after dirty cache"); deferred_work_generated = false; pac_dalloc(tsdn_fetch(), pac, from_dirty, &deferred_work_generated); deferred_work_generated = false; pac_dalloc(tsdn_fetch(), pac, from_muzzy, &deferred_work_generated); pac_test_data_destroy(data); } TEST_END TEST_BEGIN(test_pac_batched_grow_caches_trailing_dirty) { test_skip_if(!sz_large_size_classes_disabled() || !(maps_coalesce || opt_retain)); pac_test_hooks_reset(); pac_test_data_t *data = pac_test_data_init( /* custom_hooks */ true, -1, -1); pac_t *pac = &data->shard.pac; size_t size = HUGEPAGE + PAGE; size_t batched_size = sz_s2u_compute_using_delta(size); size_t next_hugepage_size = HUGEPAGE_CEILING(size); if (batched_size > next_hugepage_size) { batched_size = next_hugepage_size; } assert_zu_gt(batched_size, size, "Test size should exercise batched retained growth"); size_t dirty_before = ecache_npages_get(&pac->ecache_dirty); edata_t *large = pac_alloc_expect(data, size, /* guarded */ false); expect_zu_eq(dirty_before + ((batched_size - size) >> LG_PAGE), ecache_npages_get(&pac->ecache_dirty), "Batched grow should cache the trailing dirty extent"); if (config_stats) { expect_zu_ge(pac_mapped(pac), batched_size, "Mapped stats should include the full batched grow"); } bool deferred_work_generated = false; pac_dalloc(tsdn_fetch(), pac, large, &deferred_work_generated); pac_test_data_destroy(data); } TEST_END TEST_BEGIN(test_pac_deferred_work_signals) { pac_test_hooks_reset(); pac_test_data_t *data = pac_test_data_init( /* custom_hooks */ true, -1, -1); pac_t *pac = &data->shard.pac; edata_t *edata = pac_alloc_expect(data, PAGE, /* guarded */ false); bool deferred_work_generated = false; pac_dalloc(tsdn_fetch(), pac, edata, &deferred_work_generated); expect_true(deferred_work_generated, "Non-pinned dalloc should request deferred work"); pac_test_data_destroy(data); } TEST_END TEST_BEGIN(test_pac_decay_dirty_to_muzzy_via_purge_lazy) { pac_test_hooks_reset(); pac_test_data_t *data = pac_test_data_init( /* custom_hooks */ true, -1, -1); pac_t *pac = &data->shard.pac; edata_t *edata = pac_alloc_expect(data, PAGE, /* guarded */ false); bool deferred_work_generated = false; pac_dalloc(tsdn_fetch(), pac, edata, &deferred_work_generated); unsigned purge_lazy_before = pac_test_purge_lazy_calls; pac_decay_all_locked(pac, extent_state_dirty, /* fully_decay */ false); expect_zu_eq(0, ecache_npages_get(&pac->ecache_dirty), "Dirty decay should remove dirty pages"); expect_zu_eq(1, ecache_npages_get(&pac->ecache_muzzy), "Successful lazy purge should move dirty pages to muzzy"); expect_u_gt(pac_test_purge_lazy_calls, purge_lazy_before, "Dirty-to-muzzy decay should call purge_lazy"); pac_test_data_destroy(data); } TEST_END TEST_BEGIN(test_pac_decay_retains_when_dalloc_fails) { pac_test_hooks_reset(); pac_test_data_t *data = pac_test_data_init( /* custom_hooks */ true, -1, -1); pac_t *pac = &data->shard.pac; pac_test_dalloc_fail = true; edata_t *edata = pac_alloc_expect(data, PAGE, /* guarded */ false); bool deferred_work_generated = false; pac_dalloc(tsdn_fetch(), pac, edata, &deferred_work_generated); size_t retained_before = ecache_npages_get(&pac->ecache_retained); pac_decay_all_locked(pac, extent_state_dirty, /* fully_decay */ true); expect_zu_gt(ecache_npages_get(&pac->ecache_retained), retained_before, "Fully decayed dirty pages should be retained when dalloc fails"); expect_u_gt(pac_test_dalloc_calls, 0, "Fully decayed dirty pages should attempt dalloc first"); pac_test_data_destroy(data); } TEST_END TEST_BEGIN(test_pac_non_pinned_dalloc_signals_deferred_work) { pac_test_hooks_reset(); pac_test_data_t *data = pac_test_data_init( /* custom_hooks */ true, -1, -1); pac_t *pac = &data->shard.pac; size_t normal_size = 3 * HUGEPAGE + PAGE; edata_t *normal = pac_alloc_expect(data, normal_size, /* guarded */ false); bool deferred_work_generated = false; pac_dalloc(tsdn_fetch(), pac, normal, &deferred_work_generated); expect_true(deferred_work_generated, "Non-pinned dalloc should request deferred work"); pac_test_data_destroy(data); } TEST_END TEST_BEGIN(test_pac_non_pinned_shrink_signals_deferred_work) { test_skip_if(!maps_coalesce); pac_test_hooks_reset(); pac_test_data_t *data = pac_test_data_init( /* custom_hooks */ true, -1, -1); pac_t *pac = &data->shard.pac; size_t normal_size = 3 * HUGEPAGE + PAGE; size_t normal_shrink_size = 3 * HUGEPAGE; edata_t *normal = pac_alloc_expect(data, normal_size, /* guarded */ false); bool deferred_work_generated = false; expect_false(pac_shrink(tsdn_fetch(), pac, normal, normal_size, normal_shrink_size, &deferred_work_generated), "Unexpected non-pinned shrink failure"); expect_true(deferred_work_generated, "Non-pinned shrink should request deferred work"); pac_dalloc(tsdn_fetch(), pac, normal, &deferred_work_generated); pac_test_data_destroy(data); } TEST_END TEST_BEGIN(test_pac_large_guarded_dalloc_unguards_before_caching) { pac_test_hooks_reset(); pac_test_data_t *data = pac_test_data_init( /* custom_hooks */ true, -1, -1); edata_t *guarded = pac_alloc_expect(data, SC_LARGE_MINCLASS, /* guarded */ true); expect_true(edata_guarded_get(guarded), "Guarded allocation should set the guarded bit"); bool deferred_work_generated = false; pac_dalloc(tsdn_fetch(), &data->shard.pac, guarded, &deferred_work_generated); expect_true(deferred_work_generated, "Guarded dalloc should still request deferred work"); expect_false(edata_guarded_get(guarded), "Large guarded dalloc should unguard before caching"); pac_test_data_destroy(data); } TEST_END TEST_BEGIN(test_pac_deferred_never_does_not_purge) { /* * PAC_PURGE_NEVER is the eagerness the application free path selects * when a background thread is enabled, meaning current thread does not * purge. It is only reachable through the exposed pac_decay_deferred * primitive (the production entry pac_do_deferred_work picks eagerness * itself), so this is a deliberate internal test of that critical, * stable edge. */ pac_test_hooks_reset(); /* * Finite dirty decay so a purge is gated by eagerness, not decay_ms<=0. */ pac_test_data_t *data = pac_test_data_init( /* custom_hooks */ true, /* dirty_decay_ms */ 100 * 1000, -1); pac_t *pac = &data->shard.pac; edata_t *edata = pac_alloc_expect(data, PAGE, /* guarded */ false); size_t ndirty_before_dalloc = ecache_npages_get(&pac->ecache_dirty); /* Dalloc caches the extent as dirty and requests deferred work. */ bool deferred_work_generated = false; pac_dalloc(tsdn_fetch(), pac, edata, &deferred_work_generated); expect_true(deferred_work_generated, "Non-pinned dalloc should request deferred work"); size_t ndirty_after_dalloc = ecache_npages_get(&pac->ecache_dirty); expect_zu_gt(ndirty_after_dalloc, ndirty_before_dalloc, "Dalloc should add dirty pages"); /* PAC_PURGE_NEVER must leave the dirty pages untouched. */ pac_deferred_work_result_t result; pac_decay_deferred(tsdn_fetch(), pac, PAC_PURGE_NEVER, &result); expect_zu_eq(ecache_npages_get(&pac->ecache_dirty), ndirty_after_dalloc, "PAC_PURGE_NEVER must not purge any dirty pages"); pac_test_data_destroy(data); } TEST_END int main(void) { return test_no_reentrancy(test_pac_dirty_muzzy_alloc_priority, test_pac_batched_grow_caches_trailing_dirty, test_pac_deferred_work_signals, test_pac_decay_dirty_to_muzzy_via_purge_lazy, test_pac_decay_retains_when_dalloc_fails, test_pac_non_pinned_dalloc_signals_deferred_work, test_pac_non_pinned_shrink_signals_deferred_work, test_pac_large_guarded_dalloc_unguards_before_caching, test_pac_deferred_never_does_not_purge); } @ 1.1.1.1 log @Import jemalloc 5.4.0 (previous was 5.3.1) * 5.4.0 (Sep 17, 2026) This release contains over 160 commits, focusing on the technical debts cleaning including refactorings, bug fixes, test coverage improvement, and option cleanups. The release also includes portability improvements per upstream issues report. New features: - Add `EXTENT_ALLOC_FLAG_PINNED` so custom extent-allocation hooks can mark non-reclaimable mappings, such as HugeTLB pages, for preferential reuse outside the decay and purge pipeline. Add the mallctl interfaces `stats.pinned`, `stats.arenas..pinned`, `stats.arenas..extents..npinned`, `stats.arenas..extents..pinned_bytes`, and `stats.arenas..mutexes.extents_pinned.{counter}` to report pinned-memory usage and mutex statistics. (@@binliu19: be2de8cc) - Allow resuming per-CPU arena selection via `thread.arena`. (@@algunenano: 68c35f65) - Better align the contents of human-readable and JSON malloc statistics. (@@spredolac: 68fe1be1, 30b1a410, 04aad97f) - Replace the runtime `experimental_infallible_new` option with the compile-time `--enable-cxx-infallible-new` option, enabling compiler-level optimizations and optimization in move constructors, and fix the `new(std::nothrow)` contract (@@spredolac: 160ab9d7, fe336672) Incompatible changes: - Adapt tcache fill and retention targets per bin to demand observed between GC events, replacing the fixed refill/flush policy. Remove seven legacy non-experimental controls: `lg_tcache_nslots_mul`, `tcache_nslots_small_min`, `tcache_nslots_small_max`, `tcache_nslots_large`, `tcache_gc_delay_bytes`, `lg_tcache_flush_small_div`, and `lg_tcache_flush_large_div`. Corresponding `malloc_conf` settings are silently ignored, matching `opt.*` mallctls return ENOENT, and `tcache_ncached_max` remains supported. (@@spredolac: d13fe91a) Bug fixes: - Preserve `errno` across `free`, `free_sized`, and `free_aligned_sized`, and across `process_madvise`-based page purging. (@@spredolac: 3a779661, 86f05828) - Fix numeric overflow checks in size classes. (@@spredolac: 6b245225) - Accept NULL in `free_sized()` and `free_aligned_sized()` (C23 correctness). (@@bigbruno: 7ce8b916) - Fix TSD lifecycle edge cases by 1) initializing thread-cache bins before marking the cache enabled, preventing reentrant bootstrap allocations from using uninitialized state, and 2) avoiding TSD recreation for late deallocations after thread teardown on generic-TSD platforms. (@@fzakaria: 54f22c83, @@spredolac: fb5499aa) - Use `O_CLOEXEC` when opening the THP sysfs file in `init_thp_state`. (@@ibookstein: d0705543) - Fix duplicate `opt.stats_print` and `opt.stats_print_opts` fields in `malloc_stats_print` output. (@@spredolac: dfe3a2ed) - Fix a potential deadlock during `arena_reset`. (@@guangli-dai: 6957341f) - Fix a prof-sampling / guard-page interaction bug in the SAN. (@@gctony: e36a0fa5) Optimizations and refactors: - Modularize jemalloc's front end by extracting arena management, initialization, fork orchestration, and allocation dispatch from `jemalloc.c`; untangle tcache/arena ownership; and consolidate the internal header graph to eliminate circular dependencies. (@@spredolac: ba1e2fe4, 88745978, 9d757223, de9ad145, ...) - Simplify ctl dispatch by refactoring arena helpers, internalizing an implementation-only control, replacing control-flow macros with typed helpers, and organizing `ctl.c` by subsystem. (@@spredolac: 9e143468, 901365d4, 4e903a0a, 5bc8d6e9) - Cap the base-block growth heuristic to avoid virtual-memory exhaustion under rare racy conditions. (@@guangli-dai: 2f4db8cf) - Move background-thread lifecycle and state operations into the background-thread module, clarifying ownership independently of PAC/HPA callers. (@@guangli-dai: f1f07923) - Simplify the page-allocation boundary by replacing PAI vtable dispatch with direct PAC/HPA calls, removing the obsolete `pai_t`/`pai.h` abstraction, and moving deferred-work and decay orchestration out of the arena. (@@guangli-dai: 1dfa6f7a, 8edd1012, d410f43e) - Refactor statistics collection and rendering into separate gather/emission stages and descriptor-driven tables. (@@spredolac: 184f304d, 80c8fcb4) - Introduce an OS abstraction layer and move platform-dependent file/process I/O, time, synchronization, CPU, virtual-memory, atfork, error-handling, profiling, thread-yield, and configuration-access operations out of allocator core code. (@@guangli-dai: c4158aca, c8e2e013, ...) Portability improvements: - Replace the `std::__throw_bad_alloc` call with standard C++ (#2900). (@@lexprfuncall: 1a15fe33) - Make `arena_s` use a flexible array member (`bin_t all_bins[]`) for C99 or newer. (@@grueninger: 300b58b4) - Fix rdtscp detection with `--with-lg-vaddr`. (@@xinydev: e8a0d2b4) - Fix `malloc_getcpu` on macOS to read the current CPU number correctly. (@@algunenano: 5aabbc87) - Use `CLOCK_MONOTONIC` for background-thread sleep to prevent clock-rollback stalls, and detect monotonic-condvar support at configure time. (@@antonio2368: ebacec3a, 8361239b) - Fix compilation warnings on macOS. (@@gctony: abb0a8a0) - Fix thread-exit TSD cleanup on MinGW builds. (@@gctony: 1e923170) - Fix GCC 16 build warnings by removing `-Wpedantic` violations in macro and function syntax and explicitly NUL-terminating profiling thread-name copies to resolve `-Wstringop-truncation`. (@@grueninger: 5acdcee6, c22d929a, @@jasonangelov: 6a245e02) - Parse PID-namespace symlinks without glibc-dependent `strtok`/`atol`, and return identifiers as `uint64_t`. (@@guangli-dai: 278d90a5) @ text @@