Lines Matching refs:arena
51 static void arena_decay_to_limit(tsdn_t *tsdn, arena_t *arena,
54 static bool arena_decay_dirty(tsdn_t *tsdn, arena_t *arena,
56 static void arena_dalloc_bin_slab(tsdn_t *tsdn, arena_t *arena, extent_t *slab,
58 static void arena_bin_lower_slab(tsdn_t *tsdn, arena_t *arena, extent_t *slab,
64 arena_basic_stats_merge(UNUSED tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, in arena_basic_stats_merge() argument
67 *nthreads += arena_nthreads_get(arena, false); in arena_basic_stats_merge()
68 *dss = dss_prec_names[arena_dss_prec_get(arena)]; in arena_basic_stats_merge()
69 *dirty_decay_ms = arena_dirty_decay_ms_get(arena); in arena_basic_stats_merge()
70 *muzzy_decay_ms = arena_muzzy_decay_ms_get(arena); in arena_basic_stats_merge()
71 *nactive += atomic_load_zu(&arena->nactive, ATOMIC_RELAXED); in arena_basic_stats_merge()
72 *ndirty += extents_npages_get(&arena->extents_dirty); in arena_basic_stats_merge()
73 *nmuzzy += extents_npages_get(&arena->extents_muzzy); in arena_basic_stats_merge()
77 arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, in arena_stats_merge() argument
83 arena_basic_stats_merge(tsdn, arena, nthreads, dss, dirty_decay_ms, in arena_stats_merge()
87 base_stats_get(tsdn, arena->base, &base_allocated, &base_resident, in arena_stats_merge()
90 arena_stats_lock(tsdn, &arena->stats); in arena_stats_merge()
93 + arena_stats_read_zu(tsdn, &arena->stats, &arena->stats.mapped)); in arena_stats_merge()
95 extents_npages_get(&arena->extents_retained) << LG_PAGE); in arena_stats_merge()
98 arena_stats_read_u64(tsdn, &arena->stats, in arena_stats_merge()
99 &arena->stats.decay_dirty.npurge)); in arena_stats_merge()
101 arena_stats_read_u64(tsdn, &arena->stats, in arena_stats_merge()
102 &arena->stats.decay_dirty.nmadvise)); in arena_stats_merge()
104 arena_stats_read_u64(tsdn, &arena->stats, in arena_stats_merge()
105 &arena->stats.decay_dirty.purged)); in arena_stats_merge()
108 arena_stats_read_u64(tsdn, &arena->stats, in arena_stats_merge()
109 &arena->stats.decay_muzzy.npurge)); in arena_stats_merge()
111 arena_stats_read_u64(tsdn, &arena->stats, in arena_stats_merge()
112 &arena->stats.decay_muzzy.nmadvise)); in arena_stats_merge()
114 arena_stats_read_u64(tsdn, &arena->stats, in arena_stats_merge()
115 &arena->stats.decay_muzzy.purged)); in arena_stats_merge()
118 arena_stats_accum_zu(&astats->internal, arena_internal_get(arena)); in arena_stats_merge()
121 (((atomic_load_zu(&arena->nactive, ATOMIC_RELAXED) + in arena_stats_merge()
122 extents_npages_get(&arena->extents_dirty) + in arena_stats_merge()
123 extents_npages_get(&arena->extents_muzzy)) << LG_PAGE))); in arena_stats_merge()
126 uint64_t nmalloc = arena_stats_read_u64(tsdn, &arena->stats, in arena_stats_merge()
127 &arena->stats.lstats[i].nmalloc); in arena_stats_merge()
131 uint64_t ndalloc = arena_stats_read_u64(tsdn, &arena->stats, in arena_stats_merge()
132 &arena->stats.lstats[i].ndalloc); in arena_stats_merge()
136 uint64_t nrequests = arena_stats_read_u64(tsdn, &arena->stats, in arena_stats_merge()
137 &arena->stats.lstats[i].nrequests); in arena_stats_merge()
151 arena_stats_unlock(tsdn, &arena->stats); in arena_stats_merge()
155 malloc_mutex_lock(tsdn, &arena->tcache_ql_mtx); in arena_stats_merge()
157 ql_foreach(descriptor, &arena->cache_bin_array_descriptor_ql, link) { in arena_stats_merge()
172 &arena->tcache_ql_mtx); in arena_stats_merge()
173 malloc_mutex_unlock(tsdn, &arena->tcache_ql_mtx); in arena_stats_merge()
176 malloc_mutex_lock(tsdn, &arena->mtx); \ in arena_stats_merge()
178 &arena->mtx); \ in arena_stats_merge()
179 malloc_mutex_unlock(tsdn, &arena->mtx); in arena_stats_merge()
199 nstime_copy(&astats->uptime, &arena->create_time); in arena_stats_merge()
201 nstime_subtract(&astats->uptime, &arena->create_time); in arena_stats_merge()
204 bin_stats_merge(tsdn, &bstats[i], &arena->bins[i]); in arena_stats_merge()
209 arena_extents_dirty_dalloc(tsdn_t *tsdn, arena_t *arena, in arena_extents_dirty_dalloc() argument
214 extents_dalloc(tsdn, arena, r_extent_hooks, &arena->extents_dirty, in arena_extents_dirty_dalloc()
216 if (arena_dirty_decay_ms_get(arena) == 0) { in arena_extents_dirty_dalloc()
217 arena_decay_dirty(tsdn, arena, false, true); in arena_extents_dirty_dalloc()
219 arena_background_thread_inactivity_check(tsdn, arena, false); in arena_extents_dirty_dalloc()
278 arena_nactive_add(arena_t *arena, size_t add_pages) { in arena_nactive_add() argument
279 atomic_fetch_add_zu(&arena->nactive, add_pages, ATOMIC_RELAXED); in arena_nactive_add()
283 arena_nactive_sub(arena_t *arena, size_t sub_pages) { in arena_nactive_sub() argument
284 assert(atomic_load_zu(&arena->nactive, ATOMIC_RELAXED) >= sub_pages); in arena_nactive_sub()
285 atomic_fetch_sub_zu(&arena->nactive, sub_pages, ATOMIC_RELAXED); in arena_nactive_sub()
289 arena_large_malloc_stats_update(tsdn_t *tsdn, arena_t *arena, size_t usize) { in arena_large_malloc_stats_update() argument
300 arena_stats_add_u64(tsdn, &arena->stats, in arena_large_malloc_stats_update()
301 &arena->stats.lstats[hindex].nmalloc, 1); in arena_large_malloc_stats_update()
305 arena_large_dalloc_stats_update(tsdn_t *tsdn, arena_t *arena, size_t usize) { in arena_large_dalloc_stats_update() argument
316 arena_stats_add_u64(tsdn, &arena->stats, in arena_large_dalloc_stats_update()
317 &arena->stats.lstats[hindex].ndalloc, 1); in arena_large_dalloc_stats_update()
321 arena_large_ralloc_stats_update(tsdn_t *tsdn, arena_t *arena, size_t oldusize, in arena_large_ralloc_stats_update() argument
323 arena_large_dalloc_stats_update(tsdn, arena, oldusize); in arena_large_ralloc_stats_update()
324 arena_large_malloc_stats_update(tsdn, arena, usize); in arena_large_ralloc_stats_update()
328 arena_extent_alloc_large(tsdn_t *tsdn, arena_t *arena, size_t usize, in arena_extent_alloc_large() argument
338 extent_t *extent = extents_alloc(tsdn, arena, &extent_hooks, in arena_extent_alloc_large()
339 &arena->extents_dirty, NULL, usize, sz_large_pad, alignment, false, in arena_extent_alloc_large()
342 extent = extents_alloc(tsdn, arena, &extent_hooks, in arena_extent_alloc_large()
343 &arena->extents_muzzy, NULL, usize, sz_large_pad, alignment, in arena_extent_alloc_large()
348 extent = extent_alloc_wrapper(tsdn, arena, &extent_hooks, NULL, in arena_extent_alloc_large()
365 arena_stats_lock(tsdn, &arena->stats); in arena_extent_alloc_large()
366 arena_large_malloc_stats_update(tsdn, arena, usize); in arena_extent_alloc_large()
368 arena_stats_add_zu(tsdn, &arena->stats, in arena_extent_alloc_large()
369 &arena->stats.mapped, mapped_add); in arena_extent_alloc_large()
371 arena_stats_unlock(tsdn, &arena->stats); in arena_extent_alloc_large()
373 arena_nactive_add(arena, size >> LG_PAGE); in arena_extent_alloc_large()
380 arena_extent_dalloc_large_prep(tsdn_t *tsdn, arena_t *arena, extent_t *extent) { in arena_extent_dalloc_large_prep() argument
382 arena_stats_lock(tsdn, &arena->stats); in arena_extent_dalloc_large_prep()
383 arena_large_dalloc_stats_update(tsdn, arena, in arena_extent_dalloc_large_prep()
385 arena_stats_unlock(tsdn, &arena->stats); in arena_extent_dalloc_large_prep()
387 arena_nactive_sub(arena, extent_size_get(extent) >> LG_PAGE); in arena_extent_dalloc_large_prep()
391 arena_extent_ralloc_large_shrink(tsdn_t *tsdn, arena_t *arena, extent_t *extent, in arena_extent_ralloc_large_shrink() argument
397 arena_stats_lock(tsdn, &arena->stats); in arena_extent_ralloc_large_shrink()
398 arena_large_ralloc_stats_update(tsdn, arena, oldusize, usize); in arena_extent_ralloc_large_shrink()
399 arena_stats_unlock(tsdn, &arena->stats); in arena_extent_ralloc_large_shrink()
401 arena_nactive_sub(arena, udiff >> LG_PAGE); in arena_extent_ralloc_large_shrink()
405 arena_extent_ralloc_large_expand(tsdn_t *tsdn, arena_t *arena, extent_t *extent, in arena_extent_ralloc_large_expand() argument
411 arena_stats_lock(tsdn, &arena->stats); in arena_extent_ralloc_large_expand()
412 arena_large_ralloc_stats_update(tsdn, arena, oldusize, usize); in arena_extent_ralloc_large_expand()
413 arena_stats_unlock(tsdn, &arena->stats); in arena_extent_ralloc_large_expand()
415 arena_nactive_add(arena, udiff >> LG_PAGE); in arena_extent_ralloc_large_expand()
511 arena_decay_try_purge(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay, in arena_decay_try_purge() argument
515 arena_decay_to_limit(tsdn, arena, decay, extents, false, in arena_decay_try_purge()
546 arena_decay_epoch_advance(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay, in arena_decay_epoch_advance() argument
557 arena_decay_try_purge(tsdn, arena, decay, extents, in arena_decay_epoch_advance()
614 arena_maybe_decay(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay, in arena_maybe_decay() argument
622 arena_decay_to_limit(tsdn, arena, decay, extents, false, in arena_maybe_decay()
659 arena_decay_epoch_advance(tsdn, arena, decay, extents, &time, in arena_maybe_decay()
662 arena_decay_try_purge(tsdn, arena, decay, extents, in arena_maybe_decay()
677 arena_dirty_decay_ms_get(arena_t *arena) { in arena_dirty_decay_ms_get() argument
678 return arena_decay_ms_get(&arena->decay_dirty); in arena_dirty_decay_ms_get()
682 arena_muzzy_decay_ms_get(arena_t *arena) { in arena_muzzy_decay_ms_get() argument
683 return arena_decay_ms_get(&arena->decay_muzzy); in arena_muzzy_decay_ms_get()
687 arena_decay_ms_set(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay, in arena_decay_ms_set() argument
703 arena_maybe_decay(tsdn, arena, decay, extents, false); in arena_decay_ms_set()
710 arena_dirty_decay_ms_set(tsdn_t *tsdn, arena_t *arena, in arena_dirty_decay_ms_set() argument
712 return arena_decay_ms_set(tsdn, arena, &arena->decay_dirty, in arena_dirty_decay_ms_set()
713 &arena->extents_dirty, decay_ms); in arena_dirty_decay_ms_set()
717 arena_muzzy_decay_ms_set(tsdn_t *tsdn, arena_t *arena, in arena_muzzy_decay_ms_set() argument
719 return arena_decay_ms_set(tsdn, arena, &arena->decay_muzzy, in arena_muzzy_decay_ms_set()
720 &arena->extents_muzzy, decay_ms); in arena_muzzy_decay_ms_set()
724 arena_stash_decayed(tsdn_t *tsdn, arena_t *arena, in arena_stash_decayed() argument
734 (extent = extents_evict(tsdn, arena, r_extent_hooks, extents, in arena_stash_decayed()
743 arena_decay_stashed(tsdn_t *tsdn, arena_t *arena, in arena_decay_stashed() argument
755 ssize_t muzzy_decay_ms = arena_muzzy_decay_ms_get(arena); in arena_decay_stashed()
769 !extent_purge_lazy_wrapper(tsdn, arena, in arena_decay_stashed()
772 extents_dalloc(tsdn, arena, r_extent_hooks, in arena_decay_stashed()
773 &arena->extents_muzzy, extent); in arena_decay_stashed()
775 arena, is_background_thread); in arena_decay_stashed()
780 extent_dalloc_wrapper(tsdn, arena, r_extent_hooks, in arena_decay_stashed()
793 arena_stats_lock(tsdn, &arena->stats); in arena_decay_stashed()
794 arena_stats_add_u64(tsdn, &arena->stats, &decay->stats->npurge, in arena_decay_stashed()
796 arena_stats_add_u64(tsdn, &arena->stats, in arena_decay_stashed()
798 arena_stats_add_u64(tsdn, &arena->stats, &decay->stats->purged, in arena_decay_stashed()
800 arena_stats_sub_zu(tsdn, &arena->stats, &arena->stats.mapped, in arena_decay_stashed()
802 arena_stats_unlock(tsdn, &arena->stats); in arena_decay_stashed()
816 arena_decay_to_limit(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay, in arena_decay_to_limit() argument
829 extent_hooks_t *extent_hooks = extent_hooks_get(arena); in arena_decay_to_limit()
834 size_t npurge = arena_stash_decayed(tsdn, arena, &extent_hooks, extents, in arena_decay_to_limit()
837 UNUSED size_t npurged = arena_decay_stashed(tsdn, arena, in arena_decay_to_limit()
848 arena_decay_impl(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay, in arena_decay_impl() argument
852 arena_decay_to_limit(tsdn, arena, decay, extents, all, 0, in arena_decay_impl()
864 bool epoch_advanced = arena_maybe_decay(tsdn, arena, decay, extents, in arena_decay_impl()
875 background_thread_interval_check(tsdn, arena, decay, in arena_decay_impl()
883 arena_decay_dirty(tsdn_t *tsdn, arena_t *arena, bool is_background_thread, in arena_decay_dirty() argument
885 return arena_decay_impl(tsdn, arena, &arena->decay_dirty, in arena_decay_dirty()
886 &arena->extents_dirty, is_background_thread, all); in arena_decay_dirty()
890 arena_decay_muzzy(tsdn_t *tsdn, arena_t *arena, bool is_background_thread, in arena_decay_muzzy() argument
892 return arena_decay_impl(tsdn, arena, &arena->decay_muzzy, in arena_decay_muzzy()
893 &arena->extents_muzzy, is_background_thread, all); in arena_decay_muzzy()
897 arena_decay(tsdn_t *tsdn, arena_t *arena, bool is_background_thread, bool all) { in arena_decay() argument
898 if (arena_decay_dirty(tsdn, arena, is_background_thread, all)) { in arena_decay()
901 arena_decay_muzzy(tsdn, arena, is_background_thread, all); in arena_decay()
905 arena_slab_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *slab) { in arena_slab_dalloc() argument
906 arena_nactive_sub(arena, extent_size_get(slab) >> LG_PAGE); in arena_slab_dalloc()
909 arena_extents_dirty_dalloc(tsdn, arena, &extent_hooks, slab); in arena_slab_dalloc()
936 arena_bin_slabs_full_insert(arena_t *arena, bin_t *bin, extent_t *slab) { in arena_bin_slabs_full_insert() argument
943 if (arena_is_auto(arena)) { in arena_bin_slabs_full_insert()
950 arena_bin_slabs_full_remove(arena_t *arena, bin_t *bin, extent_t *slab) { in arena_bin_slabs_full_remove() argument
951 if (arena_is_auto(arena)) { in arena_bin_slabs_full_remove()
958 arena_reset(tsd_t *tsd, arena_t *arena) { in arena_reset() argument
974 malloc_mutex_lock(tsd_tsdn(tsd), &arena->large_mtx); in arena_reset()
976 for (extent_t *extent = extent_list_first(&arena->large); extent != in arena_reset()
977 NULL; extent = extent_list_first(&arena->large)) { in arena_reset()
981 malloc_mutex_unlock(tsd_tsdn(tsd), &arena->large_mtx); in arena_reset()
997 malloc_mutex_lock(tsd_tsdn(tsd), &arena->large_mtx); in arena_reset()
999 malloc_mutex_unlock(tsd_tsdn(tsd), &arena->large_mtx); in arena_reset()
1004 bin_t *bin = &arena->bins[i]; in arena_reset()
1010 arena_slab_dalloc(tsd_tsdn(tsd), arena, slab); in arena_reset()
1016 arena_slab_dalloc(tsd_tsdn(tsd), arena, slab); in arena_reset()
1021 arena_bin_slabs_full_remove(arena, bin, slab); in arena_reset()
1023 arena_slab_dalloc(tsd_tsdn(tsd), arena, slab); in arena_reset()
1033 atomic_store_zu(&arena->nactive, 0, ATOMIC_RELAXED); in arena_reset()
1037 arena_destroy_retained(tsdn_t *tsdn, arena_t *arena) { in arena_destroy_retained() argument
1047 extent_hooks_t *extent_hooks = extent_hooks_get(arena); in arena_destroy_retained()
1049 while ((extent = extents_evict(tsdn, arena, &extent_hooks, in arena_destroy_retained()
1050 &arena->extents_retained, 0)) != NULL) { in arena_destroy_retained()
1051 extent_destroy_wrapper(tsdn, arena, &extent_hooks, extent); in arena_destroy_retained()
1056 arena_destroy(tsd_t *tsd, arena_t *arena) { in arena_destroy() argument
1057 assert(base_ind_get(arena->base) >= narenas_auto); in arena_destroy()
1058 assert(arena_nthreads_get(arena, false) == 0); in arena_destroy()
1059 assert(arena_nthreads_get(arena, true) == 0); in arena_destroy()
1066 assert(extents_npages_get(&arena->extents_dirty) == 0); in arena_destroy()
1067 assert(extents_npages_get(&arena->extents_muzzy) == 0); in arena_destroy()
1070 arena_destroy_retained(tsd_tsdn(tsd), arena); in arena_destroy()
1082 arena_set(base_ind_get(arena->base), NULL); in arena_destroy()
1088 base_delete(tsd_tsdn(tsd), arena->base); in arena_destroy()
1092 arena_slab_alloc_hard(tsdn_t *tsdn, arena_t *arena, in arena_slab_alloc_hard() argument
1103 slab = extent_alloc_wrapper(tsdn, arena, r_extent_hooks, NULL, in arena_slab_alloc_hard()
1107 arena_stats_mapped_add(tsdn, &arena->stats, in arena_slab_alloc_hard()
1115 arena_slab_alloc(tsdn_t *tsdn, arena_t *arena, szind_t binind, in arena_slab_alloc() argument
1124 extent_t *slab = extents_alloc(tsdn, arena, &extent_hooks, in arena_slab_alloc()
1125 &arena->extents_dirty, NULL, bin_info->slab_size, 0, PAGE, true, in arena_slab_alloc()
1128 slab = extents_alloc(tsdn, arena, &extent_hooks, in arena_slab_alloc()
1129 &arena->extents_muzzy, NULL, bin_info->slab_size, 0, PAGE, in arena_slab_alloc()
1133 slab = arena_slab_alloc_hard(tsdn, arena, &extent_hooks, in arena_slab_alloc()
1146 arena_nactive_add(arena, extent_size_get(slab) >> LG_PAGE); in arena_slab_alloc()
1152 arena_bin_nonfull_slab_get(tsdn_t *tsdn, arena_t *arena, bin_t *bin, in arena_bin_nonfull_slab_get() argument
1169 slab = arena_slab_alloc(tsdn, arena, binind, bin_info); in arena_bin_nonfull_slab_get()
1195 arena_bin_malloc_hard(tsdn_t *tsdn, arena_t *arena, bin_t *bin, in arena_bin_malloc_hard() argument
1201 if (!arena_is_auto(arena) && bin->slabcur != NULL) { in arena_bin_malloc_hard()
1202 arena_bin_slabs_full_insert(arena, bin, bin->slabcur); in arena_bin_malloc_hard()
1205 slab = arena_bin_nonfull_slab_get(tsdn, arena, bin, binind); in arena_bin_malloc_hard()
1225 arena_dalloc_bin_slab(tsdn, arena, slab, in arena_bin_malloc_hard()
1228 arena_bin_lower_slab(tsdn, arena, slab, in arena_bin_malloc_hard()
1235 arena_bin_slabs_full_insert(arena, bin, bin->slabcur); in arena_bin_malloc_hard()
1250 arena_tcache_fill_small(tsdn_t *tsdn, arena_t *arena, tcache_t *tcache, in arena_tcache_fill_small() argument
1257 if (config_prof && arena_prof_accum(tsdn, arena, prof_accumbytes)) { in arena_tcache_fill_small()
1260 bin = &arena->bins[binind]; in arena_tcache_fill_small()
1270 ptr = arena_bin_malloc_hard(tsdn, arena, bin, binind); in arena_tcache_fill_small()
1303 arena_decay_tick(tsdn, arena); in arena_tcache_fill_small()
1321 arena_malloc_small(tsdn_t *tsdn, arena_t *arena, szind_t binind, bool zero) { in arena_malloc_small() argument
1328 bin = &arena->bins[binind]; in arena_malloc_small()
1335 ret = arena_bin_malloc_hard(tsdn, arena, bin, binind); in arena_malloc_small()
1349 if (config_prof && arena_prof_accum(tsdn, arena, usize)) { in arena_malloc_small()
1370 arena_decay_tick(tsdn, arena); in arena_malloc_small()
1375 arena_malloc_hard(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind, in arena_malloc_hard() argument
1377 assert(!tsdn_null(tsdn) || arena != NULL); in arena_malloc_hard()
1380 arena = arena_choose(tsdn_tsd(tsdn), arena); in arena_malloc_hard()
1382 if (unlikely(arena == NULL)) { in arena_malloc_hard()
1387 return arena_malloc_small(tsdn, arena, ind, zero); in arena_malloc_hard()
1389 return large_malloc(tsdn, arena, sz_index2size(ind), zero); in arena_malloc_hard()
1393 arena_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, in arena_palloc() argument
1400 ret = arena_malloc(tsdn, arena, usize, sz_size2index(usize), in arena_palloc()
1404 ret = large_malloc(tsdn, arena, usize, zero); in arena_palloc()
1406 ret = large_palloc(tsdn, arena, usize, alignment, zero); in arena_palloc()
1424 arena_t *arena = extent_arena_get(extent); in arena_prof_promote() local
1431 prof_accum_cancel(tsdn, &arena->prof_accum, usize); in arena_prof_promote()
1469 arena_dissociate_bin_slab(arena_t *arena, extent_t *slab, bin_t *bin) { in arena_dissociate_bin_slab() argument
1483 arena_bin_slabs_full_remove(arena, bin, slab); in arena_dissociate_bin_slab()
1491 arena_dalloc_bin_slab(tsdn_t *tsdn, arena_t *arena, extent_t *slab, in arena_dalloc_bin_slab() argument
1497 arena_slab_dalloc(tsdn, arena, slab); in arena_dalloc_bin_slab()
1506 arena_bin_lower_slab(UNUSED tsdn_t *tsdn, arena_t *arena, extent_t *slab, in arena_bin_lower_slab() argument
1521 arena_bin_slabs_full_insert(arena, bin, bin->slabcur); in arena_bin_lower_slab()
1533 arena_dalloc_bin_locked_impl(tsdn_t *tsdn, arena_t *arena, extent_t *slab, in arena_dalloc_bin_locked_impl() argument
1537 bin_t *bin = &arena->bins[binind]; in arena_dalloc_bin_locked_impl()
1547 arena_dissociate_bin_slab(arena, slab, bin); in arena_dalloc_bin_locked_impl()
1548 arena_dalloc_bin_slab(tsdn, arena, slab, bin); in arena_dalloc_bin_locked_impl()
1550 arena_bin_slabs_full_remove(arena, bin, slab); in arena_dalloc_bin_locked_impl()
1551 arena_bin_lower_slab(tsdn, arena, slab, bin); in arena_dalloc_bin_locked_impl()
1561 arena_dalloc_bin_junked_locked(tsdn_t *tsdn, arena_t *arena, extent_t *extent, in arena_dalloc_bin_junked_locked() argument
1563 arena_dalloc_bin_locked_impl(tsdn, arena, extent, ptr, true); in arena_dalloc_bin_junked_locked()
1567 arena_dalloc_bin(tsdn_t *tsdn, arena_t *arena, extent_t *extent, void *ptr) { in arena_dalloc_bin() argument
1569 bin_t *bin = &arena->bins[binind]; in arena_dalloc_bin()
1572 arena_dalloc_bin_locked_impl(tsdn, arena, extent, ptr, false); in arena_dalloc_bin()
1579 arena_t *arena = extent_arena_get(extent); in arena_dalloc_small() local
1581 arena_dalloc_bin(tsdn, arena, extent, ptr); in arena_dalloc_small()
1582 arena_decay_tick(tsdn, arena); in arena_dalloc_small()
1622 arena_ralloc_move_helper(tsdn_t *tsdn, arena_t *arena, size_t usize, in arena_ralloc_move_helper() argument
1625 return arena_malloc(tsdn, arena, usize, sz_size2index(usize), in arena_ralloc_move_helper()
1632 return ipalloct(tsdn, usize, alignment, zero, tcache, arena); in arena_ralloc_move_helper()
1636 arena_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize, in arena_ralloc() argument
1651 return large_ralloc(tsdn, arena, iealloc(tsdn, ptr), usize, in arena_ralloc()
1659 void *ret = arena_ralloc_move_helper(tsdn, arena, usize, alignment, in arena_ralloc()
1677 arena_dss_prec_get(arena_t *arena) { in arena_dss_prec_get() argument
1678 return (dss_prec_t)atomic_load_u(&arena->dss_prec, ATOMIC_ACQUIRE); in arena_dss_prec_get()
1682 arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec) { in arena_dss_prec_set() argument
1686 atomic_store_u(&arena->dss_prec, (unsigned)dss_prec, ATOMIC_RELEASE); in arena_dss_prec_set()
1719 arena_retain_grow_limit_get_set(tsd_t *tsd, arena_t *arena, size_t *old_limit, in arena_retain_grow_limit_get_set() argument
1733 malloc_mutex_lock(tsd_tsdn(tsd), &arena->extent_grow_mtx); in arena_retain_grow_limit_get_set()
1735 *old_limit = sz_pind2sz(arena->retain_grow_limit); in arena_retain_grow_limit_get_set()
1738 arena->retain_grow_limit = new_ind; in arena_retain_grow_limit_get_set()
1740 malloc_mutex_unlock(tsd_tsdn(tsd), &arena->extent_grow_mtx); in arena_retain_grow_limit_get_set()
1746 arena_nthreads_get(arena_t *arena, bool internal) { in arena_nthreads_get() argument
1747 return atomic_load_u(&arena->nthreads[internal], ATOMIC_RELAXED); in arena_nthreads_get()
1751 arena_nthreads_inc(arena_t *arena, bool internal) { in arena_nthreads_inc() argument
1752 atomic_fetch_add_u(&arena->nthreads[internal], 1, ATOMIC_RELAXED); in arena_nthreads_inc()
1756 arena_nthreads_dec(arena_t *arena, bool internal) { in arena_nthreads_dec() argument
1757 atomic_fetch_sub_u(&arena->nthreads[internal], 1, ATOMIC_RELAXED); in arena_nthreads_dec()
1761 arena_extent_sn_next(arena_t *arena) { in arena_extent_sn_next() argument
1762 return atomic_fetch_add_zu(&arena->extent_sn_next, 1, ATOMIC_RELAXED); in arena_extent_sn_next()
1767 arena_t *arena; in arena_new() local
1780 arena = (arena_t *)base_alloc(tsdn, base, sizeof(arena_t), CACHELINE); in arena_new()
1781 if (arena == NULL) { in arena_new()
1785 atomic_store_u(&arena->nthreads[0], 0, ATOMIC_RELAXED); in arena_new()
1786 atomic_store_u(&arena->nthreads[1], 0, ATOMIC_RELAXED); in arena_new()
1787 arena->last_thd = NULL; in arena_new()
1790 if (arena_stats_init(tsdn, &arena->stats)) { in arena_new()
1794 ql_new(&arena->tcache_ql); in arena_new()
1795 ql_new(&arena->cache_bin_array_descriptor_ql); in arena_new()
1796 if (malloc_mutex_init(&arena->tcache_ql_mtx, "tcache_ql", in arena_new()
1803 if (prof_accum_init(tsdn, &arena->prof_accum)) { in arena_new()
1816 atomic_store_zu(&arena->offset_state, config_debug ? ind : in arena_new()
1817 (size_t)(uintptr_t)arena, ATOMIC_RELAXED); in arena_new()
1820 atomic_store_zu(&arena->extent_sn_next, 0, ATOMIC_RELAXED); in arena_new()
1822 atomic_store_u(&arena->dss_prec, (unsigned)extent_dss_prec_get(), in arena_new()
1825 atomic_store_zu(&arena->nactive, 0, ATOMIC_RELAXED); in arena_new()
1827 extent_list_init(&arena->large); in arena_new()
1828 if (malloc_mutex_init(&arena->large_mtx, "arena_large", in arena_new()
1839 if (extents_init(tsdn, &arena->extents_dirty, extent_state_dirty, in arena_new()
1847 if (extents_init(tsdn, &arena->extents_muzzy, extent_state_muzzy, in arena_new()
1857 if (extents_init(tsdn, &arena->extents_retained, extent_state_retained, in arena_new()
1862 if (arena_decay_init(&arena->decay_dirty, in arena_new()
1863 arena_dirty_decay_ms_default_get(), &arena->stats.decay_dirty)) { in arena_new()
1866 if (arena_decay_init(&arena->decay_muzzy, in arena_new()
1867 arena_muzzy_decay_ms_default_get(), &arena->stats.decay_muzzy)) { in arena_new()
1871 arena->extent_grow_next = sz_psz2ind(HUGEPAGE); in arena_new()
1872 arena->retain_grow_limit = EXTENT_GROW_MAX_PIND; in arena_new()
1873 if (malloc_mutex_init(&arena->extent_grow_mtx, "extent_grow", in arena_new()
1878 extent_avail_new(&arena->extent_avail); in arena_new()
1879 if (malloc_mutex_init(&arena->extent_avail_mtx, "extent_avail", in arena_new()
1886 bool err = bin_init(&arena->bins[i]); in arena_new()
1892 arena->base = base; in arena_new()
1894 arena_set(ind, arena); in arena_new()
1896 nstime_init(&arena->create_time, 0); in arena_new()
1897 nstime_update(&arena->create_time); in arena_new()
1906 pre_reentrancy(tsdn_tsd(tsdn), arena); in arena_new()
1913 return arena; in arena_new()
1938 arena_prefork0(tsdn_t *tsdn, arena_t *arena) { in arena_prefork0() argument
1939 malloc_mutex_prefork(tsdn, &arena->decay_dirty.mtx); in arena_prefork0()
1940 malloc_mutex_prefork(tsdn, &arena->decay_muzzy.mtx); in arena_prefork0()
1944 arena_prefork1(tsdn_t *tsdn, arena_t *arena) { in arena_prefork1() argument
1946 malloc_mutex_prefork(tsdn, &arena->tcache_ql_mtx); in arena_prefork1()
1951 arena_prefork2(tsdn_t *tsdn, arena_t *arena) { in arena_prefork2() argument
1952 malloc_mutex_prefork(tsdn, &arena->extent_grow_mtx); in arena_prefork2()
1956 arena_prefork3(tsdn_t *tsdn, arena_t *arena) { in arena_prefork3() argument
1957 extents_prefork(tsdn, &arena->extents_dirty); in arena_prefork3()
1958 extents_prefork(tsdn, &arena->extents_muzzy); in arena_prefork3()
1959 extents_prefork(tsdn, &arena->extents_retained); in arena_prefork3()
1963 arena_prefork4(tsdn_t *tsdn, arena_t *arena) { in arena_prefork4() argument
1964 malloc_mutex_prefork(tsdn, &arena->extent_avail_mtx); in arena_prefork4()
1968 arena_prefork5(tsdn_t *tsdn, arena_t *arena) { in arena_prefork5() argument
1969 base_prefork(tsdn, arena->base); in arena_prefork5()
1973 arena_prefork6(tsdn_t *tsdn, arena_t *arena) { in arena_prefork6() argument
1974 malloc_mutex_prefork(tsdn, &arena->large_mtx); in arena_prefork6()
1978 arena_prefork7(tsdn_t *tsdn, arena_t *arena) { in arena_prefork7() argument
1980 bin_prefork(tsdn, &arena->bins[i]); in arena_prefork7()
1985 arena_postfork_parent(tsdn_t *tsdn, arena_t *arena) { in arena_postfork_parent() argument
1989 bin_postfork_parent(tsdn, &arena->bins[i]); in arena_postfork_parent()
1991 malloc_mutex_postfork_parent(tsdn, &arena->large_mtx); in arena_postfork_parent()
1992 base_postfork_parent(tsdn, arena->base); in arena_postfork_parent()
1993 malloc_mutex_postfork_parent(tsdn, &arena->extent_avail_mtx); in arena_postfork_parent()
1994 extents_postfork_parent(tsdn, &arena->extents_dirty); in arena_postfork_parent()
1995 extents_postfork_parent(tsdn, &arena->extents_muzzy); in arena_postfork_parent()
1996 extents_postfork_parent(tsdn, &arena->extents_retained); in arena_postfork_parent()
1997 malloc_mutex_postfork_parent(tsdn, &arena->extent_grow_mtx); in arena_postfork_parent()
1998 malloc_mutex_postfork_parent(tsdn, &arena->decay_dirty.mtx); in arena_postfork_parent()
1999 malloc_mutex_postfork_parent(tsdn, &arena->decay_muzzy.mtx); in arena_postfork_parent()
2001 malloc_mutex_postfork_parent(tsdn, &arena->tcache_ql_mtx); in arena_postfork_parent()
2006 arena_postfork_child(tsdn_t *tsdn, arena_t *arena) { in arena_postfork_child() argument
2009 atomic_store_u(&arena->nthreads[0], 0, ATOMIC_RELAXED); in arena_postfork_child()
2010 atomic_store_u(&arena->nthreads[1], 0, ATOMIC_RELAXED); in arena_postfork_child()
2011 if (tsd_arena_get(tsdn_tsd(tsdn)) == arena) { in arena_postfork_child()
2012 arena_nthreads_inc(arena, false); in arena_postfork_child()
2014 if (tsd_iarena_get(tsdn_tsd(tsdn)) == arena) { in arena_postfork_child()
2015 arena_nthreads_inc(arena, true); in arena_postfork_child()
2018 ql_new(&arena->tcache_ql); in arena_postfork_child()
2019 ql_new(&arena->cache_bin_array_descriptor_ql); in arena_postfork_child()
2021 if (tcache != NULL && tcache->arena == arena) { in arena_postfork_child()
2023 ql_tail_insert(&arena->tcache_ql, tcache, link); in arena_postfork_child()
2027 ql_tail_insert(&arena->cache_bin_array_descriptor_ql, in arena_postfork_child()
2033 bin_postfork_child(tsdn, &arena->bins[i]); in arena_postfork_child()
2035 malloc_mutex_postfork_child(tsdn, &arena->large_mtx); in arena_postfork_child()
2036 base_postfork_child(tsdn, arena->base); in arena_postfork_child()
2037 malloc_mutex_postfork_child(tsdn, &arena->extent_avail_mtx); in arena_postfork_child()
2038 extents_postfork_child(tsdn, &arena->extents_dirty); in arena_postfork_child()
2039 extents_postfork_child(tsdn, &arena->extents_muzzy); in arena_postfork_child()
2040 extents_postfork_child(tsdn, &arena->extents_retained); in arena_postfork_child()
2041 malloc_mutex_postfork_child(tsdn, &arena->extent_grow_mtx); in arena_postfork_child()
2042 malloc_mutex_postfork_child(tsdn, &arena->decay_dirty.mtx); in arena_postfork_child()
2043 malloc_mutex_postfork_child(tsdn, &arena->decay_muzzy.mtx); in arena_postfork_child()
2045 malloc_mutex_postfork_child(tsdn, &arena->tcache_ql_mtx); in arena_postfork_child()