Lines Matching refs:chunk
34 arena_chunk_t *chunk);
40 arena_chunk_t *chunk, arena_run_t *run, arena_bin_t *bin);
49 arena_chunk_t *chunk; in arena_miscelm_size_get() local
52 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(miscelm); in arena_miscelm_size_get()
54 mapbits = arena_mapbits_get(chunk, pageind); in arena_miscelm_size_get()
61 arena_chunk_t *chunk; in arena_miscelm_extent_get() local
63 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(miscelm); in arena_miscelm_extent_get()
64 return (&chunk->node); in arena_miscelm_extent_get()
186 arena_avail_insert(arena_t *arena, arena_chunk_t *chunk, size_t pageind, in arena_avail_insert() argument
190 arena_miscelm_get_const(chunk, pageind)))); in arena_avail_insert()
191 assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >> in arena_avail_insert()
196 arena_miscelm_get_mutable(chunk, pageind)); in arena_avail_insert()
200 arena_avail_remove(arena_t *arena, arena_chunk_t *chunk, size_t pageind, in arena_avail_remove() argument
204 arena_miscelm_get_const(chunk, pageind)))); in arena_avail_remove()
205 assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >> in arena_avail_remove()
210 arena_miscelm_get_mutable(chunk, pageind)); in arena_avail_remove()
214 arena_run_dirty_insert(arena_t *arena, arena_chunk_t *chunk, size_t pageind, in arena_run_dirty_insert() argument
217 arena_chunk_map_misc_t *miscelm = arena_miscelm_get_mutable(chunk, in arena_run_dirty_insert()
220 assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >> in arena_run_dirty_insert()
222 assert(arena_mapbits_dirty_get(chunk, pageind) == CHUNK_MAP_DIRTY); in arena_run_dirty_insert()
223 assert(arena_mapbits_dirty_get(chunk, pageind+npages-1) == in arena_run_dirty_insert()
232 arena_run_dirty_remove(arena_t *arena, arena_chunk_t *chunk, size_t pageind, in arena_run_dirty_remove() argument
235 arena_chunk_map_misc_t *miscelm = arena_miscelm_get_mutable(chunk, in arena_run_dirty_remove()
238 assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >> in arena_run_dirty_remove()
240 assert(arena_mapbits_dirty_get(chunk, pageind) == CHUNK_MAP_DIRTY); in arena_run_dirty_remove()
241 assert(arena_mapbits_dirty_get(chunk, pageind+npages-1) == in arena_run_dirty_remove()
302 arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); in arena_run_reg_dalloc() local
303 size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; in arena_run_reg_dalloc()
304 size_t mapbits = arena_mapbits_get(chunk, pageind); in arena_run_reg_dalloc()
326 arena_run_zero(arena_chunk_t *chunk, size_t run_ind, size_t npages) in arena_run_zero() argument
329 JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED((void *)((uintptr_t)chunk + in arena_run_zero()
331 memset((void *)((uintptr_t)chunk + (run_ind << LG_PAGE)), 0, in arena_run_zero()
336 arena_run_page_mark_zeroed(arena_chunk_t *chunk, size_t run_ind) in arena_run_page_mark_zeroed() argument
339 JEMALLOC_VALGRIND_MAKE_MEM_DEFINED((void *)((uintptr_t)chunk + (run_ind in arena_run_page_mark_zeroed()
344 arena_run_page_validate_zeroed(arena_chunk_t *chunk, size_t run_ind) in arena_run_page_validate_zeroed() argument
347 UNUSED size_t *p = (size_t *)((uintptr_t)chunk + (run_ind << LG_PAGE)); in arena_run_page_validate_zeroed()
349 arena_run_page_mark_zeroed(chunk, run_ind); in arena_run_page_validate_zeroed()
382 arena_run_split_remove(arena_t *arena, arena_chunk_t *chunk, size_t run_ind, in arena_run_split_remove() argument
389 total_pages = arena_mapbits_unallocated_size_get(chunk, run_ind) >> in arena_run_split_remove()
391 assert(arena_mapbits_dirty_get(chunk, run_ind+total_pages-1) == in arena_run_split_remove()
396 arena_avail_remove(arena, chunk, run_ind, total_pages); in arena_run_split_remove()
398 arena_run_dirty_remove(arena, chunk, run_ind, total_pages); in arena_run_split_remove()
407 arena_mapbits_unallocated_set(chunk, run_ind+need_pages, in arena_run_split_remove()
409 (arena_mapbits_unzeroed_get(chunk, run_ind+need_pages) & in arena_run_split_remove()
411 arena_mapbits_unallocated_set(chunk, run_ind+total_pages-1, in arena_run_split_remove()
413 (arena_mapbits_unzeroed_get(chunk, run_ind+total_pages-1) & in arena_run_split_remove()
416 arena_run_dirty_insert(arena, chunk, run_ind+need_pages, in arena_run_split_remove()
419 arena_avail_insert(arena, chunk, run_ind+need_pages, rem_pages); in arena_run_split_remove()
427 arena_chunk_t *chunk; in arena_run_split_large_helper() local
432 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); in arena_run_split_large_helper()
435 flag_dirty = arena_mapbits_dirty_get(chunk, run_ind); in arena_run_split_large_helper()
436 flag_decommitted = arena_mapbits_decommitted_get(chunk, run_ind); in arena_run_split_large_helper()
440 if (flag_decommitted != 0 && arena->chunk_hooks.commit(chunk, chunksize, in arena_run_split_large_helper()
445 arena_run_split_remove(arena, chunk, run_ind, flag_dirty, in arena_run_split_large_helper()
453 *)((uintptr_t)chunk + (run_ind << LG_PAGE)), in arena_run_split_large_helper()
457 arena_run_zero(chunk, run_ind, need_pages); in arena_run_split_large_helper()
465 if (arena_mapbits_unzeroed_get(chunk, run_ind+i) in arena_run_split_large_helper()
467 arena_run_zero(chunk, run_ind+i, 1); in arena_run_split_large_helper()
469 arena_run_page_validate_zeroed(chunk, in arena_run_split_large_helper()
472 arena_run_page_mark_zeroed(chunk, in arena_run_split_large_helper()
478 JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED((void *)((uintptr_t)chunk + in arena_run_split_large_helper()
488 arena_mapbits_large_set(chunk, run_ind+need_pages-1, 0, flag_dirty | in arena_run_split_large_helper()
489 (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk, in arena_run_split_large_helper()
491 arena_mapbits_large_set(chunk, run_ind, size, flag_dirty | in arena_run_split_large_helper()
492 (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk, run_ind))); in arena_run_split_large_helper()
514 arena_chunk_t *chunk; in arena_run_split_small() local
520 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); in arena_run_split_small()
523 flag_dirty = arena_mapbits_dirty_get(chunk, run_ind); in arena_run_split_small()
524 flag_decommitted = arena_mapbits_decommitted_get(chunk, run_ind); in arena_run_split_small()
528 if (flag_decommitted != 0 && arena->chunk_hooks.commit(chunk, chunksize, in arena_run_split_small()
532 arena_run_split_remove(arena, chunk, run_ind, flag_dirty, in arena_run_split_small()
536 size_t flag_unzeroed = arena_mapbits_unzeroed_get(chunk, in arena_run_split_small()
538 arena_mapbits_small_set(chunk, run_ind+i, i, binind, in arena_run_split_small()
541 arena_run_page_validate_zeroed(chunk, run_ind+i); in arena_run_split_small()
543 JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED((void *)((uintptr_t)chunk + in arena_run_split_small()
551 arena_chunk_t *chunk; in arena_chunk_init_spare() local
555 chunk = arena->spare; in arena_chunk_init_spare()
558 assert(arena_mapbits_allocated_get(chunk, map_bias) == 0); in arena_chunk_init_spare()
559 assert(arena_mapbits_allocated_get(chunk, chunk_npages-1) == 0); in arena_chunk_init_spare()
560 assert(arena_mapbits_unallocated_size_get(chunk, map_bias) == in arena_chunk_init_spare()
562 assert(arena_mapbits_unallocated_size_get(chunk, chunk_npages-1) == in arena_chunk_init_spare()
564 assert(arena_mapbits_dirty_get(chunk, map_bias) == in arena_chunk_init_spare()
565 arena_mapbits_dirty_get(chunk, chunk_npages-1)); in arena_chunk_init_spare()
567 return (chunk); in arena_chunk_init_spare()
571 arena_chunk_register(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk, in arena_chunk_register() argument
581 extent_node_init(&chunk->node, arena, chunk, chunksize, sn, zero, true); in arena_chunk_register()
582 extent_node_achunk_set(&chunk->node, true); in arena_chunk_register()
583 return (chunk_register(tsdn, chunk, &chunk->node)); in arena_chunk_register()
590 arena_chunk_t *chunk; in arena_chunk_alloc_internal_hard() local
595 chunk = (arena_chunk_t *)chunk_alloc_wrapper(tsdn, arena, chunk_hooks, in arena_chunk_alloc_internal_hard()
597 if (chunk != NULL && !*commit) { in arena_chunk_alloc_internal_hard()
599 if (chunk_hooks->commit(chunk, chunksize, 0, map_bias << in arena_chunk_alloc_internal_hard()
602 (void *)chunk, chunksize, sn, *zero, *commit); in arena_chunk_alloc_internal_hard()
603 chunk = NULL; in arena_chunk_alloc_internal_hard()
606 if (chunk != NULL && arena_chunk_register(tsdn, arena, chunk, sn, in arena_chunk_alloc_internal_hard()
610 chunk_hooks->decommit(chunk, chunksize, 0, map_bias << in arena_chunk_alloc_internal_hard()
613 chunk_dalloc_wrapper(tsdn, arena, chunk_hooks, (void *)chunk, in arena_chunk_alloc_internal_hard()
615 chunk = NULL; in arena_chunk_alloc_internal_hard()
619 return (chunk); in arena_chunk_alloc_internal_hard()
626 arena_chunk_t *chunk; in arena_chunk_alloc_internal() local
630 chunk = chunk_alloc_cache(tsdn, arena, &chunk_hooks, NULL, chunksize, in arena_chunk_alloc_internal()
632 if (chunk != NULL) { in arena_chunk_alloc_internal()
633 if (arena_chunk_register(tsdn, arena, chunk, sn, *zero)) { in arena_chunk_alloc_internal()
634 chunk_dalloc_cache(tsdn, arena, &chunk_hooks, chunk, in arena_chunk_alloc_internal()
639 if (chunk == NULL) { in arena_chunk_alloc_internal()
640 chunk = arena_chunk_alloc_internal_hard(tsdn, arena, in arena_chunk_alloc_internal()
644 if (config_stats && chunk != NULL) { in arena_chunk_alloc_internal()
649 return (chunk); in arena_chunk_alloc_internal()
655 arena_chunk_t *chunk; in arena_chunk_init_hard() local
663 chunk = arena_chunk_alloc_internal(tsdn, arena, &zero, &commit); in arena_chunk_init_hard()
664 if (chunk == NULL) in arena_chunk_init_hard()
667 chunk->hugepage = true; in arena_chunk_init_hard()
676 arena_mapbits_unallocated_set(chunk, map_bias, arena_maxrun, in arena_chunk_init_hard()
684 (void *)arena_bitselm_get_const(chunk, map_bias+1), in arena_chunk_init_hard()
685 (size_t)((uintptr_t)arena_bitselm_get_const(chunk, in arena_chunk_init_hard()
687 (uintptr_t)arena_bitselm_get_const(chunk, map_bias+1))); in arena_chunk_init_hard()
689 arena_mapbits_internal_set(chunk, i, flag_unzeroed); in arena_chunk_init_hard()
692 *)arena_bitselm_get_const(chunk, map_bias+1), in arena_chunk_init_hard()
693 (size_t)((uintptr_t)arena_bitselm_get_const(chunk, in arena_chunk_init_hard()
695 (uintptr_t)arena_bitselm_get_const(chunk, map_bias+1))); in arena_chunk_init_hard()
698 assert(arena_mapbits_unzeroed_get(chunk, i) == in arena_chunk_init_hard()
703 arena_mapbits_unallocated_set(chunk, chunk_npages-1, arena_maxrun, in arena_chunk_init_hard()
706 return (chunk); in arena_chunk_init_hard()
712 arena_chunk_t *chunk; in arena_chunk_alloc() local
715 chunk = arena_chunk_init_spare(arena); in arena_chunk_alloc()
717 chunk = arena_chunk_init_hard(tsdn, arena); in arena_chunk_alloc()
718 if (chunk == NULL) in arena_chunk_alloc()
722 ql_elm_new(&chunk->node, ql_link); in arena_chunk_alloc()
723 ql_tail_insert(&arena->achunks, &chunk->node, ql_link); in arena_chunk_alloc()
724 arena_avail_insert(arena, chunk, map_bias, chunk_npages-map_bias); in arena_chunk_alloc()
726 return (chunk); in arena_chunk_alloc()
730 arena_chunk_discard(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk) in arena_chunk_discard() argument
736 chunk_deregister(chunk, &chunk->node); in arena_chunk_discard()
738 sn = extent_node_sn_get(&chunk->node); in arena_chunk_discard()
739 hugepage = chunk->hugepage; in arena_chunk_discard()
740 committed = (arena_mapbits_decommitted_get(chunk, map_bias) == 0); in arena_chunk_discard()
749 chunk_hooks.decommit(chunk, chunksize, 0, map_bias << LG_PAGE, in arena_chunk_discard()
758 pages_huge(chunk, chunksize); in arena_chunk_discard()
761 chunk_dalloc_cache(tsdn, arena, &chunk_hooks, (void *)chunk, chunksize, in arena_chunk_discard()
785 arena_chunk_dalloc(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk) in arena_chunk_dalloc() argument
789 assert(arena_mapbits_allocated_get(chunk, map_bias) == 0); in arena_chunk_dalloc()
790 assert(arena_mapbits_allocated_get(chunk, chunk_npages-1) == 0); in arena_chunk_dalloc()
791 assert(arena_mapbits_unallocated_size_get(chunk, map_bias) == in arena_chunk_dalloc()
793 assert(arena_mapbits_unallocated_size_get(chunk, chunk_npages-1) == in arena_chunk_dalloc()
795 assert(arena_mapbits_dirty_get(chunk, map_bias) == in arena_chunk_dalloc()
796 arena_mapbits_dirty_get(chunk, chunk_npages-1)); in arena_chunk_dalloc()
797 assert(arena_mapbits_decommitted_get(chunk, map_bias) == in arena_chunk_dalloc()
798 arena_mapbits_decommitted_get(chunk, chunk_npages-1)); in arena_chunk_dalloc()
801 arena_avail_remove(arena, chunk, map_bias, chunk_npages-map_bias); in arena_chunk_dalloc()
803 ql_remove(&arena->achunks, &chunk->node, ql_link); in arena_chunk_dalloc()
805 arena->spare = chunk; in arena_chunk_dalloc()
970 arena_chunk_dalloc_huge(tsdn_t *tsdn, arena_t *arena, void *chunk, size_t usize, in arena_chunk_dalloc_huge() argument
984 chunk_dalloc_cache(tsdn, arena, &chunk_hooks, chunk, csize, sn, true); in arena_chunk_dalloc_huge()
989 arena_chunk_ralloc_huge_similar(tsdn_t *tsdn, arena_t *arena, void *chunk, in arena_chunk_ralloc_huge_similar() argument
1007 arena_chunk_ralloc_huge_shrink(tsdn_t *tsdn, arena_t *arena, void *chunk, in arena_chunk_ralloc_huge_shrink() argument
1023 void *nchunk = (void *)((uintptr_t)chunk + in arena_chunk_ralloc_huge_shrink()
1034 chunk_hooks_t *chunk_hooks, void *chunk, size_t oldsize, size_t usize, in arena_chunk_ralloc_huge_expand_hard() argument
1052 } else if (chunk_hooks->merge(chunk, CHUNK_CEILING(oldsize), nchunk, in arena_chunk_ralloc_huge_expand_hard()
1062 arena_chunk_ralloc_huge_expand(tsdn_t *tsdn, arena_t *arena, void *chunk, in arena_chunk_ralloc_huge_expand() argument
1067 void *nchunk = (void *)((uintptr_t)chunk + CHUNK_CEILING(oldsize)); in arena_chunk_ralloc_huge_expand()
1087 &chunk_hooks, chunk, oldsize, usize, &sn, zero, nchunk, in arena_chunk_ralloc_huge_expand()
1089 } else if (chunk_hooks.merge(chunk, CHUNK_CEILING(oldsize), nchunk, in arena_chunk_ralloc_huge_expand()
1135 arena_chunk_t *chunk; in arena_run_alloc_large() local
1149 chunk = arena_chunk_alloc(tsdn, arena); in arena_run_alloc_large()
1150 if (chunk != NULL) { in arena_run_alloc_large()
1151 run = &arena_miscelm_get_mutable(chunk, map_bias)->run; in arena_run_alloc_large()
1179 arena_chunk_t *chunk; in arena_run_alloc_small() local
1194 chunk = arena_chunk_alloc(tsdn, arena); in arena_run_alloc_small()
1195 if (chunk != NULL) { in arena_run_alloc_small()
1196 run = &arena_miscelm_get_mutable(chunk, map_bias)->run; in arena_run_alloc_small()
1544 arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE( in arena_dirty_count() local
1549 assert(arena_mapbits_allocated_get(chunk, pageind) == in arena_dirty_count()
1551 assert(arena_mapbits_large_get(chunk, pageind) == 0); in arena_dirty_count()
1552 assert(arena_mapbits_dirty_get(chunk, pageind) != 0); in arena_dirty_count()
1553 npages = arena_mapbits_unallocated_size_get(chunk, in arena_dirty_count()
1582 UNUSED void *chunk; in arena_stash_dirty() local
1596 chunk = chunk_alloc_cache(tsdn, arena, chunk_hooks, in arena_stash_dirty()
1600 assert(chunk == extent_node_addr_get(chunkselm)); in arena_stash_dirty()
1608 arena_chunk_t *chunk = in arena_stash_dirty() local
1615 arena_mapbits_unallocated_size_get(chunk, pageind); in arena_stash_dirty()
1623 assert(arena_mapbits_dirty_get(chunk, pageind) == in arena_stash_dirty()
1624 arena_mapbits_dirty_get(chunk, pageind+npages-1)); in arena_stash_dirty()
1630 if (chunk == arena->spare) in arena_stash_dirty()
1688 arena_chunk_t *chunk = in arena_purge_stashed() local
1693 run_size = arena_mapbits_large_size_get(chunk, pageind); in arena_purge_stashed()
1702 if (chunk->hugepage) { in arena_purge_stashed()
1703 pages_nohuge(chunk, chunksize); in arena_purge_stashed()
1704 chunk->hugepage = false; in arena_purge_stashed()
1708 assert(!arena_mapbits_decommitted_get(chunk, pageind)); in arena_purge_stashed()
1709 assert(!arena_mapbits_decommitted_get(chunk, in arena_purge_stashed()
1711 decommitted = !chunk_hooks->decommit(chunk, chunksize, in arena_purge_stashed()
1718 chunk_hooks, chunk, chunksize, pageind << in arena_purge_stashed()
1722 arena_mapbits_large_set(chunk, pageind+npages-1, 0, in arena_purge_stashed()
1724 arena_mapbits_large_set(chunk, pageind, run_size, in arena_purge_stashed()
1739 arena_mapbits_internal_set(chunk, pageind+i, in arena_purge_stashed()
1785 arena_chunk_t *chunk = in arena_unstash_purged() local
1790 bool decommitted = (arena_mapbits_decommitted_get(chunk, in arena_unstash_purged()
1863 arena_achunk_prof_reset(tsd_t *tsd, arena_t *arena, arena_chunk_t *chunk) in arena_achunk_prof_reset() argument
1875 if (arena_mapbits_allocated_get(chunk, pageind) != 0) { in arena_achunk_prof_reset()
1876 if (arena_mapbits_large_get(chunk, pageind) != 0) { in arena_achunk_prof_reset()
1877 void *ptr = (void *)((uintptr_t)chunk + (pageind in arena_achunk_prof_reset()
1883 npages = arena_mapbits_large_size_get(chunk, in arena_achunk_prof_reset()
1887 size_t binind = arena_mapbits_binind_get(chunk, in arena_achunk_prof_reset()
1895 npages = arena_mapbits_unallocated_size_get(chunk, in arena_achunk_prof_reset()
2007 arena_run_coalesce(arena_t *arena, arena_chunk_t *chunk, size_t *p_size, in arena_run_coalesce() argument
2017 arena_mapbits_allocated_get(chunk, run_ind+run_pages) == 0 && in arena_run_coalesce()
2018 arena_mapbits_dirty_get(chunk, run_ind+run_pages) == flag_dirty && in arena_run_coalesce()
2019 arena_mapbits_decommitted_get(chunk, run_ind+run_pages) == in arena_run_coalesce()
2021 size_t nrun_size = arena_mapbits_unallocated_size_get(chunk, in arena_run_coalesce()
2029 assert(arena_mapbits_unallocated_size_get(chunk, in arena_run_coalesce()
2031 assert(arena_mapbits_dirty_get(chunk, in arena_run_coalesce()
2033 assert(arena_mapbits_decommitted_get(chunk, in arena_run_coalesce()
2035 arena_avail_remove(arena, chunk, run_ind+run_pages, nrun_pages); in arena_run_coalesce()
2042 arena_run_dirty_remove(arena, chunk, run_ind+run_pages, in arena_run_coalesce()
2049 arena_mapbits_unallocated_size_set(chunk, run_ind, size); in arena_run_coalesce()
2050 arena_mapbits_unallocated_size_set(chunk, run_ind+run_pages-1, in arena_run_coalesce()
2055 if (run_ind > map_bias && arena_mapbits_allocated_get(chunk, in arena_run_coalesce()
2056 run_ind-1) == 0 && arena_mapbits_dirty_get(chunk, run_ind-1) == in arena_run_coalesce()
2057 flag_dirty && arena_mapbits_decommitted_get(chunk, run_ind-1) == in arena_run_coalesce()
2059 size_t prun_size = arena_mapbits_unallocated_size_get(chunk, in arena_run_coalesce()
2069 assert(arena_mapbits_unallocated_size_get(chunk, run_ind) == in arena_run_coalesce()
2071 assert(arena_mapbits_dirty_get(chunk, run_ind) == flag_dirty); in arena_run_coalesce()
2072 assert(arena_mapbits_decommitted_get(chunk, run_ind) == in arena_run_coalesce()
2074 arena_avail_remove(arena, chunk, run_ind, prun_pages); in arena_run_coalesce()
2081 arena_run_dirty_remove(arena, chunk, run_ind, in arena_run_coalesce()
2088 arena_mapbits_unallocated_size_set(chunk, run_ind, size); in arena_run_coalesce()
2089 arena_mapbits_unallocated_size_set(chunk, run_ind+run_pages-1, in arena_run_coalesce()
2099 arena_run_size_get(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, in arena_run_size_get() argument
2107 if (arena_mapbits_large_get(chunk, run_ind) != 0) { in arena_run_size_get()
2108 size = arena_mapbits_large_size_get(chunk, run_ind); in arena_run_size_get()
2109 assert(size == PAGE || arena_mapbits_large_size_get(chunk, in arena_run_size_get()
2123 arena_chunk_t *chunk; in arena_run_dalloc() local
2127 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); in arena_run_dalloc()
2132 size = arena_run_size_get(arena, chunk, run, run_ind); in arena_run_dalloc()
2141 assert(arena_mapbits_dirty_get(chunk, run_ind) == in arena_run_dalloc()
2142 arena_mapbits_dirty_get(chunk, run_ind+run_pages-1)); in arena_run_dalloc()
2143 if (!cleaned && !decommitted && arena_mapbits_dirty_get(chunk, run_ind) in arena_run_dalloc()
2152 arena_mapbits_unallocated_set(chunk, run_ind, size, flags); in arena_run_dalloc()
2153 arena_mapbits_unallocated_set(chunk, run_ind+run_pages-1, size, in arena_run_dalloc()
2156 arena_mapbits_unallocated_set(chunk, run_ind, size, in arena_run_dalloc()
2157 arena_mapbits_unzeroed_get(chunk, run_ind)); in arena_run_dalloc()
2158 arena_mapbits_unallocated_set(chunk, run_ind+run_pages-1, size, in arena_run_dalloc()
2159 arena_mapbits_unzeroed_get(chunk, run_ind+run_pages-1)); in arena_run_dalloc()
2162 arena_run_coalesce(arena, chunk, &size, &run_ind, &run_pages, in arena_run_dalloc()
2166 assert(arena_mapbits_unallocated_size_get(chunk, run_ind) == in arena_run_dalloc()
2167 arena_mapbits_unallocated_size_get(chunk, run_ind+run_pages-1)); in arena_run_dalloc()
2168 assert(arena_mapbits_dirty_get(chunk, run_ind) == in arena_run_dalloc()
2169 arena_mapbits_dirty_get(chunk, run_ind+run_pages-1)); in arena_run_dalloc()
2170 assert(arena_mapbits_decommitted_get(chunk, run_ind) == in arena_run_dalloc()
2171 arena_mapbits_decommitted_get(chunk, run_ind+run_pages-1)); in arena_run_dalloc()
2172 arena_avail_insert(arena, chunk, run_ind, run_pages); in arena_run_dalloc()
2175 arena_run_dirty_insert(arena, chunk, run_ind, run_pages); in arena_run_dalloc()
2181 arena_chunk_dalloc(tsdn, arena, chunk); in arena_run_dalloc()
2196 arena_run_trim_head(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk, in arena_run_trim_head() argument
2202 size_t flag_dirty = arena_mapbits_dirty_get(chunk, pageind); in arena_run_trim_head()
2203 size_t flag_decommitted = arena_mapbits_decommitted_get(chunk, pageind); in arena_run_trim_head()
2214 assert(arena_mapbits_large_size_get(chunk, pageind) == oldsize); in arena_run_trim_head()
2215 arena_mapbits_large_set(chunk, pageind+head_npages-1, 0, flag_dirty | in arena_run_trim_head()
2216 (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk, in arena_run_trim_head()
2218 arena_mapbits_large_set(chunk, pageind, oldsize-newsize, flag_dirty | in arena_run_trim_head()
2219 (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk, pageind))); in arena_run_trim_head()
2223 assert(arena_mapbits_large_size_get(chunk, in arena_run_trim_head()
2225 assert(arena_mapbits_dirty_get(chunk, in arena_run_trim_head()
2228 arena_mapbits_large_set(chunk, pageind+head_npages, newsize, in arena_run_trim_head()
2229 flag_dirty | (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk, in arena_run_trim_head()
2237 arena_run_trim_tail(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk, in arena_run_trim_tail() argument
2243 size_t flag_dirty = arena_mapbits_dirty_get(chunk, pageind); in arena_run_trim_tail()
2244 size_t flag_decommitted = arena_mapbits_decommitted_get(chunk, pageind); in arena_run_trim_tail()
2257 assert(arena_mapbits_large_size_get(chunk, pageind) == oldsize); in arena_run_trim_tail()
2258 arena_mapbits_large_set(chunk, pageind+head_npages-1, 0, flag_dirty | in arena_run_trim_tail()
2259 (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk, in arena_run_trim_tail()
2261 arena_mapbits_large_set(chunk, pageind, newsize, flag_dirty | in arena_run_trim_tail()
2262 (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk, pageind))); in arena_run_trim_tail()
2266 assert(arena_mapbits_large_size_get(chunk, in arena_run_trim_tail()
2268 assert(arena_mapbits_dirty_get(chunk, in arena_run_trim_tail()
2271 arena_mapbits_large_set(chunk, pageind+head_npages, oldsize-newsize, in arena_run_trim_tail()
2272 flag_dirty | (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk, in arena_run_trim_tail()
2275 tail_miscelm = arena_miscelm_get_mutable(chunk, pageind + head_npages); in arena_run_trim_tail()
2374 arena_chunk_t *chunk; in arena_bin_malloc_hard() local
2384 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); in arena_bin_malloc_hard()
2386 arena_dalloc_bin_run(tsdn, arena, chunk, run, in arena_bin_malloc_hard()
2708 arena_chunk_t *chunk; in arena_palloc_large() local
2729 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); in arena_palloc_large()
2741 miscelm = arena_miscelm_get_mutable(chunk, in arena_palloc_large()
2746 arena_run_trim_head(tsdn, arena, chunk, head_run, alloc_size, in arena_palloc_large()
2750 arena_run_trim_tail(tsdn, arena, chunk, run, usize + large_pad + in arena_palloc_large()
2756 bool dirty = (arena_mapbits_dirty_get(chunk, run_ind) != 0); in arena_palloc_large()
2757 bool decommitted = (arena_mapbits_decommitted_get(chunk, in arena_palloc_large()
2827 arena_chunk_t *chunk; in arena_prof_promoted() local
2838 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); in arena_prof_promoted()
2839 pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; in arena_prof_promoted()
2842 arena_mapbits_large_binind_set(chunk, pageind, binind); in arena_prof_promoted()
2849 arena_dissociate_bin_run(arena_chunk_t *chunk, arena_run_t *run, in arena_dissociate_bin_run() argument
2858 &chunk->node), bin); in arena_dissociate_bin_run()
2876 arena_dalloc_bin_run(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk, in arena_dalloc_bin_run() argument
2917 arena_dalloc_bin_locked_impl(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk, in arena_dalloc_bin_locked_impl() argument
2926 pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; in arena_dalloc_bin_locked_impl()
2927 rpages_ind = pageind - arena_mapbits_small_runind_get(chunk, pageind); in arena_dalloc_bin_locked_impl()
2928 run = &arena_miscelm_get_mutable(chunk, rpages_ind)->run; in arena_dalloc_bin_locked_impl()
2938 arena_dissociate_bin_run(chunk, run, bin); in arena_dalloc_bin_locked_impl()
2939 arena_dalloc_bin_run(tsdn, arena, chunk, run, bin); in arena_dalloc_bin_locked_impl()
2951 arena_chunk_t *chunk, void *ptr, arena_chunk_map_bits_t *bitselm) in arena_dalloc_bin_junked_locked() argument
2954 arena_dalloc_bin_locked_impl(tsdn, arena, chunk, ptr, bitselm, true); in arena_dalloc_bin_junked_locked()
2958 arena_dalloc_bin(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk, void *ptr, in arena_dalloc_bin() argument
2965 rpages_ind = pageind - arena_mapbits_small_runind_get(chunk, pageind); in arena_dalloc_bin()
2966 run = &arena_miscelm_get_mutable(chunk, rpages_ind)->run; in arena_dalloc_bin()
2969 arena_dalloc_bin_locked_impl(tsdn, arena, chunk, ptr, bitselm, false); in arena_dalloc_bin()
2974 arena_dalloc_small(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk, in arena_dalloc_small() argument
2981 assert(arena_ptr_small_binind_get(ptr, arena_mapbits_get(chunk, in arena_dalloc_small()
2984 bitselm = arena_bitselm_get_mutable(chunk, pageind); in arena_dalloc_small()
2985 arena_dalloc_bin(tsdn, arena, chunk, ptr, pageind, bitselm); in arena_dalloc_small()
3009 arena_chunk_t *chunk, void *ptr, bool junked) in arena_dalloc_large_locked_impl() argument
3011 size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; in arena_dalloc_large_locked_impl()
3012 arena_chunk_map_misc_t *miscelm = arena_miscelm_get_mutable(chunk, in arena_dalloc_large_locked_impl()
3017 size_t usize = arena_mapbits_large_size_get(chunk, pageind) - in arena_dalloc_large_locked_impl()
3037 arena_chunk_t *chunk, void *ptr) in arena_dalloc_large_junked_locked() argument
3040 arena_dalloc_large_locked_impl(tsdn, arena, chunk, ptr, true); in arena_dalloc_large_junked_locked()
3044 arena_dalloc_large(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk, in arena_dalloc_large() argument
3049 arena_dalloc_large_locked_impl(tsdn, arena, chunk, ptr, false); in arena_dalloc_large()
3055 arena_ralloc_large_shrink(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk, in arena_ralloc_large_shrink() argument
3058 size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; in arena_ralloc_large_shrink()
3059 arena_chunk_map_misc_t *miscelm = arena_miscelm_get_mutable(chunk, in arena_ralloc_large_shrink()
3070 arena_run_trim_tail(tsdn, arena, chunk, run, oldsize + large_pad, size + in arena_ralloc_large_shrink()
3092 arena_ralloc_large_grow(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk, in arena_ralloc_large_grow() argument
3095 size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; in arena_ralloc_large_grow()
3099 assert(oldsize == arena_mapbits_large_size_get(chunk, pageind) - in arena_ralloc_large_grow()
3104 if (pageind+npages >= chunk_npages || arena_mapbits_allocated_get(chunk, in arena_ralloc_large_grow()
3107 followsize = arena_mapbits_unallocated_size_get(chunk, pageind+npages); in arena_ralloc_large_grow()
3126 run = &arena_miscelm_get_mutable(chunk, pageind+npages)->run; in arena_ralloc_large_grow()
3157 flag_dirty = arena_mapbits_dirty_get(chunk, pageind) | in arena_ralloc_large_grow()
3158 arena_mapbits_dirty_get(chunk, pageind+npages-1); in arena_ralloc_large_grow()
3160 arena_mapbits_large_set(chunk, pageind, size + large_pad, in arena_ralloc_large_grow()
3162 arena_mapbits_unzeroed_get(chunk, pageind))); in arena_ralloc_large_grow()
3163 arena_mapbits_large_set(chunk, pageind+npages-1, 0, flag_dirty | in arena_ralloc_large_grow()
3164 (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk, in arena_ralloc_large_grow()
3219 arena_chunk_t *chunk; in arena_ralloc_large() local
3227 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); in arena_ralloc_large()
3228 arena = extent_node_arena_get(&chunk->node); in arena_ralloc_large()
3231 bool ret = arena_ralloc_large_grow(tsdn, arena, chunk, ptr, in arena_ralloc_large()
3249 arena_ralloc_large_shrink(tsdn, arena, chunk, ptr, oldsize, usize_max); in arena_ralloc_large()
3268 arena_chunk_t *chunk; in arena_ralloc_no_move() local
3289 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); in arena_ralloc_no_move()
3290 arena_decay_tick(tsdn, extent_node_arena_get(&chunk->node)); in arena_ralloc_no_move()