Lines Matching refs:run_ind
326 arena_run_zero(arena_chunk_t *chunk, size_t run_ind, size_t npages) in arena_run_zero() argument
330 (run_ind << LG_PAGE)), (npages << LG_PAGE)); in arena_run_zero()
331 memset((void *)((uintptr_t)chunk + (run_ind << LG_PAGE)), 0, in arena_run_zero()
336 arena_run_page_mark_zeroed(arena_chunk_t *chunk, size_t run_ind) in arena_run_page_mark_zeroed() argument
339 JEMALLOC_VALGRIND_MAKE_MEM_DEFINED((void *)((uintptr_t)chunk + (run_ind in arena_run_page_mark_zeroed()
344 arena_run_page_validate_zeroed(arena_chunk_t *chunk, size_t run_ind) in arena_run_page_validate_zeroed() argument
347 UNUSED size_t *p = (size_t *)((uintptr_t)chunk + (run_ind << LG_PAGE)); in arena_run_page_validate_zeroed()
349 arena_run_page_mark_zeroed(chunk, run_ind); in arena_run_page_validate_zeroed()
382 arena_run_split_remove(arena_t *arena, arena_chunk_t *chunk, size_t run_ind, in arena_run_split_remove() argument
389 total_pages = arena_mapbits_unallocated_size_get(chunk, run_ind) >> in arena_run_split_remove()
391 assert(arena_mapbits_dirty_get(chunk, run_ind+total_pages-1) == in arena_run_split_remove()
396 arena_avail_remove(arena, chunk, run_ind, total_pages); in arena_run_split_remove()
398 arena_run_dirty_remove(arena, chunk, run_ind, total_pages); in arena_run_split_remove()
407 arena_mapbits_unallocated_set(chunk, run_ind+need_pages, in arena_run_split_remove()
409 (arena_mapbits_unzeroed_get(chunk, run_ind+need_pages) & in arena_run_split_remove()
411 arena_mapbits_unallocated_set(chunk, run_ind+total_pages-1, in arena_run_split_remove()
413 (arena_mapbits_unzeroed_get(chunk, run_ind+total_pages-1) & in arena_run_split_remove()
416 arena_run_dirty_insert(arena, chunk, run_ind+need_pages, in arena_run_split_remove()
419 arena_avail_insert(arena, chunk, run_ind+need_pages, rem_pages); in arena_run_split_remove()
429 size_t flag_dirty, flag_decommitted, run_ind, need_pages; in arena_run_split_large_helper() local
434 run_ind = arena_miscelm_to_pageind(miscelm); in arena_run_split_large_helper()
435 flag_dirty = arena_mapbits_dirty_get(chunk, run_ind); in arena_run_split_large_helper()
436 flag_decommitted = arena_mapbits_decommitted_get(chunk, run_ind); in arena_run_split_large_helper()
441 run_ind << LG_PAGE, size, arena->ind)) in arena_run_split_large_helper()
445 arena_run_split_remove(arena, chunk, run_ind, flag_dirty, in arena_run_split_large_helper()
453 *)((uintptr_t)chunk + (run_ind << LG_PAGE)), in arena_run_split_large_helper()
457 arena_run_zero(chunk, run_ind, need_pages); in arena_run_split_large_helper()
465 if (arena_mapbits_unzeroed_get(chunk, run_ind+i) in arena_run_split_large_helper()
467 arena_run_zero(chunk, run_ind+i, 1); in arena_run_split_large_helper()
470 run_ind+i); in arena_run_split_large_helper()
473 run_ind+i); in arena_run_split_large_helper()
479 (run_ind << LG_PAGE)), (need_pages << LG_PAGE)); in arena_run_split_large_helper()
488 arena_mapbits_large_set(chunk, run_ind+need_pages-1, 0, flag_dirty | in arena_run_split_large_helper()
490 run_ind+need_pages-1))); in arena_run_split_large_helper()
491 arena_mapbits_large_set(chunk, run_ind, size, flag_dirty | in arena_run_split_large_helper()
492 (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk, run_ind))); in arena_run_split_large_helper()
516 size_t flag_dirty, flag_decommitted, run_ind, need_pages, i; in arena_run_split_small() local
522 run_ind = arena_miscelm_to_pageind(miscelm); in arena_run_split_small()
523 flag_dirty = arena_mapbits_dirty_get(chunk, run_ind); in arena_run_split_small()
524 flag_decommitted = arena_mapbits_decommitted_get(chunk, run_ind); in arena_run_split_small()
529 run_ind << LG_PAGE, size, arena->ind)) in arena_run_split_small()
532 arena_run_split_remove(arena, chunk, run_ind, flag_dirty, in arena_run_split_small()
537 run_ind+i); in arena_run_split_small()
538 arena_mapbits_small_set(chunk, run_ind+i, i, binind, in arena_run_split_small()
541 arena_run_page_validate_zeroed(chunk, run_ind+i); in arena_run_split_small()
544 (run_ind << LG_PAGE)), (need_pages << LG_PAGE)); in arena_run_split_small()
2012 size_t run_ind = *p_run_ind; in arena_run_coalesce() local
2016 if (run_ind + run_pages < chunk_npages && in arena_run_coalesce()
2017 arena_mapbits_allocated_get(chunk, run_ind+run_pages) == 0 && in arena_run_coalesce()
2018 arena_mapbits_dirty_get(chunk, run_ind+run_pages) == flag_dirty && in arena_run_coalesce()
2019 arena_mapbits_decommitted_get(chunk, run_ind+run_pages) == in arena_run_coalesce()
2022 run_ind+run_pages); in arena_run_coalesce()
2030 run_ind+run_pages+nrun_pages-1) == nrun_size); in arena_run_coalesce()
2032 run_ind+run_pages+nrun_pages-1) == flag_dirty); in arena_run_coalesce()
2034 run_ind+run_pages+nrun_pages-1) == flag_decommitted); in arena_run_coalesce()
2035 arena_avail_remove(arena, chunk, run_ind+run_pages, nrun_pages); in arena_run_coalesce()
2042 arena_run_dirty_remove(arena, chunk, run_ind+run_pages, in arena_run_coalesce()
2049 arena_mapbits_unallocated_size_set(chunk, run_ind, size); in arena_run_coalesce()
2050 arena_mapbits_unallocated_size_set(chunk, run_ind+run_pages-1, in arena_run_coalesce()
2055 if (run_ind > map_bias && arena_mapbits_allocated_get(chunk, in arena_run_coalesce()
2056 run_ind-1) == 0 && arena_mapbits_dirty_get(chunk, run_ind-1) == in arena_run_coalesce()
2057 flag_dirty && arena_mapbits_decommitted_get(chunk, run_ind-1) == in arena_run_coalesce()
2060 run_ind-1); in arena_run_coalesce()
2063 run_ind -= prun_pages; in arena_run_coalesce()
2069 assert(arena_mapbits_unallocated_size_get(chunk, run_ind) == in arena_run_coalesce()
2071 assert(arena_mapbits_dirty_get(chunk, run_ind) == flag_dirty); in arena_run_coalesce()
2072 assert(arena_mapbits_decommitted_get(chunk, run_ind) == in arena_run_coalesce()
2074 arena_avail_remove(arena, chunk, run_ind, prun_pages); in arena_run_coalesce()
2081 arena_run_dirty_remove(arena, chunk, run_ind, in arena_run_coalesce()
2088 arena_mapbits_unallocated_size_set(chunk, run_ind, size); in arena_run_coalesce()
2089 arena_mapbits_unallocated_size_set(chunk, run_ind+run_pages-1, in arena_run_coalesce()
2094 *p_run_ind = run_ind; in arena_run_coalesce()
2100 size_t run_ind) in arena_run_size_get() argument
2104 assert(run_ind >= map_bias); in arena_run_size_get()
2105 assert(run_ind < chunk_npages); in arena_run_size_get()
2107 if (arena_mapbits_large_get(chunk, run_ind) != 0) { in arena_run_size_get()
2108 size = arena_mapbits_large_size_get(chunk, run_ind); in arena_run_size_get()
2110 run_ind+(size>>LG_PAGE)-1) == 0); in arena_run_size_get()
2125 size_t size, run_ind, run_pages, flag_dirty, flag_decommitted; in arena_run_dalloc() local
2129 run_ind = arena_miscelm_to_pageind(miscelm); in arena_run_dalloc()
2130 assert(run_ind >= map_bias); in arena_run_dalloc()
2131 assert(run_ind < chunk_npages); in arena_run_dalloc()
2132 size = arena_run_size_get(arena, chunk, run, run_ind); in arena_run_dalloc()
2141 assert(arena_mapbits_dirty_get(chunk, run_ind) == in arena_run_dalloc()
2142 arena_mapbits_dirty_get(chunk, run_ind+run_pages-1)); in arena_run_dalloc()
2143 if (!cleaned && !decommitted && arena_mapbits_dirty_get(chunk, run_ind) in arena_run_dalloc()
2152 arena_mapbits_unallocated_set(chunk, run_ind, size, flags); in arena_run_dalloc()
2153 arena_mapbits_unallocated_set(chunk, run_ind+run_pages-1, size, in arena_run_dalloc()
2156 arena_mapbits_unallocated_set(chunk, run_ind, size, in arena_run_dalloc()
2157 arena_mapbits_unzeroed_get(chunk, run_ind)); in arena_run_dalloc()
2158 arena_mapbits_unallocated_set(chunk, run_ind+run_pages-1, size, in arena_run_dalloc()
2159 arena_mapbits_unzeroed_get(chunk, run_ind+run_pages-1)); in arena_run_dalloc()
2162 arena_run_coalesce(arena, chunk, &size, &run_ind, &run_pages, in arena_run_dalloc()
2166 assert(arena_mapbits_unallocated_size_get(chunk, run_ind) == in arena_run_dalloc()
2167 arena_mapbits_unallocated_size_get(chunk, run_ind+run_pages-1)); in arena_run_dalloc()
2168 assert(arena_mapbits_dirty_get(chunk, run_ind) == in arena_run_dalloc()
2169 arena_mapbits_dirty_get(chunk, run_ind+run_pages-1)); in arena_run_dalloc()
2170 assert(arena_mapbits_decommitted_get(chunk, run_ind) == in arena_run_dalloc()
2171 arena_mapbits_decommitted_get(chunk, run_ind+run_pages-1)); in arena_run_dalloc()
2172 arena_avail_insert(arena, chunk, run_ind, run_pages); in arena_run_dalloc()
2175 arena_run_dirty_insert(arena, chunk, run_ind, run_pages); in arena_run_dalloc()
2179 assert(run_ind == map_bias); in arena_run_dalloc()
2754 size_t run_ind = in arena_palloc_large() local
2756 bool dirty = (arena_mapbits_dirty_get(chunk, run_ind) != 0); in arena_palloc_large()
2758 run_ind) != 0); in arena_palloc_large()