Lines Matching refs:tsd
476 arena_bind(tsd_t *tsd, unsigned ind, bool internal) in arena_bind() argument
480 if (!tsd_nominal(tsd)) in arena_bind()
483 arena = arena_get(tsd_tsdn(tsd), ind, false); in arena_bind()
487 tsd_iarena_set(tsd, arena); in arena_bind()
489 tsd_arena_set(tsd, arena); in arena_bind()
493 arena_migrate(tsd_t *tsd, unsigned oldind, unsigned newind) in arena_migrate() argument
497 oldarena = arena_get(tsd_tsdn(tsd), oldind, false); in arena_migrate()
498 newarena = arena_get(tsd_tsdn(tsd), newind, false); in arena_migrate()
501 tsd_arena_set(tsd, newarena); in arena_migrate()
505 arena_unbind(tsd_t *tsd, unsigned ind, bool internal) in arena_unbind() argument
509 arena = arena_get(tsd_tsdn(tsd), ind, false); in arena_unbind()
512 tsd_iarena_set(tsd, NULL); in arena_unbind()
514 tsd_arena_set(tsd, NULL); in arena_unbind()
518 arena_tdata_get_hard(tsd_t *tsd, unsigned ind) in arena_tdata_get_hard() argument
521 arena_tdata_t *arenas_tdata = tsd_arenas_tdata_get(tsd); in arena_tdata_get_hard()
523 unsigned narenas_tdata = tsd_narenas_tdata_get(tsd); in arena_tdata_get_hard()
535 tsd_arenas_tdata_set(tsd, arenas_tdata); in arena_tdata_get_hard()
536 tsd_narenas_tdata_set(tsd, narenas_tdata); in arena_tdata_get_hard()
544 bool *arenas_tdata_bypassp = tsd_arenas_tdata_bypassp_get(tsd); in arena_tdata_get_hard()
547 if (tsd_nominal(tsd) && !*arenas_tdata_bypassp) { in arena_tdata_get_hard()
557 assert(tsd_nominal(tsd) && !*arenas_tdata_bypassp); in arena_tdata_get_hard()
558 tsd_arenas_tdata_set(tsd, arenas_tdata); in arena_tdata_get_hard()
559 tsd_narenas_tdata_set(tsd, narenas_tdata); in arena_tdata_get_hard()
595 arena_choose_hard(tsd_t *tsd, bool internal) in arena_choose_hard() argument
614 malloc_mutex_lock(tsd_tsdn(tsd), &arenas_lock); in arena_choose_hard()
615 assert(arena_get(tsd_tsdn(tsd), 0, false) != NULL); in arena_choose_hard()
617 if (arena_get(tsd_tsdn(tsd), i, false) != NULL) { in arena_choose_hard()
624 tsd_tsdn(tsd), i, false), !!j) < in arena_choose_hard()
626 tsd_tsdn(tsd), choose[j], false), in arena_choose_hard()
645 if (arena_nthreads_get(arena_get(tsd_tsdn(tsd), in arena_choose_hard()
653 ret = arena_get(tsd_tsdn(tsd), in arena_choose_hard()
661 arena = arena_init_locked(tsd_tsdn(tsd), in arena_choose_hard()
664 malloc_mutex_unlock(tsd_tsdn(tsd), in arena_choose_hard()
671 arena_bind(tsd, choose[j], !!j); in arena_choose_hard()
673 malloc_mutex_unlock(tsd_tsdn(tsd), &arenas_lock); in arena_choose_hard()
675 ret = arena_get(tsd_tsdn(tsd), 0, false); in arena_choose_hard()
676 arena_bind(tsd, 0, false); in arena_choose_hard()
677 arena_bind(tsd, 0, true); in arena_choose_hard()
684 thread_allocated_cleanup(tsd_t *tsd) in thread_allocated_cleanup() argument
691 thread_deallocated_cleanup(tsd_t *tsd) in thread_deallocated_cleanup() argument
698 iarena_cleanup(tsd_t *tsd) in iarena_cleanup() argument
702 iarena = tsd_iarena_get(tsd); in iarena_cleanup()
704 arena_unbind(tsd, iarena->ind, true); in iarena_cleanup()
708 arena_cleanup(tsd_t *tsd) in arena_cleanup() argument
712 arena = tsd_arena_get(tsd); in arena_cleanup()
714 arena_unbind(tsd, arena->ind, false); in arena_cleanup()
718 arenas_tdata_cleanup(tsd_t *tsd) in arenas_tdata_cleanup() argument
723 *tsd_arenas_tdata_bypassp_get(tsd) = true; in arenas_tdata_cleanup()
725 arenas_tdata = tsd_arenas_tdata_get(tsd); in arenas_tdata_cleanup()
727 tsd_arenas_tdata_set(tsd, NULL); in arenas_tdata_cleanup()
733 narenas_tdata_cleanup(tsd_t *tsd) in narenas_tdata_cleanup() argument
740 arenas_tdata_bypass_cleanup(tsd_t *tsd) in arenas_tdata_bypass_cleanup() argument
1487 tsd_t *tsd; local
1506 tsd = malloc_tsd_boot0();
1507 if (tsd == NULL)
1511 malloc_mutex_lock(tsd_tsdn(tsd), &init_lock);
1513 if (config_prof && prof_boot2(tsd)) {
1514 malloc_mutex_unlock(tsd_tsdn(tsd), &init_lock);
1518 if (malloc_init_hard_finish(tsd_tsdn(tsd))) {
1519 malloc_mutex_unlock(tsd_tsdn(tsd), &init_lock);
1523 malloc_mutex_unlock(tsd_tsdn(tsd), &init_lock);
1537 ialloc_prof_sample(tsd_t *tsd, size_t usize, szind_t ind, bool zero, argument
1546 p = ialloc(tsd, LARGE_MINCLASS, ind_large, zero, slow_path);
1549 arena_prof_promoted(tsd_tsdn(tsd), p, usize);
1551 p = ialloc(tsd, usize, ind, zero, slow_path);
1557 ialloc_prof(tsd_t *tsd, size_t usize, szind_t ind, bool zero, bool slow_path) argument
1562 tctx = prof_alloc_prep(tsd, usize, prof_active_get_unlocked(), true);
1564 p = ialloc_prof_sample(tsd, usize, ind, zero, tctx, slow_path);
1566 p = ialloc(tsd, usize, ind, zero, slow_path);
1568 prof_alloc_rollback(tsd, tctx, true);
1571 prof_malloc(tsd_tsdn(tsd), p, usize, tctx);
1586 tsd_t *tsd; local
1594 tsd = tsd_fetch();
1595 *tsdn = tsd_tsdn(tsd);
1596 witness_assert_lockless(tsd_tsdn(tsd));
1609 return (ialloc_prof(tsd, *usize, ind, zero, slow_path));
1611 return (ialloc(tsd, size, ind, zero, slow_path));
1663 imemalign_prof_sample(tsd_t *tsd, size_t alignment, size_t usize, argument
1672 p = ipalloc(tsd, LARGE_MINCLASS, alignment, false);
1675 arena_prof_promoted(tsd_tsdn(tsd), p, usize);
1677 p = ipalloc(tsd, usize, alignment, false);
1683 imemalign_prof(tsd_t *tsd, size_t alignment, size_t usize) argument
1688 tctx = prof_alloc_prep(tsd, usize, prof_active_get_unlocked(), true);
1690 p = imemalign_prof_sample(tsd, alignment, usize, tctx);
1692 p = ipalloc(tsd, usize, alignment, false);
1694 prof_alloc_rollback(tsd, tctx, true);
1697 prof_malloc(tsd_tsdn(tsd), p, usize, tctx);
1707 tsd_t *tsd; local
1714 tsd = NULL;
1718 tsd = tsd_fetch();
1719 witness_assert_lockless(tsd_tsdn(tsd));
1743 result = imemalign_prof(tsd, alignment, usize);
1745 result = ipalloc(tsd, usize, alignment, false);
1754 assert(usize == isalloc(tsd_tsdn(tsd), result, config_prof));
1755 *tsd_thread_allocatedp_get(tsd) += usize;
1758 JEMALLOC_VALGRIND_MALLOC(result != NULL, tsd_tsdn(tsd), result, usize,
1760 witness_assert_lockless(tsd_tsdn(tsd));
1770 witness_assert_lockless(tsd_tsdn(tsd));
1840 irealloc_prof_sample(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t usize, argument
1848 p = iralloc(tsd, old_ptr, old_usize, LARGE_MINCLASS, 0, false);
1851 arena_prof_promoted(tsd_tsdn(tsd), p, usize);
1853 p = iralloc(tsd, old_ptr, old_usize, usize, 0, false);
1859 irealloc_prof(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t usize) argument
1866 old_tctx = prof_tctx_get(tsd_tsdn(tsd), old_ptr);
1867 tctx = prof_alloc_prep(tsd, usize, prof_active, true);
1869 p = irealloc_prof_sample(tsd, old_ptr, old_usize, usize, tctx);
1871 p = iralloc(tsd, old_ptr, old_usize, usize, 0, false);
1873 prof_alloc_rollback(tsd, tctx, true);
1876 prof_realloc(tsd, p, usize, tctx, prof_active, true, old_ptr, old_usize,
1883 ifree(tsd_t *tsd, void *ptr, tcache_t *tcache, bool slow_path) argument
1888 witness_assert_lockless(tsd_tsdn(tsd));
1894 usize = isalloc(tsd_tsdn(tsd), ptr, config_prof);
1895 prof_free(tsd, ptr, usize);
1897 usize = isalloc(tsd_tsdn(tsd), ptr, config_prof);
1899 *tsd_thread_deallocatedp_get(tsd) += usize;
1902 iqalloc(tsd, ptr, tcache, false);
1905 rzsize = p2rz(tsd_tsdn(tsd), ptr);
1906 iqalloc(tsd, ptr, tcache, true);
1912 isfree(tsd_t *tsd, void *ptr, size_t usize, tcache_t *tcache, bool slow_path) argument
1916 witness_assert_lockless(tsd_tsdn(tsd));
1922 prof_free(tsd, ptr, usize);
1924 *tsd_thread_deallocatedp_get(tsd) += usize;
1926 rzsize = p2rz(tsd_tsdn(tsd), ptr);
1927 isqalloc(tsd, ptr, usize, tcache, slow_path);
1944 tsd_t *tsd; local
1948 tsd = tsd_fetch();
1949 ifree(tsd, ptr, tcache_get(tsd, false), true);
1956 tsd_t *tsd; local
1960 tsd = tsd_fetch();
1962 witness_assert_lockless(tsd_tsdn(tsd));
1964 old_usize = isalloc(tsd_tsdn(tsd), ptr, config_prof);
1966 old_rzsize = config_prof ? p2rz(tsd_tsdn(tsd), ptr) :
1973 NULL : irealloc_prof(tsd, ptr, old_usize, usize);
1978 ret = iralloc(tsd, ptr, old_usize, size, 0, false);
1980 tsdn = tsd_tsdn(tsd);
1999 tsd_t *tsd; local
2002 tsd = tsdn_tsd(tsdn);
2003 *tsd_thread_allocatedp_get(tsd) += usize;
2004 *tsd_thread_deallocatedp_get(tsd) += old_usize;
2019 tsd_t *tsd = tsd_fetch(); local
2020 witness_assert_lockless(tsd_tsdn(tsd));
2022 ifree(tsd, ptr, tcache_get(tsd, false), false);
2024 ifree(tsd, ptr, tcache_get(tsd, false), true);
2025 witness_assert_lockless(tsd_tsdn(tsd));
2122 imallocx_flags_decode(tsd_t *tsd, size_t size, int flags, size_t *usize, argument
2140 *tcache = tcaches_get(tsd, MALLOCX_TCACHE_GET(flags));
2142 *tcache = tcache_get(tsd, true);
2145 *arena = arena_get(tsd_tsdn(tsd), arena_ind, true);
2190 imallocx_prof(tsd_t *tsd, size_t size, int flags, size_t *usize, bool slow_path) argument
2199 if (unlikely(imallocx_flags_decode(tsd, size, flags, usize, &alignment,
2202 tctx = prof_alloc_prep(tsd, *usize, prof_active_get_unlocked(), true);
2204 p = imallocx_flags(tsd_tsdn(tsd), *usize, alignment, zero,
2207 p = imallocx_prof_sample(tsd_tsdn(tsd), *usize, alignment, zero,
2212 prof_alloc_rollback(tsd, tctx, true);
2215 prof_malloc(tsd_tsdn(tsd), p, *usize, tctx);
2222 imallocx_no_prof(tsd_t *tsd, size_t size, int flags, size_t *usize, argument
2231 if (unlikely(imallocx_flags_decode(tsd, size, flags, usize, &alignment,
2234 p = imallocx_flags(tsd_tsdn(tsd), *usize, alignment, zero, tcache,
2245 tsd_t *tsd; local
2252 tsd = tsd_fetch();
2253 *tsdn = tsd_tsdn(tsd);
2254 witness_assert_lockless(tsd_tsdn(tsd));
2267 return (ialloc_prof(tsd, *usize, ind, false,
2271 return (ialloc(tsd, size, ind, false, slow_path));
2275 return (imallocx_prof(tsd, size, flags, usize, slow_path));
2277 return (imallocx_no_prof(tsd, size, flags, usize, slow_path));
2306 irallocx_prof_sample(tsd_t *tsd, void *old_ptr, size_t old_usize, argument
2315 p = iralloct(tsd, old_ptr, old_usize, LARGE_MINCLASS, alignment,
2319 arena_prof_promoted(tsd_tsdn(tsd), p, usize);
2321 p = iralloct(tsd, old_ptr, old_usize, usize, alignment, zero,
2329 irallocx_prof(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t size, argument
2338 old_tctx = prof_tctx_get(tsd_tsdn(tsd), old_ptr);
2339 tctx = prof_alloc_prep(tsd, *usize, prof_active, false);
2341 p = irallocx_prof_sample(tsd, old_ptr, old_usize, *usize,
2344 p = iralloct(tsd, old_ptr, old_usize, size, alignment, zero,
2348 prof_alloc_rollback(tsd, tctx, false);
2361 *usize = isalloc(tsd_tsdn(tsd), p, config_prof);
2363 prof_realloc(tsd, p, *usize, tctx, prof_active, false, old_ptr,
2375 tsd_t *tsd; local
2388 tsd = tsd_fetch();
2389 witness_assert_lockless(tsd_tsdn(tsd));
2393 arena = arena_get(tsd_tsdn(tsd), arena_ind, true);
2403 tcache = tcaches_get(tsd, MALLOCX_TCACHE_GET(flags));
2405 tcache = tcache_get(tsd, true);
2407 old_usize = isalloc(tsd_tsdn(tsd), ptr, config_prof);
2415 p = irallocx_prof(tsd, ptr, old_usize, size, alignment, &usize,
2420 p = iralloct(tsd, ptr, old_usize, size, alignment, zero,
2425 usize = isalloc(tsd_tsdn(tsd), p, config_prof);
2430 *tsd_thread_allocatedp_get(tsd) += usize;
2431 *tsd_thread_deallocatedp_get(tsd) += old_usize;
2434 JEMALLOC_VALGRIND_REALLOC(maybe, tsd_tsdn(tsd), p, usize, no, ptr,
2436 witness_assert_lockless(tsd_tsdn(tsd));
2444 witness_assert_lockless(tsd_tsdn(tsd));
2476 ixallocx_prof(tsd_t *tsd, void *ptr, size_t old_usize, size_t size, argument
2484 old_tctx = prof_tctx_get(tsd_tsdn(tsd), ptr);
2506 tctx = prof_alloc_prep(tsd, usize_max, prof_active, false);
2509 usize = ixallocx_prof_sample(tsd_tsdn(tsd), ptr, old_usize,
2512 usize = ixallocx_helper(tsd_tsdn(tsd), ptr, old_usize, size,
2516 prof_alloc_rollback(tsd, tctx, false);
2519 prof_realloc(tsd, ptr, usize, tctx, prof_active, false, ptr, old_usize,
2528 tsd_t *tsd; local
2539 tsd = tsd_fetch();
2540 witness_assert_lockless(tsd_tsdn(tsd));
2542 old_usize = isalloc(tsd_tsdn(tsd), ptr, config_prof);
2564 usize = ixallocx_prof(tsd, ptr, old_usize, size, extra,
2567 usize = ixallocx_helper(tsd_tsdn(tsd), ptr, old_usize, size,
2574 *tsd_thread_allocatedp_get(tsd) += usize;
2575 *tsd_thread_deallocatedp_get(tsd) += old_usize;
2577 JEMALLOC_VALGRIND_REALLOC(no, tsd_tsdn(tsd), ptr, usize, no, ptr,
2581 witness_assert_lockless(tsd_tsdn(tsd));
2610 tsd_t *tsd; local
2616 tsd = tsd_fetch();
2617 witness_assert_lockless(tsd_tsdn(tsd));
2622 tcache = tcaches_get(tsd, MALLOCX_TCACHE_GET(flags));
2624 tcache = tcache_get(tsd, false);
2628 ifree(tsd, ptr, tcache, false);
2630 ifree(tsd, ptr, tcache, true);
2631 witness_assert_lockless(tsd_tsdn(tsd));
2652 tsd_t *tsd; local
2658 tsd = tsd_fetch();
2659 usize = inallocx(tsd_tsdn(tsd), size, flags);
2660 assert(usize == isalloc(tsd_tsdn(tsd), ptr, config_prof));
2662 witness_assert_lockless(tsd_tsdn(tsd));
2667 tcache = tcaches_get(tsd, MALLOCX_TCACHE_GET(flags));
2669 tcache = tcache_get(tsd, false);
2673 isfree(tsd, ptr, usize, tcache, false);
2675 isfree(tsd, ptr, usize, tcache, true);
2676 witness_assert_lockless(tsd_tsdn(tsd));
2707 tsd_t *tsd; local
2712 tsd = tsd_fetch();
2713 witness_assert_lockless(tsd_tsdn(tsd));
2714 ret = ctl_byname(tsd, name, oldp, oldlenp, newp, newlen);
2715 witness_assert_lockless(tsd_tsdn(tsd));
2740 tsd_t *tsd; local
2745 tsd = tsd_fetch();
2746 witness_assert_lockless(tsd_tsdn(tsd));
2747 ret = ctl_bymib(tsd, mib, miblen, oldp, oldlenp, newp, newlen);
2748 witness_assert_lockless(tsd_tsdn(tsd));
2825 tsd_t *tsd; local
2835 tsd = tsd_fetch();
2839 witness_prefork(tsd);
2841 ctl_prefork(tsd_tsdn(tsd));
2842 tcache_prefork(tsd_tsdn(tsd));
2843 malloc_mutex_prefork(tsd_tsdn(tsd), &arenas_lock);
2844 prof_prefork0(tsd_tsdn(tsd));
2847 if ((arena = arena_get(tsd_tsdn(tsd), j, false)) !=
2851 arena_prefork0(tsd_tsdn(tsd), arena);
2854 arena_prefork1(tsd_tsdn(tsd), arena);
2857 arena_prefork2(tsd_tsdn(tsd), arena);
2864 base_prefork(tsd_tsdn(tsd));
2866 if ((arena = arena_get(tsd_tsdn(tsd), i, false)) != NULL)
2867 arena_prefork3(tsd_tsdn(tsd), arena);
2869 prof_prefork1(tsd_tsdn(tsd));
2880 tsd_t *tsd; local
2889 tsd = tsd_fetch();
2891 witness_postfork_parent(tsd);
2893 base_postfork_parent(tsd_tsdn(tsd));
2897 if ((arena = arena_get(tsd_tsdn(tsd), i, false)) != NULL)
2898 arena_postfork_parent(tsd_tsdn(tsd), arena);
2900 prof_postfork_parent(tsd_tsdn(tsd));
2901 malloc_mutex_postfork_parent(tsd_tsdn(tsd), &arenas_lock);
2902 tcache_postfork_parent(tsd_tsdn(tsd));
2903 ctl_postfork_parent(tsd_tsdn(tsd));
2909 tsd_t *tsd; local
2914 tsd = tsd_fetch();
2916 witness_postfork_child(tsd);
2918 base_postfork_child(tsd_tsdn(tsd));
2922 if ((arena = arena_get(tsd_tsdn(tsd), i, false)) != NULL)
2923 arena_postfork_child(tsd_tsdn(tsd), arena);
2925 prof_postfork_child(tsd_tsdn(tsd));
2926 malloc_mutex_postfork_child(tsd_tsdn(tsd), &arenas_lock);
2927 tcache_postfork_child(tsd_tsdn(tsd));
2928 ctl_postfork_child(tsd_tsdn(tsd));