Lines Matching refs:tcache
44 tcache_event_hard(tsd_t *tsd, tcache_t *tcache) { in tcache_event_hard() argument
45 szind_t binind = tcache->next_gc_bin; in tcache_event_hard()
49 tbin = tcache_small_bin_get(tcache, binind); in tcache_event_hard()
51 tbin = tcache_large_bin_get(tcache, binind); in tcache_event_hard()
58 tcache_bin_flush_small(tsd, tcache, tbin, binind, in tcache_event_hard()
67 (tcache->lg_fill_div[binind] + 1)) >= 1) { in tcache_event_hard()
68 tcache->lg_fill_div[binind]++; in tcache_event_hard()
72 - tbin->low_water + (tbin->low_water >> 2), tcache); in tcache_event_hard()
79 if (binind < NBINS && tcache->lg_fill_div[binind] > 1) { in tcache_event_hard()
80 tcache->lg_fill_div[binind]--; in tcache_event_hard()
85 tcache->next_gc_bin++; in tcache_event_hard()
86 if (tcache->next_gc_bin == nhbins) { in tcache_event_hard()
87 tcache->next_gc_bin = 0; in tcache_event_hard()
92 tcache_alloc_small_hard(tsdn_t *tsdn, arena_t *arena, tcache_t *tcache, in tcache_alloc_small_hard() argument
96 assert(tcache->arena != NULL); in tcache_alloc_small_hard()
97 arena_tcache_fill_small(tsdn, arena, tcache, tbin, binind, in tcache_alloc_small_hard()
98 config_prof ? tcache->prof_accumbytes : 0); in tcache_alloc_small_hard()
100 tcache->prof_accumbytes = 0; in tcache_alloc_small_hard()
108 tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, cache_bin_t *tbin, in tcache_bin_flush_small() argument
115 arena_t *arena = tcache->arena; in tcache_bin_flush_small()
132 tcache->prof_accumbytes)) { in tcache_bin_flush_small()
135 tcache->prof_accumbytes = 0; in tcache_bin_flush_small()
198 unsigned rem, tcache_t *tcache) { in tcache_bin_flush_large() argument
206 arena_t *arena = tcache->arena; in tcache_bin_flush_large()
238 tcache->prof_accumbytes); in tcache_bin_flush_large()
239 tcache->prof_accumbytes = 0; in tcache_bin_flush_large()
301 tcache_arena_associate(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena) { in tcache_arena_associate() argument
302 assert(tcache->arena == NULL); in tcache_arena_associate()
303 tcache->arena = arena; in tcache_arena_associate()
309 ql_elm_new(tcache, link); in tcache_arena_associate()
310 ql_tail_insert(&arena->tcache_ql, tcache, link); in tcache_arena_associate()
312 &tcache->cache_bin_array_descriptor, tcache->bins_small, in tcache_arena_associate()
313 tcache->bins_large); in tcache_arena_associate()
315 &tcache->cache_bin_array_descriptor, link); in tcache_arena_associate()
322 tcache_arena_dissociate(tsdn_t *tsdn, tcache_t *tcache) { in tcache_arena_dissociate() argument
323 arena_t *arena = tcache->arena; in tcache_arena_dissociate()
332 if (iter == tcache) { in tcache_arena_dissociate()
339 ql_remove(&arena->tcache_ql, tcache, link); in tcache_arena_dissociate()
341 &tcache->cache_bin_array_descriptor, link); in tcache_arena_dissociate()
342 tcache_stats_merge(tsdn, tcache, arena); in tcache_arena_dissociate()
345 tcache->arena = NULL; in tcache_arena_dissociate()
349 tcache_arena_reassociate(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena) { in tcache_arena_reassociate() argument
350 tcache_arena_dissociate(tsdn, tcache); in tcache_arena_reassociate()
351 tcache_arena_associate(tsdn, tcache, arena); in tcache_arena_reassociate()
370 tcache_init(tsd_t *tsd, tcache_t *tcache, void *avail_stack) { in tcache_init() argument
371 memset(&tcache->link, 0, sizeof(ql_elm(tcache_t))); in tcache_init()
372 tcache->prof_accumbytes = 0; in tcache_init()
373 tcache->next_gc_bin = 0; in tcache_init()
374 tcache->arena = NULL; in tcache_init()
376 ticker_init(&tcache->gc_ticker, TCACHE_GC_INCR); in tcache_init()
380 memset(tcache->bins_small, 0, sizeof(cache_bin_t) * NBINS); in tcache_init()
381 memset(tcache->bins_large, 0, sizeof(cache_bin_t) * (nhbins - NBINS)); in tcache_init()
384 tcache->lg_fill_div[i] = 1; in tcache_init()
391 tcache_small_bin_get(tcache, i)->avail = in tcache_init()
396 tcache_large_bin_get(tcache, i)->avail = in tcache_init()
405 tcache_t *tcache = tsd_tcachep_get_unsafe(tsd); in tsd_tcache_data_init() local
406 assert(tcache_small_bin_get(tcache, 0)->avail == NULL); in tsd_tcache_data_init()
417 tcache_init(tsd, tcache, avail_array); in tsd_tcache_data_init()
426 tcache->arena = NULL; in tsd_tcache_data_init()
431 tcache_arena_associate(tsd_tsdn(tsd), tcache, arena); in tsd_tcache_data_init()
435 if (tcache->arena == NULL) { in tsd_tcache_data_init()
436 tcache_arena_associate(tsd_tsdn(tsd), tcache, arena); in tsd_tcache_data_init()
439 assert(arena == tcache->arena); in tsd_tcache_data_init()
447 tcache_t *tcache; in tcache_create_explicit() local
458 tcache = ipallocztm(tsd_tsdn(tsd), size, CACHELINE, true, NULL, true, in tcache_create_explicit()
460 if (tcache == NULL) { in tcache_create_explicit()
464 tcache_init(tsd, tcache, in tcache_create_explicit()
465 (void *)((uintptr_t)tcache + (uintptr_t)stack_offset)); in tcache_create_explicit()
466 tcache_arena_associate(tsd_tsdn(tsd), tcache, arena_ichoose(tsd, NULL)); in tcache_create_explicit()
468 return tcache; in tcache_create_explicit()
472 tcache_flush_cache(tsd_t *tsd, tcache_t *tcache) { in tcache_flush_cache() argument
473 assert(tcache->arena != NULL); in tcache_flush_cache()
476 cache_bin_t *tbin = tcache_small_bin_get(tcache, i); in tcache_flush_cache()
477 tcache_bin_flush_small(tsd, tcache, tbin, i, 0); in tcache_flush_cache()
486 cache_bin_t *tbin = tcache_large_bin_get(tcache, i); in tcache_flush_cache()
487 tcache_bin_flush_large(tsd, tbin, i, 0, tcache); in tcache_flush_cache()
496 if (config_prof && tcache->prof_accumbytes > 0 && in tcache_flush_cache()
497 arena_prof_accum(tsd_tsdn(tsd), tcache->arena, in tcache_flush_cache()
498 tcache->prof_accumbytes)) { in tcache_flush_cache()
510 tcache_destroy(tsd_t *tsd, tcache_t *tcache, bool tsd_tcache) { in tcache_destroy() argument
511 tcache_flush_cache(tsd, tcache); in tcache_destroy()
512 tcache_arena_dissociate(tsd_tsdn(tsd), tcache); in tcache_destroy()
517 (void *)((uintptr_t)tcache_small_bin_get(tcache, 0)->avail - in tcache_destroy()
522 idalloctm(tsd_tsdn(tsd), tcache, NULL, NULL, true, true); in tcache_destroy()
529 tcache_t *tcache = tsd_tcachep_get(tsd); in tcache_cleanup() local
533 assert(tcache_small_bin_get(tcache, 0)->avail == NULL); in tcache_cleanup()
538 assert(tcache_small_bin_get(tcache, 0)->avail != NULL); in tcache_cleanup()
540 tcache_destroy(tsd, tcache, true); in tcache_cleanup()
542 tcache_small_bin_get(tcache, 0)->avail = NULL; in tcache_cleanup()
547 tcache_stats_merge(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena) { in tcache_stats_merge() argument
556 cache_bin_t *tbin = tcache_small_bin_get(tcache, i); in tcache_stats_merge()
564 cache_bin_t *tbin = tcache_large_bin_get(tcache, i); in tcache_stats_merge()
609 tcache_t *tcache = tcache_create_explicit(tsd); in tcaches_create() local
610 if (tcache == NULL) { in tcaches_create()
620 elm->tcache = tcache; in tcaches_create()
624 elm->tcache = tcache; in tcaches_create()
640 if (elm->tcache == NULL) { in tcaches_elm_remove()
643 tcache_t *tcache = elm->tcache; in tcaches_elm_remove() local
644 elm->tcache = NULL; in tcaches_elm_remove()
645 return tcache; in tcaches_elm_remove()
651 tcache_t *tcache = tcaches_elm_remove(tsd, &tcaches[ind]); in tcaches_flush() local
653 if (tcache != NULL) { in tcaches_flush()
654 tcache_destroy(tsd, tcache, false); in tcaches_flush()
662 tcache_t *tcache = tcaches_elm_remove(tsd, elm); in tcaches_destroy() local
666 if (tcache != NULL) { in tcaches_destroy()
667 tcache_destroy(tsd, tcache, false); in tcaches_destroy()