Lines Matching refs:decay
52 arena_decay_t *decay, extents_t *extents, bool all, size_t npages_limit,
419 arena_decay_ms_read(arena_decay_t *decay) { in arena_decay_ms_read() argument
420 return atomic_load_zd(&decay->time_ms, ATOMIC_RELAXED); in arena_decay_ms_read()
424 arena_decay_ms_write(arena_decay_t *decay, ssize_t decay_ms) { in arena_decay_ms_write() argument
425 atomic_store_zd(&decay->time_ms, decay_ms, ATOMIC_RELAXED); in arena_decay_ms_write()
429 arena_decay_deadline_init(arena_decay_t *decay) { in arena_decay_deadline_init() argument
434 nstime_copy(&decay->deadline, &decay->epoch); in arena_decay_deadline_init()
435 nstime_add(&decay->deadline, &decay->interval); in arena_decay_deadline_init()
436 if (arena_decay_ms_read(decay) > 0) { in arena_decay_deadline_init()
439 nstime_init(&jitter, prng_range_u64(&decay->jitter_state, in arena_decay_deadline_init()
440 nstime_ns(&decay->interval))); in arena_decay_deadline_init()
441 nstime_add(&decay->deadline, &jitter); in arena_decay_deadline_init()
446 arena_decay_deadline_reached(const arena_decay_t *decay, const nstime_t *time) { in arena_decay_deadline_reached() argument
447 return (nstime_compare(&decay->deadline, time) <= 0); in arena_decay_deadline_reached()
451 arena_decay_backlog_npages_limit(const arena_decay_t *decay) { in arena_decay_backlog_npages_limit() argument
463 sum += decay->backlog[i] * h_steps[i]; in arena_decay_backlog_npages_limit()
471 arena_decay_backlog_update_last(arena_decay_t *decay, size_t current_npages) { in arena_decay_backlog_update_last() argument
472 size_t npages_delta = (current_npages > decay->nunpurged) ? in arena_decay_backlog_update_last()
473 current_npages - decay->nunpurged : 0; in arena_decay_backlog_update_last()
474 decay->backlog[SMOOTHSTEP_NSTEPS-1] = npages_delta; in arena_decay_backlog_update_last()
477 if (current_npages > decay->ceil_npages) { in arena_decay_backlog_update_last()
478 decay->ceil_npages = current_npages; in arena_decay_backlog_update_last()
480 size_t npages_limit = arena_decay_backlog_npages_limit(decay); in arena_decay_backlog_update_last()
481 assert(decay->ceil_npages >= npages_limit); in arena_decay_backlog_update_last()
482 if (decay->ceil_npages > npages_limit) { in arena_decay_backlog_update_last()
483 decay->ceil_npages = npages_limit; in arena_decay_backlog_update_last()
489 arena_decay_backlog_update(arena_decay_t *decay, uint64_t nadvance_u64, in arena_decay_backlog_update() argument
492 memset(decay->backlog, 0, (SMOOTHSTEP_NSTEPS-1) * in arena_decay_backlog_update()
499 memmove(decay->backlog, &decay->backlog[nadvance_z], in arena_decay_backlog_update()
502 memset(&decay->backlog[SMOOTHSTEP_NSTEPS - in arena_decay_backlog_update()
507 arena_decay_backlog_update_last(decay, current_npages); in arena_decay_backlog_update()
511 arena_decay_try_purge(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay, in arena_decay_try_purge() argument
515 arena_decay_to_limit(tsdn, arena, decay, extents, false, in arena_decay_try_purge()
522 arena_decay_epoch_advance_helper(arena_decay_t *decay, const nstime_t *time, in arena_decay_epoch_advance_helper() argument
524 assert(arena_decay_deadline_reached(decay, time)); in arena_decay_epoch_advance_helper()
528 nstime_subtract(&delta, &decay->epoch); in arena_decay_epoch_advance_helper()
530 uint64_t nadvance_u64 = nstime_divide(&delta, &decay->interval); in arena_decay_epoch_advance_helper()
534 nstime_copy(&delta, &decay->interval); in arena_decay_epoch_advance_helper()
536 nstime_add(&decay->epoch, &delta); in arena_decay_epoch_advance_helper()
539 arena_decay_deadline_init(decay); in arena_decay_epoch_advance_helper()
542 arena_decay_backlog_update(decay, nadvance_u64, current_npages); in arena_decay_epoch_advance_helper()
546 arena_decay_epoch_advance(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay, in arena_decay_epoch_advance() argument
549 arena_decay_epoch_advance_helper(decay, time, current_npages); in arena_decay_epoch_advance()
551 size_t npages_limit = arena_decay_backlog_npages_limit(decay); in arena_decay_epoch_advance()
553 decay->nunpurged = (npages_limit > current_npages) ? npages_limit : in arena_decay_epoch_advance()
557 arena_decay_try_purge(tsdn, arena, decay, extents, in arena_decay_epoch_advance()
563 arena_decay_reinit(arena_decay_t *decay, ssize_t decay_ms) { in arena_decay_reinit() argument
564 arena_decay_ms_write(decay, decay_ms); in arena_decay_reinit()
566 nstime_init(&decay->interval, (uint64_t)decay_ms * in arena_decay_reinit()
568 nstime_idivide(&decay->interval, SMOOTHSTEP_NSTEPS); in arena_decay_reinit()
571 nstime_init(&decay->epoch, 0); in arena_decay_reinit()
572 nstime_update(&decay->epoch); in arena_decay_reinit()
573 decay->jitter_state = (uint64_t)(uintptr_t)decay; in arena_decay_reinit()
574 arena_decay_deadline_init(decay); in arena_decay_reinit()
575 decay->nunpurged = 0; in arena_decay_reinit()
576 memset(decay->backlog, 0, SMOOTHSTEP_NSTEPS * sizeof(size_t)); in arena_decay_reinit()
580 arena_decay_init(arena_decay_t *decay, ssize_t decay_ms, in arena_decay_init() argument
584 assert(((char *)decay)[i] == 0); in arena_decay_init()
586 decay->ceil_npages = 0; in arena_decay_init()
588 if (malloc_mutex_init(&decay->mtx, "decay", WITNESS_RANK_DECAY, in arena_decay_init()
592 decay->purging = false; in arena_decay_init()
593 arena_decay_reinit(decay, decay_ms); in arena_decay_init()
596 decay->stats = stats; in arena_decay_init()
614 arena_maybe_decay(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay, in arena_maybe_decay() argument
616 malloc_mutex_assert_owner(tsdn, &decay->mtx); in arena_maybe_decay()
619 ssize_t decay_ms = arena_decay_ms_read(decay); in arena_maybe_decay()
622 arena_decay_to_limit(tsdn, arena, decay, extents, false, in arena_maybe_decay()
632 if (unlikely(!nstime_monotonic() && nstime_compare(&decay->epoch, &time) in arena_maybe_decay()
643 nstime_copy(&decay->epoch, &time); in arena_maybe_decay()
644 arena_decay_deadline_init(decay); in arena_maybe_decay()
647 assert(nstime_compare(&decay->epoch, &time) <= 0); in arena_maybe_decay()
657 bool advance_epoch = arena_decay_deadline_reached(decay, &time); in arena_maybe_decay()
659 arena_decay_epoch_advance(tsdn, arena, decay, extents, &time, in arena_maybe_decay()
662 arena_decay_try_purge(tsdn, arena, decay, extents, in arena_maybe_decay()
664 arena_decay_backlog_npages_limit(decay), in arena_maybe_decay()
672 arena_decay_ms_get(arena_decay_t *decay) { in arena_decay_ms_get() argument
673 return arena_decay_ms_read(decay); in arena_decay_ms_get()
687 arena_decay_ms_set(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay, in arena_decay_ms_set() argument
693 malloc_mutex_lock(tsdn, &decay->mtx); in arena_decay_ms_set()
702 arena_decay_reinit(decay, decay_ms); in arena_decay_ms_set()
703 arena_maybe_decay(tsdn, arena, decay, extents, false); in arena_decay_ms_set()
704 malloc_mutex_unlock(tsdn, &decay->mtx); in arena_decay_ms_set()
744 extent_hooks_t **r_extent_hooks, arena_decay_t *decay, extents_t *extents, in arena_decay_stashed() argument
794 arena_stats_add_u64(tsdn, &arena->stats, &decay->stats->npurge, in arena_decay_stashed()
797 &decay->stats->nmadvise, nmadvise); in arena_decay_stashed()
798 arena_stats_add_u64(tsdn, &arena->stats, &decay->stats->purged, in arena_decay_stashed()
816 arena_decay_to_limit(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay, in arena_decay_to_limit() argument
821 malloc_mutex_assert_owner(tsdn, &decay->mtx); in arena_decay_to_limit()
823 if (decay->purging) { in arena_decay_to_limit()
826 decay->purging = true; in arena_decay_to_limit()
827 malloc_mutex_unlock(tsdn, &decay->mtx); in arena_decay_to_limit()
838 &extent_hooks, decay, extents, all, &decay_extents, in arena_decay_to_limit()
843 malloc_mutex_lock(tsdn, &decay->mtx); in arena_decay_to_limit()
844 decay->purging = false; in arena_decay_to_limit()
848 arena_decay_impl(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay, in arena_decay_impl() argument
851 malloc_mutex_lock(tsdn, &decay->mtx); in arena_decay_impl()
852 arena_decay_to_limit(tsdn, arena, decay, extents, all, 0, in arena_decay_impl()
854 malloc_mutex_unlock(tsdn, &decay->mtx); in arena_decay_impl()
859 if (malloc_mutex_trylock(tsdn, &decay->mtx)) { in arena_decay_impl()
864 bool epoch_advanced = arena_maybe_decay(tsdn, arena, decay, extents, in arena_decay_impl()
869 npages_new = decay->backlog[SMOOTHSTEP_NSTEPS-1]; in arena_decay_impl()
871 malloc_mutex_unlock(tsdn, &decay->mtx); in arena_decay_impl()
875 background_thread_interval_check(tsdn, arena, decay, in arena_decay_impl()