Lines Matching refs:lrugen
3371 unsigned long max_seq = READ_ONCE((lruvec)->lrugen.max_seq)
3375 READ_ONCE((lruvec)->lrugen.min_seq[LRU_GEN_ANON]), \
3376 READ_ONCE((lruvec)->lrugen.min_seq[LRU_GEN_FILE]), \
3428 return lruvec->lrugen.max_seq - lruvec->lrugen.min_seq[type] + 1; in get_nr_gens()
3818 struct lru_gen_folio *lrugen = &lruvec->lrugen; in read_ctrl_pos() local
3819 int hist = lru_hist_from_seq(lrugen->min_seq[type]); in read_ctrl_pos()
3821 pos->refaulted = lrugen->avg_refaulted[type][tier] + in read_ctrl_pos()
3822 atomic_long_read(&lrugen->refaulted[hist][type][tier]); in read_ctrl_pos()
3823 pos->total = lrugen->avg_total[type][tier] + in read_ctrl_pos()
3824 atomic_long_read(&lrugen->evicted[hist][type][tier]); in read_ctrl_pos()
3826 pos->total += lrugen->protected[hist][type][tier - 1]; in read_ctrl_pos()
3833 struct lru_gen_folio *lrugen = &lruvec->lrugen; in reset_ctrl_pos() local
3835 unsigned long seq = carryover ? lrugen->min_seq[type] : lrugen->max_seq + 1; in reset_ctrl_pos()
3848 sum = lrugen->avg_refaulted[type][tier] + in reset_ctrl_pos()
3849 atomic_long_read(&lrugen->refaulted[hist][type][tier]); in reset_ctrl_pos()
3850 WRITE_ONCE(lrugen->avg_refaulted[type][tier], sum / 2); in reset_ctrl_pos()
3852 sum = lrugen->avg_total[type][tier] + in reset_ctrl_pos()
3853 atomic_long_read(&lrugen->evicted[hist][type][tier]); in reset_ctrl_pos()
3855 sum += lrugen->protected[hist][type][tier - 1]; in reset_ctrl_pos()
3856 WRITE_ONCE(lrugen->avg_total[type][tier], sum / 2); in reset_ctrl_pos()
3860 atomic_long_set(&lrugen->refaulted[hist][type][tier], 0); in reset_ctrl_pos()
3861 atomic_long_set(&lrugen->evicted[hist][type][tier], 0); in reset_ctrl_pos()
3863 WRITE_ONCE(lrugen->protected[hist][type][tier - 1], 0); in reset_ctrl_pos()
3910 struct lru_gen_folio *lrugen = &lruvec->lrugen; in folio_inc_gen() local
3911 int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]); in folio_inc_gen()
3955 struct lru_gen_folio *lrugen = &lruvec->lrugen; in reset_batch_size() local
3967 WRITE_ONCE(lrugen->nr_pages[gen][type][zone], in reset_batch_size()
3968 lrugen->nr_pages[gen][type][zone] + delta); in reset_batch_size()
4474 struct lru_gen_folio *lrugen = &lruvec->lrugen; in inc_min_seq() local
4475 int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]); in inc_min_seq()
4482 struct list_head *head = &lrugen->folios[old_gen][type][zone]; in inc_min_seq()
4493 list_move_tail(&folio->lru, &lrugen->folios[new_gen][type][zone]); in inc_min_seq()
4501 WRITE_ONCE(lrugen->min_seq[type], lrugen->min_seq[type] + 1); in inc_min_seq()
4510 struct lru_gen_folio *lrugen = &lruvec->lrugen; in try_to_inc_min_seq() local
4517 while (min_seq[type] + MIN_NR_GENS <= lrugen->max_seq) { in try_to_inc_min_seq()
4521 if (!list_empty(&lrugen->folios[gen][type][zone])) in try_to_inc_min_seq()
4534 min_seq[LRU_GEN_FILE] = max(min_seq[LRU_GEN_ANON], lrugen->min_seq[LRU_GEN_FILE]); in try_to_inc_min_seq()
4538 if (min_seq[type] == lrugen->min_seq[type]) in try_to_inc_min_seq()
4542 WRITE_ONCE(lrugen->min_seq[type], min_seq[type]); in try_to_inc_min_seq()
4553 struct lru_gen_folio *lrugen = &lruvec->lrugen; in inc_max_seq() local
4579 prev = lru_gen_from_seq(lrugen->max_seq - 1); in inc_max_seq()
4580 next = lru_gen_from_seq(lrugen->max_seq + 1); in inc_max_seq()
4585 long delta = lrugen->nr_pages[prev][type][zone] - in inc_max_seq()
4586 lrugen->nr_pages[next][type][zone]; in inc_max_seq()
4599 WRITE_ONCE(lrugen->timestamps[next], jiffies); in inc_max_seq()
4601 smp_store_release(&lrugen->max_seq, lrugen->max_seq + 1); in inc_max_seq()
4612 struct lru_gen_folio *lrugen = &lruvec->lrugen; in try_to_inc_max_seq() local
4614 VM_WARN_ON_ONCE(max_seq > READ_ONCE(lrugen->max_seq)); in try_to_inc_max_seq()
4691 struct lru_gen_folio *lrugen = &lruvec->lrugen; in lruvec_is_sizable() local
4703 total += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L); in lruvec_is_sizable()
4727 birth = READ_ONCE(lruvec->lrugen.timestamps[gen]); in lruvec_is_reclaimable()
4900 return READ_ONCE(lruvec->lrugen.seg); in lru_gen_memcg_seg()
4913 VM_WARN_ON_ONCE(hlist_nulls_unhashed(&lruvec->lrugen.list)); in lru_gen_rotate_memcg()
4916 new = old = lruvec->lrugen.gen; in lru_gen_rotate_memcg()
4930 WRITE_ONCE(lruvec->lrugen.seg, seg); in lru_gen_rotate_memcg()
4931 WRITE_ONCE(lruvec->lrugen.gen, new); in lru_gen_rotate_memcg()
4933 hlist_nulls_del_rcu(&lruvec->lrugen.list); in lru_gen_rotate_memcg()
4936 hlist_nulls_add_head_rcu(&lruvec->lrugen.list, &pgdat->memcg_lru.fifo[new][bin]); in lru_gen_rotate_memcg()
4938 hlist_nulls_add_tail_rcu(&lruvec->lrugen.list, &pgdat->memcg_lru.fifo[new][bin]); in lru_gen_rotate_memcg()
4961 VM_WARN_ON_ONCE(!hlist_nulls_unhashed(&lruvec->lrugen.list)); in lru_gen_online_memcg()
4965 lruvec->lrugen.gen = gen; in lru_gen_online_memcg()
4967 hlist_nulls_add_tail_rcu(&lruvec->lrugen.list, &pgdat->memcg_lru.fifo[gen][bin]); in lru_gen_online_memcg()
4996 if (hlist_nulls_unhashed(&lruvec->lrugen.list)) in lru_gen_release_memcg()
4999 gen = lruvec->lrugen.gen; in lru_gen_release_memcg()
5001 hlist_nulls_del_init_rcu(&lruvec->lrugen.list); in lru_gen_release_memcg()
5043 struct lru_gen_folio *lrugen = &lruvec->lrugen; in sort_folio() local
5067 if (gen != lru_gen_from_seq(lrugen->min_seq[type])) { in sort_folio()
5068 list_move(&folio->lru, &lrugen->folios[gen][type][zone]); in sort_folio()
5074 int hist = lru_hist_from_seq(lrugen->min_seq[type]); in sort_folio()
5077 list_move_tail(&folio->lru, &lrugen->folios[gen][type][zone]); in sort_folio()
5079 WRITE_ONCE(lrugen->protected[hist][type][tier - 1], in sort_folio()
5080 lrugen->protected[hist][type][tier - 1] + delta); in sort_folio()
5087 list_move_tail(&folio->lru, &lrugen->folios[gen][type][zone]); in sort_folio()
5095 list_move(&folio->lru, &lrugen->folios[gen][type][zone]); in sort_folio()
5146 struct lru_gen_folio *lrugen = &lruvec->lrugen; in scan_folios() local
5154 gen = lru_gen_from_seq(lrugen->min_seq[type]); in scan_folios()
5160 struct list_head *head = &lrugen->folios[gen][type][zone]; in scan_folios()
5400 struct lru_gen_folio *lrugen = &lruvec->lrugen; in should_run_aging() local
5419 size += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L); in should_run_aging()
5601 struct lru_gen_folio *lrugen; in shrink_many() local
5613 hlist_nulls_for_each_entry_rcu(lrugen, pos, &pgdat->memcg_lru.fifo[gen][bin], list) { in shrink_many()
5622 if (gen != READ_ONCE(lrugen->gen)) in shrink_many()
5625 lruvec = container_of(lrugen, struct lruvec, lrugen); in shrink_many()
5747 struct lru_gen_folio *lrugen = &lruvec->lrugen; in state_is_valid() local
5749 if (lrugen->enabled) { in state_is_valid()
5760 if (!list_empty(&lrugen->folios[gen][type][zone])) in state_is_valid()
5805 struct list_head *head = &lruvec->lrugen.folios[gen][type][zone]; in drain_evictable()
5859 lruvec->lrugen.enabled = enabled; in lru_gen_change_state()
6022 struct lru_gen_folio *lrugen = &lruvec->lrugen; in lru_gen_seq_show_full() local
6032 n[0] = READ_ONCE(lrugen->avg_refaulted[type][tier]); in lru_gen_seq_show_full()
6033 n[1] = READ_ONCE(lrugen->avg_total[type][tier]); in lru_gen_seq_show_full()
6036 n[0] = atomic_long_read(&lrugen->refaulted[hist][type][tier]); in lru_gen_seq_show_full()
6037 n[1] = atomic_long_read(&lrugen->evicted[hist][type][tier]); in lru_gen_seq_show_full()
6039 n[2] = READ_ONCE(lrugen->protected[hist][type][tier - 1]); in lru_gen_seq_show_full()
6072 struct lru_gen_folio *lrugen = &lruvec->lrugen; in lru_gen_seq_show() local
6100 unsigned long birth = READ_ONCE(lruvec->lrugen.timestamps[gen]); in lru_gen_seq_show()
6109 size += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L); in lru_gen_seq_show()
6326 struct lru_gen_folio *lrugen = &lruvec->lrugen; in lru_gen_init_lruvec() local
6328 lrugen->max_seq = MIN_NR_GENS + 1; in lru_gen_init_lruvec()
6329 lrugen->enabled = lru_gen_enabled(); in lru_gen_init_lruvec()
6332 lrugen->timestamps[i] = jiffies; in lru_gen_init_lruvec()
6335 INIT_LIST_HEAD(&lrugen->folios[gen][type][zone]); in lru_gen_init_lruvec()
6370 VM_WARN_ON_ONCE(memchr_inv(lruvec->lrugen.nr_pages, 0, in lru_gen_exit_memcg()
6371 sizeof(lruvec->lrugen.nr_pages))); in lru_gen_exit_memcg()
6373 lruvec->lrugen.list.next = LIST_POISON1; in lru_gen_exit_memcg()