Lines Matching refs:lru
21 static void list_lru_register(struct list_lru *lru) in list_lru_register() argument
24 list_add(&lru->list, &list_lrus); in list_lru_register()
28 static void list_lru_unregister(struct list_lru *lru) in list_lru_unregister() argument
31 list_del(&lru->list); in list_lru_unregister()
35 static int lru_shrinker_id(struct list_lru *lru) in lru_shrinker_id() argument
37 return lru->shrinker_id; in lru_shrinker_id()
40 static inline bool list_lru_memcg_aware(struct list_lru *lru) in list_lru_memcg_aware() argument
42 return lru->memcg_aware; in list_lru_memcg_aware()
56 return memcg_lrus->lru[idx]; in list_lru_from_memcg_idx()
57 return &nlru->lru; in list_lru_from_memcg_idx()
74 struct list_lru_one *l = &nlru->lru; in list_lru_from_kmem()
91 static void list_lru_register(struct list_lru *lru) in list_lru_register() argument
95 static void list_lru_unregister(struct list_lru *lru) in list_lru_unregister() argument
99 static int lru_shrinker_id(struct list_lru *lru) in lru_shrinker_id() argument
104 static inline bool list_lru_memcg_aware(struct list_lru *lru) in list_lru_memcg_aware() argument
112 return &nlru->lru; in list_lru_from_memcg_idx()
121 return &nlru->lru; in list_lru_from_kmem()
125 bool list_lru_add(struct list_lru *lru, struct list_head *item) in list_lru_add() argument
128 struct list_lru_node *nlru = &lru->node[nid]; in list_lru_add()
139 lru_shrinker_id(lru)); in list_lru_add()
149 bool list_lru_del(struct list_lru *lru, struct list_head *item) in list_lru_del() argument
152 struct list_lru_node *nlru = &lru->node[nid]; in list_lru_del()
184 unsigned long list_lru_count_one(struct list_lru *lru, in list_lru_count_one() argument
187 struct list_lru_node *nlru = &lru->node[nid]; in list_lru_count_one()
200 unsigned long list_lru_count_node(struct list_lru *lru, int nid) in list_lru_count_node() argument
204 nlru = &lru->node[nid]; in list_lru_count_node()
268 list_lru_walk_one(struct list_lru *lru, int nid, struct mem_cgroup *memcg, in list_lru_walk_one() argument
272 struct list_lru_node *nlru = &lru->node[nid]; in list_lru_walk_one()
284 list_lru_walk_one_irq(struct list_lru *lru, int nid, struct mem_cgroup *memcg, in list_lru_walk_one_irq() argument
288 struct list_lru_node *nlru = &lru->node[nid]; in list_lru_walk_one_irq()
298 unsigned long list_lru_walk_node(struct list_lru *lru, int nid, in list_lru_walk_node() argument
305 isolated += list_lru_walk_one(lru, nid, NULL, isolate, cb_arg, in list_lru_walk_node()
307 if (*nr_to_walk > 0 && list_lru_memcg_aware(lru)) { in list_lru_walk_node()
309 struct list_lru_node *nlru = &lru->node[nid]; in list_lru_walk_node()
338 kfree(memcg_lrus->lru[i]); in __memcg_destroy_list_lru_node()
354 memcg_lrus->lru[i] = l; in __memcg_init_list_lru_node()
419 memcpy(&new->lru, &old->lru, old_size * sizeof(void *)); in memcg_update_list_lru_node()
448 static int memcg_init_list_lru(struct list_lru *lru, bool memcg_aware) in memcg_init_list_lru() argument
452 lru->memcg_aware = memcg_aware; in memcg_init_list_lru()
458 if (memcg_init_list_lru_node(&lru->node[i])) in memcg_init_list_lru()
464 if (!lru->node[i].memcg_lrus) in memcg_init_list_lru()
466 memcg_destroy_list_lru_node(&lru->node[i]); in memcg_init_list_lru()
471 static void memcg_destroy_list_lru(struct list_lru *lru) in memcg_destroy_list_lru() argument
475 if (!list_lru_memcg_aware(lru)) in memcg_destroy_list_lru()
479 memcg_destroy_list_lru_node(&lru->node[i]); in memcg_destroy_list_lru()
482 static int memcg_update_list_lru(struct list_lru *lru, in memcg_update_list_lru() argument
487 if (!list_lru_memcg_aware(lru)) in memcg_update_list_lru()
491 if (memcg_update_list_lru_node(&lru->node[i], in memcg_update_list_lru()
498 if (!lru->node[i].memcg_lrus) in memcg_update_list_lru()
501 memcg_cancel_update_list_lru_node(&lru->node[i], in memcg_update_list_lru()
507 static void memcg_cancel_update_list_lru(struct list_lru *lru, in memcg_cancel_update_list_lru() argument
512 if (!list_lru_memcg_aware(lru)) in memcg_cancel_update_list_lru()
516 memcg_cancel_update_list_lru_node(&lru->node[i], in memcg_cancel_update_list_lru()
523 struct list_lru *lru; in memcg_update_all_list_lrus() local
527 list_for_each_entry(lru, &list_lrus, list) { in memcg_update_all_list_lrus()
528 ret = memcg_update_list_lru(lru, old_size, new_size); in memcg_update_all_list_lrus()
536 list_for_each_entry_continue_reverse(lru, &list_lrus, list) in memcg_update_all_list_lrus()
537 memcg_cancel_update_list_lru(lru, old_size, new_size); in memcg_update_all_list_lrus()
541 static void memcg_drain_list_lru_node(struct list_lru *lru, int nid, in memcg_drain_list_lru_node() argument
544 struct list_lru_node *nlru = &lru->node[nid]; in memcg_drain_list_lru_node()
561 memcg_set_shrinker_bit(dst_memcg, nid, lru_shrinker_id(lru)); in memcg_drain_list_lru_node()
568 static void memcg_drain_list_lru(struct list_lru *lru, in memcg_drain_list_lru() argument
573 if (!list_lru_memcg_aware(lru)) in memcg_drain_list_lru()
577 memcg_drain_list_lru_node(lru, i, src_idx, dst_memcg); in memcg_drain_list_lru()
582 struct list_lru *lru; in memcg_drain_all_list_lrus() local
585 list_for_each_entry(lru, &list_lrus, list) in memcg_drain_all_list_lrus()
586 memcg_drain_list_lru(lru, src_idx, dst_memcg); in memcg_drain_all_list_lrus()
590 static int memcg_init_list_lru(struct list_lru *lru, bool memcg_aware) in memcg_init_list_lru() argument
595 static void memcg_destroy_list_lru(struct list_lru *lru) in memcg_destroy_list_lru() argument
600 int __list_lru_init(struct list_lru *lru, bool memcg_aware, in __list_lru_init() argument
608 lru->shrinker_id = shrinker->id; in __list_lru_init()
610 lru->shrinker_id = -1; in __list_lru_init()
614 lru->node = kcalloc(nr_node_ids, sizeof(*lru->node), GFP_KERNEL); in __list_lru_init()
615 if (!lru->node) in __list_lru_init()
619 spin_lock_init(&lru->node[i].lock); in __list_lru_init()
621 lockdep_set_class(&lru->node[i].lock, key); in __list_lru_init()
622 init_one_lru(&lru->node[i].lru); in __list_lru_init()
625 err = memcg_init_list_lru(lru, memcg_aware); in __list_lru_init()
627 kfree(lru->node); in __list_lru_init()
629 lru->node = NULL; in __list_lru_init()
633 list_lru_register(lru); in __list_lru_init()
640 void list_lru_destroy(struct list_lru *lru) in list_lru_destroy() argument
643 if (!lru->node) in list_lru_destroy()
648 list_lru_unregister(lru); in list_lru_destroy()
650 memcg_destroy_list_lru(lru); in list_lru_destroy()
651 kfree(lru->node); in list_lru_destroy()
652 lru->node = NULL; in list_lru_destroy()
655 lru->shrinker_id = -1; in list_lru_destroy()