Lines Matching refs:lru
21 static void list_lru_register(struct list_lru *lru) in list_lru_register() argument
24 list_add(&lru->list, &list_lrus); in list_lru_register()
28 static void list_lru_unregister(struct list_lru *lru) in list_lru_unregister() argument
31 list_del(&lru->list); in list_lru_unregister()
35 static int lru_shrinker_id(struct list_lru *lru) in lru_shrinker_id() argument
37 return lru->shrinker_id; in lru_shrinker_id()
40 static inline bool list_lru_memcg_aware(struct list_lru *lru) in list_lru_memcg_aware() argument
42 return lru->memcg_aware; in list_lru_memcg_aware()
56 return memcg_lrus->lru[idx]; in list_lru_from_memcg_idx()
57 return &nlru->lru; in list_lru_from_memcg_idx()
64 struct list_lru_one *l = &nlru->lru; in list_lru_from_kmem()
81 static void list_lru_register(struct list_lru *lru) in list_lru_register() argument
85 static void list_lru_unregister(struct list_lru *lru) in list_lru_unregister() argument
89 static int lru_shrinker_id(struct list_lru *lru) in lru_shrinker_id() argument
94 static inline bool list_lru_memcg_aware(struct list_lru *lru) in list_lru_memcg_aware() argument
102 return &nlru->lru; in list_lru_from_memcg_idx()
111 return &nlru->lru; in list_lru_from_kmem()
115 bool list_lru_add(struct list_lru *lru, struct list_head *item) in list_lru_add() argument
118 struct list_lru_node *nlru = &lru->node[nid]; in list_lru_add()
129 lru_shrinker_id(lru)); in list_lru_add()
139 bool list_lru_del(struct list_lru *lru, struct list_head *item) in list_lru_del() argument
142 struct list_lru_node *nlru = &lru->node[nid]; in list_lru_del()
174 unsigned long list_lru_count_one(struct list_lru *lru, in list_lru_count_one() argument
177 struct list_lru_node *nlru = &lru->node[nid]; in list_lru_count_one()
190 unsigned long list_lru_count_node(struct list_lru *lru, int nid) in list_lru_count_node() argument
194 nlru = &lru->node[nid]; in list_lru_count_node()
258 list_lru_walk_one(struct list_lru *lru, int nid, struct mem_cgroup *memcg, in list_lru_walk_one() argument
262 struct list_lru_node *nlru = &lru->node[nid]; in list_lru_walk_one()
274 list_lru_walk_one_irq(struct list_lru *lru, int nid, struct mem_cgroup *memcg, in list_lru_walk_one_irq() argument
278 struct list_lru_node *nlru = &lru->node[nid]; in list_lru_walk_one_irq()
288 unsigned long list_lru_walk_node(struct list_lru *lru, int nid, in list_lru_walk_node() argument
295 isolated += list_lru_walk_one(lru, nid, NULL, isolate, cb_arg, in list_lru_walk_node()
297 if (*nr_to_walk > 0 && list_lru_memcg_aware(lru)) { in list_lru_walk_node()
299 struct list_lru_node *nlru = &lru->node[nid]; in list_lru_walk_node()
328 kfree(memcg_lrus->lru[i]); in __memcg_destroy_list_lru_node()
344 memcg_lrus->lru[i] = l; in __memcg_init_list_lru_node()
401 memcpy(&new->lru, &old->lru, old_size * sizeof(void *)); in memcg_update_list_lru_node()
430 static int memcg_init_list_lru(struct list_lru *lru, bool memcg_aware) in memcg_init_list_lru() argument
434 lru->memcg_aware = memcg_aware; in memcg_init_list_lru()
440 if (memcg_init_list_lru_node(&lru->node[i])) in memcg_init_list_lru()
446 if (!lru->node[i].memcg_lrus) in memcg_init_list_lru()
448 memcg_destroy_list_lru_node(&lru->node[i]); in memcg_init_list_lru()
453 static void memcg_destroy_list_lru(struct list_lru *lru) in memcg_destroy_list_lru() argument
457 if (!list_lru_memcg_aware(lru)) in memcg_destroy_list_lru()
461 memcg_destroy_list_lru_node(&lru->node[i]); in memcg_destroy_list_lru()
464 static int memcg_update_list_lru(struct list_lru *lru, in memcg_update_list_lru() argument
469 if (!list_lru_memcg_aware(lru)) in memcg_update_list_lru()
473 if (memcg_update_list_lru_node(&lru->node[i], in memcg_update_list_lru()
480 if (!lru->node[i].memcg_lrus) in memcg_update_list_lru()
483 memcg_cancel_update_list_lru_node(&lru->node[i], in memcg_update_list_lru()
489 static void memcg_cancel_update_list_lru(struct list_lru *lru, in memcg_cancel_update_list_lru() argument
494 if (!list_lru_memcg_aware(lru)) in memcg_cancel_update_list_lru()
498 memcg_cancel_update_list_lru_node(&lru->node[i], in memcg_cancel_update_list_lru()
505 struct list_lru *lru; in memcg_update_all_list_lrus() local
509 list_for_each_entry(lru, &list_lrus, list) { in memcg_update_all_list_lrus()
510 ret = memcg_update_list_lru(lru, old_size, new_size); in memcg_update_all_list_lrus()
518 list_for_each_entry_continue_reverse(lru, &list_lrus, list) in memcg_update_all_list_lrus()
519 memcg_cancel_update_list_lru(lru, old_size, new_size); in memcg_update_all_list_lrus()
523 static void memcg_drain_list_lru_node(struct list_lru *lru, int nid, in memcg_drain_list_lru_node() argument
526 struct list_lru_node *nlru = &lru->node[nid]; in memcg_drain_list_lru_node()
543 set_shrinker_bit(dst_memcg, nid, lru_shrinker_id(lru)); in memcg_drain_list_lru_node()
550 static void memcg_drain_list_lru(struct list_lru *lru, in memcg_drain_list_lru() argument
555 if (!list_lru_memcg_aware(lru)) in memcg_drain_list_lru()
559 memcg_drain_list_lru_node(lru, i, src_idx, dst_memcg); in memcg_drain_list_lru()
564 struct list_lru *lru; in memcg_drain_all_list_lrus() local
567 list_for_each_entry(lru, &list_lrus, list) in memcg_drain_all_list_lrus()
568 memcg_drain_list_lru(lru, src_idx, dst_memcg); in memcg_drain_all_list_lrus()
572 static int memcg_init_list_lru(struct list_lru *lru, bool memcg_aware) in memcg_init_list_lru() argument
577 static void memcg_destroy_list_lru(struct list_lru *lru) in memcg_destroy_list_lru() argument
582 int __list_lru_init(struct list_lru *lru, bool memcg_aware, in __list_lru_init() argument
590 lru->shrinker_id = shrinker->id; in __list_lru_init()
592 lru->shrinker_id = -1; in __list_lru_init()
596 lru->node = kcalloc(nr_node_ids, sizeof(*lru->node), GFP_KERNEL); in __list_lru_init()
597 if (!lru->node) in __list_lru_init()
601 spin_lock_init(&lru->node[i].lock); in __list_lru_init()
603 lockdep_set_class(&lru->node[i].lock, key); in __list_lru_init()
604 init_one_lru(&lru->node[i].lru); in __list_lru_init()
607 err = memcg_init_list_lru(lru, memcg_aware); in __list_lru_init()
609 kfree(lru->node); in __list_lru_init()
611 lru->node = NULL; in __list_lru_init()
615 list_lru_register(lru); in __list_lru_init()
622 void list_lru_destroy(struct list_lru *lru) in list_lru_destroy() argument
625 if (!lru->node) in list_lru_destroy()
630 list_lru_unregister(lru); in list_lru_destroy()
632 memcg_destroy_list_lru(lru); in list_lru_destroy()
633 kfree(lru->node); in list_lru_destroy()
634 lru->node = NULL; in list_lru_destroy()
637 lru->shrinker_id = -1; in list_lru_destroy()