372 static inline bool folio_memcg_kmem(struct folio *folio);
379 int mem_cgroup_move_account(struct folio *folio,
406 static inline struct mem_cgroup *__folio_memcg(struct folio *folio)  in __folio_memcg()  argument
408 	unsigned long memcg_data = folio->memcg_data;  in __folio_memcg()
410 	VM_BUG_ON_FOLIO(folio_test_slab(folio), folio);  in __folio_memcg()
411 	VM_BUG_ON_FOLIO(memcg_data & MEMCG_DATA_OBJEXTS, folio);  in __folio_memcg()
412 	VM_BUG_ON_FOLIO(memcg_data & MEMCG_DATA_KMEM, folio);  in __folio_memcg()
427 static inline struct obj_cgroup *__folio_objcg(struct folio *folio)  in __folio_objcg()  argument
429 	unsigned long memcg_data = folio->memcg_data;  in __folio_objcg()
431 	VM_BUG_ON_FOLIO(folio_test_slab(folio), folio);  in __folio_objcg()
432 	VM_BUG_ON_FOLIO(memcg_data & MEMCG_DATA_OBJEXTS, folio);  in __folio_objcg()
433 	VM_BUG_ON_FOLIO(!(memcg_data & MEMCG_DATA_KMEM), folio);  in __folio_objcg()
459 static inline struct mem_cgroup *folio_memcg(struct folio *folio)  in folio_memcg()  argument
461 	if (folio_memcg_kmem(folio))  in folio_memcg()
462 		return obj_cgroup_memcg(__folio_objcg(folio));  in folio_memcg()
463 	return __folio_memcg(folio);  in folio_memcg()
472 static inline bool folio_memcg_charged(struct folio *folio)  in folio_memcg_charged()  argument
474 	if (folio_memcg_kmem(folio))  in folio_memcg_charged()
475 		return __folio_objcg(folio) != NULL;  in folio_memcg_charged()
476 	return __folio_memcg(folio) != NULL;  in folio_memcg_charged()
490 static inline struct mem_cgroup *folio_memcg_rcu(struct folio *folio)  in folio_memcg_rcu()  argument
492 	unsigned long memcg_data = READ_ONCE(folio->memcg_data);  in folio_memcg_rcu()
494 	VM_BUG_ON_FOLIO(folio_test_slab(folio), folio);  in folio_memcg_rcu()
530 static inline struct mem_cgroup *folio_memcg_check(struct folio *folio)  in folio_memcg_check()  argument
536 	unsigned long memcg_data = READ_ONCE(folio->memcg_data);  in folio_memcg_check()
555 	return folio_memcg_check((struct folio *)page);  in page_memcg_check()
580 static inline bool folio_memcg_kmem(struct folio *folio)  in folio_memcg_kmem()  argument
582 	VM_BUG_ON_PGFLAGS(PageTail(&folio->page), &folio->page);  in folio_memcg_kmem()
583 	VM_BUG_ON_FOLIO(folio->memcg_data & MEMCG_DATA_OBJEXTS, folio);  in folio_memcg_kmem()
584 	return folio->memcg_data & MEMCG_DATA_KMEM;  in folio_memcg_kmem()
687 void mem_cgroup_commit_charge(struct folio *folio, struct mem_cgroup *memcg);
689 int __mem_cgroup_charge(struct folio *folio, struct mm_struct *mm, gfp_t gfp);
705 static inline int mem_cgroup_charge(struct folio *folio, struct mm_struct *mm,  in mem_cgroup_charge()  argument
710 	return __mem_cgroup_charge(folio, mm, gfp);  in mem_cgroup_charge()
716 int mem_cgroup_swapin_charge_folio(struct folio *folio, struct mm_struct *mm,
721 void __mem_cgroup_uncharge(struct folio *folio);
729 static inline void mem_cgroup_uncharge(struct folio *folio)  in mem_cgroup_uncharge()  argument
733 	__mem_cgroup_uncharge(folio);  in mem_cgroup_uncharge()
745 void mem_cgroup_replace_folio(struct folio *old, struct folio *new);
746 void mem_cgroup_migrate(struct folio *old, struct folio *new);
790 static inline struct lruvec *folio_lruvec(struct folio *folio)  in folio_lruvec()  argument
792 	struct mem_cgroup *memcg = folio_memcg(folio);  in folio_lruvec()
794 	VM_WARN_ON_ONCE_FOLIO(!memcg && !mem_cgroup_disabled(), folio);  in folio_lruvec()
795 	return mem_cgroup_lruvec(memcg, folio_pgdat(folio));  in folio_lruvec()
804 struct mem_cgroup *get_mem_cgroup_from_folio(struct folio *folio);
806 struct lruvec *folio_lruvec_lock(struct folio *folio);
807 struct lruvec *folio_lruvec_lock_irq(struct folio *folio);
808 struct lruvec *folio_lruvec_lock_irqsave(struct folio *folio,
812 void lruvec_memcg_debug(struct lruvec *lruvec, struct folio *folio);
815 void lruvec_memcg_debug(struct lruvec *lruvec, struct folio *folio)  in lruvec_memcg_debug()  argument
940 struct cgroup_subsys_state *mem_cgroup_css_from_folio(struct folio *folio);
1041 static inline void count_memcg_folio_events(struct folio *folio,  in count_memcg_folio_events()  argument
1044 	struct mem_cgroup *memcg = folio_memcg(folio);  in count_memcg_folio_events()
1118 static inline struct mem_cgroup *folio_memcg(struct folio *folio)  in folio_memcg()  argument
1123 static inline struct mem_cgroup *folio_memcg_rcu(struct folio *folio)  in folio_memcg_rcu()  argument
1129 static inline struct mem_cgroup *folio_memcg_check(struct folio *folio)  in folio_memcg_check()  argument
1144 static inline bool folio_memcg_kmem(struct folio *folio)  in folio_memcg_kmem()  argument
1154 static inline int mem_cgroup_move_account(struct folio *folio,  in mem_cgroup_move_account()  argument
1212 static inline void mem_cgroup_commit_charge(struct folio *folio,  in mem_cgroup_commit_charge()  argument
1217 static inline int mem_cgroup_charge(struct folio *folio,  in mem_cgroup_charge()  argument
1229 static inline int mem_cgroup_swapin_charge_folio(struct folio *folio,  in mem_cgroup_swapin_charge_folio()  argument
1239 static inline void mem_cgroup_uncharge(struct folio *folio)  in mem_cgroup_uncharge()  argument
1252 static inline void mem_cgroup_replace_folio(struct folio *old,  in mem_cgroup_replace_folio()
1253 		struct folio *new)  in mem_cgroup_replace_folio()
1257 static inline void mem_cgroup_migrate(struct folio *old, struct folio *new)  in mem_cgroup_migrate()
1267 static inline struct lruvec *folio_lruvec(struct folio *folio)  in folio_lruvec()  argument
1269 	struct pglist_data *pgdat = folio_pgdat(folio);  in folio_lruvec()
1274 void lruvec_memcg_debug(struct lruvec *lruvec, struct folio *folio)  in lruvec_memcg_debug()  argument
1299 static inline struct mem_cgroup *get_mem_cgroup_from_folio(struct folio *folio)  in get_mem_cgroup_from_folio()  argument
1328 static inline struct lruvec *folio_lruvec_lock(struct folio *folio)  in folio_lruvec_lock()  argument
1330 	struct pglist_data *pgdat = folio_pgdat(folio);  in folio_lruvec_lock()
1336 static inline struct lruvec *folio_lruvec_lock_irq(struct folio *folio)  in folio_lruvec_lock_irq()  argument
1338 	struct pglist_data *pgdat = folio_pgdat(folio);  in folio_lruvec_lock_irq()
1344 static inline struct lruvec *folio_lruvec_lock_irqsave(struct folio *folio,  in folio_lruvec_lock_irqsave()  argument
1347 	struct pglist_data *pgdat = folio_pgdat(folio);  in folio_lruvec_lock_irqsave()
1521 static inline void count_memcg_folio_events(struct folio *folio,  in count_memcg_folio_events()  argument
1596 static inline bool folio_matches_lruvec(struct folio *folio,  in folio_matches_lruvec()  argument
1599 	return lruvec_pgdat(lruvec) == folio_pgdat(folio) &&  in folio_matches_lruvec()
1600 	       lruvec_memcg(lruvec) == folio_memcg(folio);  in folio_matches_lruvec()
1604 static inline struct lruvec *folio_lruvec_relock_irq(struct folio *folio,  in folio_lruvec_relock_irq()  argument
1608 		if (folio_matches_lruvec(folio, locked_lruvec))  in folio_lruvec_relock_irq()
1614 	return folio_lruvec_lock_irq(folio);  in folio_lruvec_relock_irq()
1618 static inline void folio_lruvec_relock_irqsave(struct folio *folio,  in folio_lruvec_relock_irqsave()  argument
1622 		if (folio_matches_lruvec(folio, *lruvecp))  in folio_lruvec_relock_irqsave()
1628 	*lruvecp = folio_lruvec_lock_irqsave(folio, flags);  in folio_lruvec_relock_irqsave()
1638 void mem_cgroup_track_foreign_dirty_slowpath(struct folio *folio,
1641 static inline void mem_cgroup_track_foreign_dirty(struct folio *folio,  in mem_cgroup_track_foreign_dirty()  argument
1649 	memcg = folio_memcg(folio);  in mem_cgroup_track_foreign_dirty()
1651 		mem_cgroup_track_foreign_dirty_slowpath(folio, wb);  in mem_cgroup_track_foreign_dirty()
1671 static inline void mem_cgroup_track_foreign_dirty(struct folio *folio,  in mem_cgroup_track_foreign_dirty()  argument
1737 struct obj_cgroup *get_obj_cgroup_from_folio(struct folio *folio);
1831 static inline struct obj_cgroup *get_obj_cgroup_from_folio(struct folio *folio)  in get_obj_cgroup_from_folio()  argument
1904 void folio_memcg_lock(struct folio *folio);
1905 void folio_memcg_unlock(struct folio *folio);
1945 static inline void folio_memcg_lock(struct folio *folio)  in folio_memcg_lock()  argument
1949 static inline void folio_memcg_unlock(struct folio *folio)  in folio_memcg_unlock()  argument