41 static __always_inline void __update_lru_size(struct lruvec *lruvec,  in __update_lru_size()  argument
45 	struct pglist_data *pgdat = lruvec_pgdat(lruvec);  in __update_lru_size()
47 	lockdep_assert_held(&lruvec->lru_lock);  in __update_lru_size()
50 	__mod_lruvec_state(lruvec, NR_LRU_BASE + lru, nr_pages);  in __update_lru_size()
55 static __always_inline void update_lru_size(struct lruvec *lruvec,  in update_lru_size()  argument
59 	__update_lru_size(lruvec, lru, zid, nr_pages);  in update_lru_size()
61 	mem_cgroup_update_lru_size(lruvec, lru, zid, nr_pages);  in update_lru_size()
167 static inline bool lru_gen_is_active(struct lruvec *lruvec, int gen)  in lru_gen_is_active()  argument
169 	unsigned long max_seq = lruvec->lrugen.max_seq;  in lru_gen_is_active()
177 static inline void lru_gen_update_size(struct lruvec *lruvec, struct folio *folio,  in lru_gen_update_size()  argument
184 	struct lru_gen_folio *lrugen = &lruvec->lrugen;  in lru_gen_update_size()
199 		if (lru_gen_is_active(lruvec, new_gen))  in lru_gen_update_size()
201 		__update_lru_size(lruvec, lru, zone, delta);  in lru_gen_update_size()
207 		if (lru_gen_is_active(lruvec, old_gen))  in lru_gen_update_size()
209 		__update_lru_size(lruvec, lru, zone, -delta);  in lru_gen_update_size()
214 	if (!lru_gen_is_active(lruvec, old_gen) && lru_gen_is_active(lruvec, new_gen)) {  in lru_gen_update_size()
215 		__update_lru_size(lruvec, lru, zone, -delta);  in lru_gen_update_size()
216 		__update_lru_size(lruvec, lru + LRU_ACTIVE, zone, delta);  in lru_gen_update_size()
220 	VM_WARN_ON_ONCE(lru_gen_is_active(lruvec, old_gen) && !lru_gen_is_active(lruvec, new_gen));  in lru_gen_update_size()
223 static inline unsigned long lru_gen_folio_seq(struct lruvec *lruvec, struct folio *folio,  in lru_gen_folio_seq()  argument
228 	struct lru_gen_folio *lrugen = &lruvec->lrugen;  in lru_gen_folio_seq()
256 static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming)  in lru_gen_add_folio()  argument
263 	struct lru_gen_folio *lrugen = &lruvec->lrugen;  in lru_gen_add_folio()
270 	seq = lru_gen_folio_seq(lruvec, folio, reclaiming);  in lru_gen_add_folio()
276 	lru_gen_update_size(lruvec, folio, -1, gen);  in lru_gen_add_folio()
286 static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming)  in lru_gen_del_folio()  argument
298 	flags = !reclaiming && lru_gen_is_active(lruvec, gen) ? BIT(PG_active) : 0;  in lru_gen_del_folio()
302 	lru_gen_update_size(lruvec, folio, gen, -1);  in lru_gen_del_folio()
326 static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming)  in lru_gen_add_folio()  argument
331 static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming)  in lru_gen_del_folio()  argument
343 void lruvec_add_folio(struct lruvec *lruvec, struct folio *folio)  in lruvec_add_folio()  argument
348 	trace_android_vh_lruvec_add_folio(lruvec, folio, lru, false, &skip);  in lruvec_add_folio()
352 	if (lru_gen_add_folio(lruvec, folio, false))  in lruvec_add_folio()
355 	update_lru_size(lruvec, lru, folio_zonenum(folio),  in lruvec_add_folio()
358 		list_add(&folio->lru, &lruvec->lists[lru]);  in lruvec_add_folio()
362 void lruvec_add_folio_tail(struct lruvec *lruvec, struct folio *folio)  in lruvec_add_folio_tail()  argument
367 	trace_android_vh_lruvec_add_folio(lruvec, folio, lru, true, &skip);  in lruvec_add_folio_tail()
371 	if (lru_gen_add_folio(lruvec, folio, true))  in lruvec_add_folio_tail()
374 	update_lru_size(lruvec, lru, folio_zonenum(folio),  in lruvec_add_folio_tail()
377 	list_add_tail(&folio->lru, &lruvec->lists[lru]);  in lruvec_add_folio_tail()
381 void lruvec_del_folio(struct lruvec *lruvec, struct folio *folio)  in lruvec_del_folio()  argument
386 	trace_android_vh_lruvec_del_folio(lruvec, folio, lru, &skip);  in lruvec_del_folio()
390 	if (lru_gen_del_folio(lruvec, folio, false))  in lruvec_del_folio()
395 	update_lru_size(lruvec, lru, folio_zonenum(folio),  in lruvec_del_folio()