31 static inline int folio_is_file_lru(struct folio *folio)  in folio_is_file_lru()  argument
33 	return !folio_test_swapbacked(folio);  in folio_is_file_lru()
69 static __always_inline void __folio_clear_lru_flags(struct folio *folio)  in __folio_clear_lru_flags()  argument
71 	VM_BUG_ON_FOLIO(!folio_test_lru(folio), folio);  in __folio_clear_lru_flags()
73 	__folio_clear_lru(folio);  in __folio_clear_lru_flags()
76 	if (folio_test_active(folio) && folio_test_unevictable(folio))  in __folio_clear_lru_flags()
79 	__folio_clear_active(folio);  in __folio_clear_lru_flags()
80 	__folio_clear_unevictable(folio);  in __folio_clear_lru_flags()
90 static __always_inline enum lru_list folio_lru_list(struct folio *folio)  in folio_lru_list()  argument
94 	VM_BUG_ON_FOLIO(folio_test_active(folio) && folio_test_unevictable(folio), folio);  in folio_lru_list()
96 	if (folio_test_unevictable(folio))  in folio_lru_list()
99 	lru = folio_is_file_lru(folio) ? LRU_INACTIVE_FILE : LRU_INACTIVE_ANON;  in folio_lru_list()
100 	if (folio_test_active(folio))  in folio_lru_list()
147 static inline int folio_lru_refs(struct folio *folio)  in folio_lru_refs()  argument
149 	unsigned long flags = READ_ONCE(folio->flags);  in folio_lru_refs()
160 static inline int folio_lru_gen(struct folio *folio)  in folio_lru_gen()  argument
162 	unsigned long flags = READ_ONCE(folio->flags);  in folio_lru_gen()
177 static inline void lru_gen_update_size(struct lruvec *lruvec, struct folio *folio,  in lru_gen_update_size()  argument
180 	int type = folio_is_file_lru(folio);  in lru_gen_update_size()
181 	int zone = folio_zonenum(folio);  in lru_gen_update_size()
182 	int delta = folio_nr_pages(folio);  in lru_gen_update_size()
223 static inline unsigned long lru_gen_folio_seq(struct lruvec *lruvec, struct folio *folio,  in lru_gen_folio_seq()  argument
227 	int type = folio_is_file_lru(folio);  in lru_gen_folio_seq()
242 	if (folio_test_active(folio))  in lru_gen_folio_seq()
243 		gen = MIN_NR_GENS - folio_test_workingset(folio);  in lru_gen_folio_seq()
246 	else if ((!folio_is_file_lru(folio) && !folio_test_swapcache(folio)) ||  in lru_gen_folio_seq()
247 		 (folio_test_reclaim(folio) &&  in lru_gen_folio_seq()
248 		  (folio_test_dirty(folio) || folio_test_writeback(folio))))  in lru_gen_folio_seq()
251 		gen = MAX_NR_GENS - folio_test_workingset(folio);  in lru_gen_folio_seq()
256 static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming)  in lru_gen_add_folio()  argument
260 	int gen = folio_lru_gen(folio);  in lru_gen_add_folio()
261 	int type = folio_is_file_lru(folio);  in lru_gen_add_folio()
262 	int zone = folio_zonenum(folio);  in lru_gen_add_folio()
265 	VM_WARN_ON_ONCE_FOLIO(gen != -1, folio);  in lru_gen_add_folio()
267 	if (folio_test_unevictable(folio) || !lrugen->enabled)  in lru_gen_add_folio()
270 	seq = lru_gen_folio_seq(lruvec, folio, reclaiming);  in lru_gen_add_folio()
274 	set_mask_bits(&folio->flags, LRU_GEN_MASK | BIT(PG_active), flags);  in lru_gen_add_folio()
276 	lru_gen_update_size(lruvec, folio, -1, gen);  in lru_gen_add_folio()
279 		list_add_tail(&folio->lru, &lrugen->folios[gen][type][zone]);  in lru_gen_add_folio()
281 		list_add(&folio->lru, &lrugen->folios[gen][type][zone]);  in lru_gen_add_folio()
286 static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming)  in lru_gen_del_folio()  argument
289 	int gen = folio_lru_gen(folio);  in lru_gen_del_folio()
294 	VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio);  in lru_gen_del_folio()
295 	VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio);  in lru_gen_del_folio()
299 	flags = set_mask_bits(&folio->flags, LRU_GEN_MASK, flags);  in lru_gen_del_folio()
302 	lru_gen_update_size(lruvec, folio, gen, -1);  in lru_gen_del_folio()
303 	list_del(&folio->lru);  in lru_gen_del_folio()
308 static inline void folio_migrate_refs(struct folio *new, struct folio *old)  in folio_migrate_refs()
326 static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming)  in lru_gen_add_folio()  argument
331 static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming)  in lru_gen_del_folio()  argument
336 static inline void folio_migrate_refs(struct folio *new, struct folio *old)  in folio_migrate_refs()
343 void lruvec_add_folio(struct lruvec *lruvec, struct folio *folio)  in lruvec_add_folio()  argument
345 	enum lru_list lru = folio_lru_list(folio);  in lruvec_add_folio()
348 	trace_android_vh_lruvec_add_folio(lruvec, folio, lru, false, &skip);  in lruvec_add_folio()
352 	if (lru_gen_add_folio(lruvec, folio, false))  in lruvec_add_folio()
355 	update_lru_size(lruvec, lru, folio_zonenum(folio),  in lruvec_add_folio()
356 			folio_nr_pages(folio));  in lruvec_add_folio()
358 		list_add(&folio->lru, &lruvec->lists[lru]);  in lruvec_add_folio()
362 void lruvec_add_folio_tail(struct lruvec *lruvec, struct folio *folio)  in lruvec_add_folio_tail()  argument
364 	enum lru_list lru = folio_lru_list(folio);  in lruvec_add_folio_tail()
367 	trace_android_vh_lruvec_add_folio(lruvec, folio, lru, true, &skip);  in lruvec_add_folio_tail()
371 	if (lru_gen_add_folio(lruvec, folio, true))  in lruvec_add_folio_tail()
374 	update_lru_size(lruvec, lru, folio_zonenum(folio),  in lruvec_add_folio_tail()
375 			folio_nr_pages(folio));  in lruvec_add_folio_tail()
377 	list_add_tail(&folio->lru, &lruvec->lists[lru]);  in lruvec_add_folio_tail()
381 void lruvec_del_folio(struct lruvec *lruvec, struct folio *folio)  in lruvec_del_folio()  argument
383 	enum lru_list lru = folio_lru_list(folio);  in lruvec_del_folio()
386 	trace_android_vh_lruvec_del_folio(lruvec, folio, lru, &skip);  in lruvec_del_folio()
390 	if (lru_gen_del_folio(lruvec, folio, false))  in lruvec_del_folio()
394 		list_del(&folio->lru);  in lruvec_del_folio()
395 	update_lru_size(lruvec, lru, folio_zonenum(folio),  in lruvec_del_folio()
396 			-folio_nr_pages(folio));  in lruvec_del_folio()