228 #define folio_page_idx(folio, p)	(page_to_pfn(p) - folio_pfn(folio))  argument
231 #define folio_page_idx(folio, p)	((p) - &(folio)->page)  argument
243 static inline struct folio *lru_to_folio(struct list_head *head)  in lru_to_folio()
245 	return list_entry((head)->prev, struct folio, lru);  in lru_to_folio()
1203 	struct folio *folio = (struct folio *)page;  in compound_order()  local
1205 	if (!test_bit(PG_head, &folio->flags))  in compound_order()
1207 	return folio->_flags_1 & 0xff;  in compound_order()
1219 static inline unsigned int folio_order(const struct folio *folio)  in folio_order()  argument
1221 	if (!folio_test_large(folio))  in folio_order()
1223 	return folio->_flags_1 & 0xff;  in folio_order()
1255 static inline int folio_put_testzero(struct folio *folio)  in folio_put_testzero()  argument
1257 	return put_page_testzero(&folio->page);  in folio_put_testzero()
1271 static inline struct folio *folio_get_nontail_page(struct page *page)  in folio_get_nontail_page()
1275 	return (struct folio *)page;  in folio_get_nontail_page()
1318 static inline int folio_entire_mapcount(const struct folio *folio)  in folio_entire_mapcount()  argument
1320 	VM_BUG_ON_FOLIO(!folio_test_large(folio), folio);  in folio_entire_mapcount()
1321 	return atomic_read(&folio->_entire_mapcount) + 1;  in folio_entire_mapcount()
1324 static inline int folio_large_mapcount(const struct folio *folio)  in folio_large_mapcount()  argument
1326 	VM_WARN_ON_FOLIO(!folio_test_large(folio), folio);  in folio_large_mapcount()
1327 	return atomic_read(&folio->_large_mapcount) + 1;  in folio_large_mapcount()
1350 static inline int folio_mapcount(const struct folio *folio)  in folio_mapcount()  argument
1354 	if (likely(!folio_test_large(folio))) {  in folio_mapcount()
1355 		mapcount = atomic_read(&folio->_mapcount) + 1;  in folio_mapcount()
1360 	return folio_large_mapcount(folio);  in folio_mapcount()
1369 static inline bool folio_mapped(const struct folio *folio)  in folio_mapped()  argument
1371 	return folio_mapcount(folio) >= 1;  in folio_mapped()
1391 static inline struct folio *virt_to_folio(const void *x)  in virt_to_folio()
1398 void __folio_put(struct folio *folio);
1403 void folio_copy(struct folio *dst, struct folio *src);
1404 int folio_mc_copy(struct folio *dst, struct folio *src);
1456 void set_pte_range(struct vm_fault *vmf, struct folio *folio,
1525 bool __put_devmap_managed_folio_refs(struct folio *folio, int refs);
1526 static inline bool put_devmap_managed_folio_refs(struct folio *folio, int refs)  in put_devmap_managed_folio_refs()  argument
1530 	if (!folio_is_zone_device(folio))  in put_devmap_managed_folio_refs()
1532 	return __put_devmap_managed_folio_refs(folio, refs);  in put_devmap_managed_folio_refs()
1535 static inline bool put_devmap_managed_folio_refs(struct folio *folio, int refs)  in put_devmap_managed_folio_refs()  argument
1542 #define folio_ref_zero_or_close_to_overflow(folio) \  argument
1543 	((unsigned int) folio_ref_count(folio) + 127u <= 127u)
1553 static inline void folio_get(struct folio *folio)  in folio_get()  argument
1555 	VM_BUG_ON_FOLIO(folio_ref_zero_or_close_to_overflow(folio), folio);  in folio_get()
1556 	folio_ref_inc(folio);  in folio_get()
1586 static inline void folio_put(struct folio *folio)  in folio_put()  argument
1588 	if (folio_put_testzero(folio))  in folio_put()
1589 		__folio_put(folio);  in folio_put()
1606 static inline void folio_put_refs(struct folio *folio, int refs)  in folio_put_refs()  argument
1608 	if (folio_ref_sub_and_test(folio, refs))  in folio_put_refs()
1609 		__folio_put(folio);  in folio_put_refs()
1627 	struct folio **folios;
1653 	struct folio *folio = page_folio(page);  in put_page()  local
1659 	if (put_devmap_managed_folio_refs(folio, 1))  in put_page()
1661 	folio_put(folio);  in put_page()
1697 void unpin_folio(struct folio *folio);
1703 void unpin_user_folio(struct folio *folio, unsigned long npages);
1704 void unpin_folios(struct folio **folios, unsigned long nfolios);
1752 static inline int folio_nid(const struct folio *folio)  in folio_nid()  argument
1754 	return page_to_nid(&folio->page);  in folio_nid()
1807 static inline int folio_xchg_last_cpupid(struct folio *folio, int cpupid)  in folio_xchg_last_cpupid()  argument
1809 	return xchg(&folio->_last_cpupid, cpupid & LAST_CPUPID_MASK);  in folio_xchg_last_cpupid()
1812 static inline int folio_last_cpupid(struct folio *folio)  in folio_last_cpupid()  argument
1814 	return folio->_last_cpupid;  in folio_last_cpupid()
1821 static inline int folio_last_cpupid(struct folio *folio)  in folio_last_cpupid()  argument
1823 	return (folio->flags >> LAST_CPUPID_PGSHIFT) & LAST_CPUPID_MASK;  in folio_last_cpupid()
1826 int folio_xchg_last_cpupid(struct folio *folio, int cpupid);
1834 static inline int folio_xchg_access_time(struct folio *folio, int time)  in folio_xchg_access_time()  argument
1838 	last_time = folio_xchg_last_cpupid(folio,  in folio_xchg_access_time()
1853 bool folio_use_access_time(struct folio *folio);
1855 static inline int folio_xchg_last_cpupid(struct folio *folio, int cpupid)  in folio_xchg_last_cpupid()  argument
1857 	return folio_nid(folio); /* XXX */  in folio_xchg_last_cpupid()
1860 static inline int folio_xchg_access_time(struct folio *folio, int time)  in folio_xchg_access_time()  argument
1865 static inline int folio_last_cpupid(struct folio *folio)  in folio_last_cpupid()  argument
1867 	return folio_nid(folio); /* XXX */  in folio_last_cpupid()
1907 static inline bool folio_use_access_time(struct folio *folio)  in folio_use_access_time()  argument
1977 static inline struct zone *folio_zone(const struct folio *folio)  in folio_zone()  argument
1979 	return page_zone(&folio->page);  in folio_zone()
1982 static inline pg_data_t *folio_pgdat(const struct folio *folio)  in folio_pgdat()  argument
1984 	return page_pgdat(&folio->page);  in folio_pgdat()
2009 static inline unsigned long folio_pfn(struct folio *folio)  in folio_pfn()  argument
2011 	return page_to_pfn(&folio->page);  in folio_pfn()
2014 static inline struct folio *pfn_folio(unsigned long pfn)  in pfn_folio()
2044 static inline bool folio_maybe_dma_pinned(struct folio *folio)  in folio_maybe_dma_pinned()  argument
2046 	if (folio_test_large(folio))  in folio_maybe_dma_pinned()
2047 		return atomic_read(&folio->_pincount) > 0;  in folio_maybe_dma_pinned()
2057 	return ((unsigned int)folio_ref_count(folio)) >=  in folio_maybe_dma_pinned()
2068 					  struct folio *folio)  in folio_needs_cow_for_dma()  argument
2075 	return folio_maybe_dma_pinned(folio);  in folio_needs_cow_for_dma()
2095 static inline bool is_zero_folio(const struct folio *folio)  in is_zero_folio()  argument
2097 	return is_zero_page(&folio->page);  in is_zero_folio()
2102 extern void _trace_android_vh_mm_customize_longterm_pinnable(struct folio *folio,
2105 static inline bool folio_is_longterm_pinnable(struct folio *folio)  in folio_is_longterm_pinnable()  argument
2109 	_trace_android_vh_mm_customize_longterm_pinnable(folio, &is_longterm_pinnable);  in folio_is_longterm_pinnable()
2114 	int mt = folio_migratetype(folio);  in folio_is_longterm_pinnable()
2120 	if (is_zero_folio(folio))  in folio_is_longterm_pinnable()
2124 	if (folio_is_device_coherent(folio))  in folio_is_longterm_pinnable()
2128 	return !folio_is_zone_movable(folio);  in folio_is_longterm_pinnable()
2132 static inline bool folio_is_longterm_pinnable(struct folio *folio)  in folio_is_longterm_pinnable()  argument
2166 static inline long folio_nr_pages(const struct folio *folio)  in folio_nr_pages()  argument
2168 	if (!folio_test_large(folio))  in folio_nr_pages()
2171 	return folio->_folio_nr_pages;  in folio_nr_pages()
2173 	return 1L << (folio->_flags_1 & 0xff);  in folio_nr_pages()
2191 	struct folio *folio = (struct folio *)page;  in compound_nr()  local
2193 	if (!test_bit(PG_head, &folio->flags))  in compound_nr()
2196 	return folio->_folio_nr_pages;  in compound_nr()
2198 	return 1L << (folio->_flags_1 & 0xff);  in compound_nr()
2208 	return folio_nr_pages((struct folio *)page);  in thp_nr_pages()
2225 static inline struct folio *folio_next(struct folio *folio)  in folio_next()  argument
2227 	return (struct folio *)folio_page(folio, folio_nr_pages(folio));  in folio_next()
2242 static inline unsigned int folio_shift(const struct folio *folio)  in folio_shift()  argument
2244 	return PAGE_SHIFT + folio_order(folio);  in folio_shift()
2255 static inline size_t folio_size(const struct folio *folio)  in folio_size()  argument
2257 	return PAGE_SIZE << folio_order(folio);  in folio_size()
2303 static inline bool folio_likely_mapped_shared(struct folio *folio)  in folio_likely_mapped_shared()  argument
2305 	int mapcount = folio_mapcount(folio);  in folio_likely_mapped_shared()
2308 	if (!folio_test_large(folio) || unlikely(folio_test_hugetlb(folio)))  in folio_likely_mapped_shared()
2316 	if (folio_entire_mapcount(folio) || mapcount > folio_nr_pages(folio))  in folio_likely_mapped_shared()
2320 	return atomic_read(&folio->_mapcount) > 0;  in folio_likely_mapped_shared()
2356 static inline int folio_expected_ref_count(const struct folio *folio)  in folio_expected_ref_count()  argument
2358 	const int order = folio_order(folio);  in folio_expected_ref_count()
2361 	if (WARN_ON_ONCE(folio_test_slab(folio)))  in folio_expected_ref_count()
2364 	if (folio_test_anon(folio)) {  in folio_expected_ref_count()
2366 		ref_count += folio_test_swapcache(folio) << order;  in folio_expected_ref_count()
2367 	} else if (!((unsigned long)folio->mapping & PAGE_MAPPING_FLAGS)) {  in folio_expected_ref_count()
2369 		ref_count += !!folio->mapping << order;  in folio_expected_ref_count()
2371 		ref_count += folio_test_private(folio);  in folio_expected_ref_count()
2375 	return ref_count + folio_mapcount(folio);  in folio_expected_ref_count()
2379 static inline int arch_make_folio_accessible(struct folio *folio)  in arch_make_folio_accessible()  argument
2423 static inline void *folio_address(const struct folio *folio)  in folio_address()  argument
2425 	return page_address(&folio->page);  in folio_address()
2448 static inline bool folio_is_pfmemalloc(const struct folio *folio)  in folio_is_pfmemalloc()  argument
2455 	return (uintptr_t)folio->lru.next & BIT(1);  in folio_is_pfmemalloc()
2479 #define offset_in_folio(folio, p) ((unsigned long)(p) & (folio_size(folio) - 1))  argument
2485 	struct folio *single_folio;	/* Locked folio to be unmapped */
2533 struct folio *vm_normal_folio(struct vm_area_struct *vma, unsigned long addr,
2537 struct folio *vm_normal_folio_pmd(struct vm_area_struct *vma,
2600 		struct folio *folio);
2700 		      struct folio **folios, unsigned int max_folios,
2707 void folio_add_pin(struct folio *folio);
2716 bool folio_mark_dirty(struct folio *folio);
2799 static inline int mm_counter_file(struct folio *folio)  in mm_counter_file()  argument
2801 	if (folio_test_swapbacked(folio))  in mm_counter_file()
2806 static inline int mm_counter(struct folio *folio)  in mm_counter()  argument
2808 	if (folio_test_anon(folio))  in mm_counter()
2810 	return mm_counter_file(folio);  in mm_counter()
3163 	struct folio *folio = ptdesc_folio(ptdesc);  in pagetable_pte_ctor()  local
3167 	__folio_set_pgtable(folio);  in pagetable_pte_ctor()
3168 	lruvec_stat_add_folio(folio, NR_PAGETABLE);  in pagetable_pte_ctor()
3174 	struct folio *folio = ptdesc_folio(ptdesc);  in pagetable_pte_dtor()  local
3177 	__folio_clear_pgtable(folio);  in pagetable_pte_dtor()
3178 	lruvec_stat_sub_folio(folio, NR_PAGETABLE);  in pagetable_pte_dtor()
3281 	struct folio *folio = ptdesc_folio(ptdesc);  in pagetable_pmd_ctor()  local
3285 	__folio_set_pgtable(folio);  in pagetable_pmd_ctor()
3287 	lruvec_stat_add_folio(folio, NR_PAGETABLE);  in pagetable_pmd_ctor()
3293 	struct folio *folio = ptdesc_folio(ptdesc);  in pagetable_pmd_dtor()  local
3296 	__folio_clear_pgtable(folio);  in pagetable_pmd_dtor()
3297 	lruvec_stat_sub_folio(folio, NR_PAGETABLE);  in pagetable_pmd_dtor()
3321 	struct folio *folio = ptdesc_folio(ptdesc);  in pagetable_pud_ctor()  local
3323 	__folio_set_pgtable(folio);  in pagetable_pud_ctor()
3324 	lruvec_stat_add_folio(folio, NR_PAGETABLE);  in pagetable_pud_ctor()
3329 	struct folio *folio = ptdesc_folio(ptdesc);  in pagetable_pud_dtor()  local
3331 	__folio_clear_pgtable(folio);  in pagetable_pud_dtor()
3332 	lruvec_stat_sub_folio(folio, NR_PAGETABLE);  in pagetable_pud_dtor()
4207 void folio_zero_user(struct folio *folio, unsigned long addr_hint);
4208 int copy_user_large_folio(struct folio *dst, struct folio *src,
4211 long copy_folio_from_user(struct folio *dst_folio,