Lines Matching refs:page
214 static __always_inline const struct page *page_fixed_fake_head(const struct page *page) in page_fixed_fake_head() argument
217 return page; in page_fixed_fake_head()
225 if (IS_ALIGNED((unsigned long)page, PAGE_SIZE) && in page_fixed_fake_head()
226 test_bit(PG_head, &page->flags)) { in page_fixed_fake_head()
232 unsigned long head = READ_ONCE(page[1].compound_head); in page_fixed_fake_head()
235 return (const struct page *)(head - 1); in page_fixed_fake_head()
237 return page; in page_fixed_fake_head()
240 static inline const struct page *page_fixed_fake_head(const struct page *page) in page_fixed_fake_head() argument
242 return page; in page_fixed_fake_head()
246 static __always_inline int page_is_fake_head(struct page *page) in page_is_fake_head() argument
248 return page_fixed_fake_head(page) != page; in page_is_fake_head()
251 static inline unsigned long _compound_head(const struct page *page) in _compound_head() argument
253 unsigned long head = READ_ONCE(page->compound_head); in _compound_head()
257 return (unsigned long)page_fixed_fake_head(page); in _compound_head()
260 #define compound_head(page) ((typeof(page))_compound_head(page)) argument
276 const struct page *: (const struct folio *)_compound_head(p), \
277 struct page *: (struct folio *)_compound_head(p)))
288 #define folio_page(folio, n) nth_page(&(folio)->page, n)
290 static __always_inline int PageTail(struct page *page) in PageTail() argument
292 return READ_ONCE(page->compound_head) & 1 || page_is_fake_head(page); in PageTail()
295 static __always_inline int PageCompound(struct page *page) in PageCompound() argument
297 return test_bit(PG_head, &page->flags) || in PageCompound()
298 READ_ONCE(page->compound_head) & 1; in PageCompound()
302 static inline int PagePoisoned(const struct page *page) in PagePoisoned() argument
304 return READ_ONCE(page->flags) == PAGE_POISON_PATTERN; in PagePoisoned()
308 void page_init_poison(struct page *page, size_t size);
310 static inline void page_init_poison(struct page *page, size_t size) in page_init_poison() argument
317 struct page *page = &folio->page; in folio_flags() local
319 VM_BUG_ON_PGFLAGS(PageTail(page), page); in folio_flags()
320 VM_BUG_ON_PGFLAGS(n > 0 && !test_bit(PG_head, &page->flags), page); in folio_flags()
321 return &page[n].flags; in folio_flags()
350 #define PF_POISONED_CHECK(page) ({ \ argument
351 VM_BUG_ON_PGFLAGS(PagePoisoned(page), page); \
352 page; })
353 #define PF_ANY(page, enforce) PF_POISONED_CHECK(page) argument
354 #define PF_HEAD(page, enforce) PF_POISONED_CHECK(compound_head(page)) argument
355 #define PF_ONLY_HEAD(page, enforce) ({ \ argument
356 VM_BUG_ON_PGFLAGS(PageTail(page), page); \
357 PF_POISONED_CHECK(page); })
358 #define PF_NO_TAIL(page, enforce) ({ \ argument
359 VM_BUG_ON_PGFLAGS(enforce && PageTail(page), page); \
360 PF_POISONED_CHECK(compound_head(page)); })
361 #define PF_NO_COMPOUND(page, enforce) ({ \ argument
362 VM_BUG_ON_PGFLAGS(enforce && PageCompound(page), page); \
363 PF_POISONED_CHECK(page); })
364 #define PF_SECOND(page, enforce) ({ \ argument
365 VM_BUG_ON_PGFLAGS(!PageHead(page), page); \
366 PF_POISONED_CHECK(&page[1]); })
382 static __always_inline int Page##uname(struct page *page) \
383 { return test_bit(PG_##lname, &policy(page, 0)->flags); }
389 static __always_inline void SetPage##uname(struct page *page) \
390 { set_bit(PG_##lname, &policy(page, 1)->flags); }
396 static __always_inline void ClearPage##uname(struct page *page) \
397 { clear_bit(PG_##lname, &policy(page, 1)->flags); }
403 static __always_inline void __SetPage##uname(struct page *page) \
404 { __set_bit(PG_##lname, &policy(page, 1)->flags); }
410 static __always_inline void __ClearPage##uname(struct page *page) \
411 { __clear_bit(PG_##lname, &policy(page, 1)->flags); }
417 static __always_inline int TestSetPage##uname(struct page *page) \
418 { return test_and_set_bit(PG_##lname, &policy(page, 1)->flags); }
424 static __always_inline int TestClearPage##uname(struct page *page) \
425 { return test_and_clear_bit(PG_##lname, &policy(page, 1)->flags); }
443 static inline int Page##uname(const struct page *page) { return 0; }
447 static inline void SetPage##uname(struct page *page) { }
451 static inline void ClearPage##uname(struct page *page) { }
455 static inline void __ClearPage##uname(struct page *page) { }
460 static inline int TestSetPage##uname(struct page *page) { return 0; }
465 static inline int TestClearPage##uname(struct page *page) { return 0; }
547 static __always_inline bool PageSwapCache(struct page *page) in PageSwapCache() argument
549 return folio_test_swapcache(page_folio(page)); in PageSwapCache()
582 extern void SetPageHWPoisonTakenOff(struct page *page);
583 extern void ClearPageHWPoisonTakenOff(struct page *page);
584 extern bool take_page_off_buddy(struct page *page);
585 extern bool put_page_back_buddy(struct page *page);
657 static __always_inline int PageMappingFlags(struct page *page) in PageMappingFlags() argument
659 return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) != 0; in PageMappingFlags()
667 static __always_inline bool PageAnon(struct page *page) in PageAnon() argument
669 return folio_test_anon(page_folio(page)); in PageAnon()
678 static __always_inline int __PageMovable(struct page *page) in __PageMovable() argument
680 return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) == in __PageMovable()
697 static __always_inline bool PageKsm(struct page *page) in PageKsm() argument
699 return folio_test_ksm(page_folio(page)); in PageKsm()
705 u64 stable_page_flags(struct page *page);
734 static inline int PageUptodate(struct page *page) in PageUptodate() argument
736 return folio_test_uptodate(page_folio(page)); in PageUptodate()
756 static __always_inline void __SetPageUptodate(struct page *page) in __SetPageUptodate() argument
758 __folio_mark_uptodate((struct folio *)page); in __SetPageUptodate()
761 static __always_inline void SetPageUptodate(struct page *page) in SetPageUptodate() argument
763 folio_mark_uptodate((struct folio *)page); in SetPageUptodate()
769 bool set_page_writeback(struct page *page);
776 static inline void set_page_writeback_keepwrite(struct page *page) in set_page_writeback_keepwrite() argument
778 folio_start_writeback_keepwrite(page_folio(page)); in set_page_writeback_keepwrite()
781 static inline bool test_set_page_writeback(struct page *page) in test_set_page_writeback() argument
783 return set_page_writeback(page); in test_set_page_writeback()
791 static __always_inline int PageHead(struct page *page) in PageHead() argument
793 PF_POISONED_CHECK(page); in PageHead()
794 return test_bit(PG_head, &page->flags) && !page_is_fake_head(page); in PageHead()
812 static __always_inline void set_compound_head(struct page *page, struct page *head) in set_compound_head() argument
814 WRITE_ONCE(page->compound_head, (unsigned long)head + 1); in set_compound_head()
817 static __always_inline void clear_compound_head(struct page *page) in clear_compound_head() argument
819 WRITE_ONCE(page->compound_head, 0); in clear_compound_head()
823 static inline void ClearPageCompound(struct page *page) in ClearPageCompound() argument
825 BUG_ON(!PageHead(page)); in ClearPageCompound()
826 ClearPageHead(page); in ClearPageCompound()
833 int PageHuge(struct page *page);
834 int PageHeadHuge(struct page *page);
837 return PageHeadHuge(&folio->page); in folio_test_hugetlb()
853 static inline int PageTransHuge(struct page *page) in TESTPAGEFLAG_FALSE()
855 VM_BUG_ON_PAGE(PageTail(page), page); in TESTPAGEFLAG_FALSE()
856 return PageHead(page); in TESTPAGEFLAG_FALSE()
869 static inline int PageTransCompound(struct page *page) in PageTransCompound() argument
871 return PageCompound(page); in PageTransCompound()
879 static inline int PageTransTail(struct page *page) in PageTransTail() argument
881 return PageTail(page); in PageTransTail()
927 static inline bool is_page_hwpoison(struct page *page) in PAGEFLAG()
929 if (PageHWPoison(page)) in PAGEFLAG()
931 return PageHuge(page) && PageHWPoison(compound_head(page)); in PAGEFLAG()
951 #define PageType(page, flag) \ argument
952 ((page->page_type & (PAGE_TYPE_BASE | flag)) == PAGE_TYPE_BASE)
954 static inline int page_has_type(struct page *page) in page_has_type() argument
956 return (int)page->page_type < PAGE_MAPCOUNT_RESERVE; in page_has_type()
960 static __always_inline int Page##uname(struct page *page) \
962 return PageType(page, PG_##lname); \
964 static __always_inline void __SetPage##uname(struct page *page) \
966 VM_BUG_ON_PAGE(!PageType(page, 0), page); \
967 page->page_type &= ~PG_##lname; \
969 static __always_inline void __ClearPage##uname(struct page *page) \
971 VM_BUG_ON_PAGE(!Page##uname(page), page); \
972 page->page_type |= PG_##lname; \
1020 extern bool is_free_buddy_page(struct page *page);
1024 static __always_inline int PageAnonExclusive(struct page *page) in PageAnonExclusive() argument
1026 VM_BUG_ON_PGFLAGS(!PageAnon(page), page); in PageAnonExclusive()
1027 VM_BUG_ON_PGFLAGS(PageHuge(page) && !PageHead(page), page); in PageAnonExclusive()
1028 return test_bit(PG_anon_exclusive, &PF_ANY(page, 1)->flags); in PageAnonExclusive()
1031 static __always_inline void SetPageAnonExclusive(struct page *page) in SetPageAnonExclusive() argument
1033 VM_BUG_ON_PGFLAGS(!PageAnon(page) || PageKsm(page), page); in SetPageAnonExclusive()
1034 VM_BUG_ON_PGFLAGS(PageHuge(page) && !PageHead(page), page); in SetPageAnonExclusive()
1035 set_bit(PG_anon_exclusive, &PF_ANY(page, 1)->flags); in SetPageAnonExclusive()
1038 static __always_inline void ClearPageAnonExclusive(struct page *page) in ClearPageAnonExclusive() argument
1040 VM_BUG_ON_PGFLAGS(!PageAnon(page) || PageKsm(page), page); in ClearPageAnonExclusive()
1041 VM_BUG_ON_PGFLAGS(PageHuge(page) && !PageHead(page), page); in ClearPageAnonExclusive()
1042 clear_bit(PG_anon_exclusive, &PF_ANY(page, 1)->flags); in ClearPageAnonExclusive()
1045 static __always_inline void __ClearPageAnonExclusive(struct page *page) in __ClearPageAnonExclusive() argument
1047 VM_BUG_ON_PGFLAGS(!PageAnon(page), page); in __ClearPageAnonExclusive()
1048 VM_BUG_ON_PGFLAGS(PageHuge(page) && !PageHead(page), page); in __ClearPageAnonExclusive()
1049 __clear_bit(PG_anon_exclusive, &PF_ANY(page, 1)->flags); in __ClearPageAnonExclusive()
1089 static inline int page_has_private(struct page *page) in page_has_private() argument
1091 return !!(page->flags & PAGE_FLAGS_PRIVATE); in page_has_private()
1096 return page_has_private(&folio->page); in folio_has_private()