Lines Matching refs:page
213 static __always_inline const struct page *page_fixed_fake_head(const struct page *page) in page_fixed_fake_head() argument
216 return page; in page_fixed_fake_head()
224 if (IS_ALIGNED((unsigned long)page, PAGE_SIZE) && in page_fixed_fake_head()
225 test_bit(PG_head, &page->flags)) { in page_fixed_fake_head()
231 unsigned long head = READ_ONCE(page[1].compound_head); in page_fixed_fake_head()
234 return (const struct page *)(head - 1); in page_fixed_fake_head()
236 return page; in page_fixed_fake_head()
239 static inline const struct page *page_fixed_fake_head(const struct page *page) in page_fixed_fake_head() argument
241 return page; in page_fixed_fake_head()
245 static __always_inline int page_is_fake_head(const struct page *page) in page_is_fake_head() argument
247 return page_fixed_fake_head(page) != page; in page_is_fake_head()
250 static __always_inline unsigned long _compound_head(const struct page *page) in _compound_head() argument
252 unsigned long head = READ_ONCE(page->compound_head); in _compound_head()
256 return (unsigned long)page_fixed_fake_head(page); in _compound_head()
259 #define compound_head(page) ((typeof(page))_compound_head(page)) argument
275 const struct page *: (const struct folio *)_compound_head(p), \
276 struct page *: (struct folio *)_compound_head(p)))
287 #define folio_page(folio, n) nth_page(&(folio)->page, n)
289 static __always_inline int PageTail(const struct page *page) in PageTail() argument
291 return READ_ONCE(page->compound_head) & 1 || page_is_fake_head(page); in PageTail()
294 static __always_inline int PageCompound(const struct page *page) in PageCompound() argument
296 return test_bit(PG_head, &page->flags) || in PageCompound()
297 READ_ONCE(page->compound_head) & 1; in PageCompound()
301 static inline int PagePoisoned(const struct page *page) in PagePoisoned() argument
303 return READ_ONCE(page->flags) == PAGE_POISON_PATTERN; in PagePoisoned()
307 void page_init_poison(struct page *page, size_t size);
309 static inline void page_init_poison(struct page *page, size_t size) in page_init_poison() argument
317 const struct page *page = &folio->page; in const_folio_flags() local
319 VM_BUG_ON_PGFLAGS(page->compound_head & 1, page); in const_folio_flags()
320 VM_BUG_ON_PGFLAGS(n > 0 && !test_bit(PG_head, &page->flags), page); in const_folio_flags()
321 return &page[n].flags; in const_folio_flags()
326 struct page *page = &folio->page; in folio_flags() local
328 VM_BUG_ON_PGFLAGS(page->compound_head & 1, page); in folio_flags()
329 VM_BUG_ON_PGFLAGS(n > 0 && !test_bit(PG_head, &page->flags), page); in folio_flags()
330 return &page[n].flags; in folio_flags()
356 #define PF_POISONED_CHECK(page) ({ \ argument
357 VM_BUG_ON_PGFLAGS(PagePoisoned(page), page); \
358 page; })
359 #define PF_ANY(page, enforce) PF_POISONED_CHECK(page) argument
360 #define PF_HEAD(page, enforce) PF_POISONED_CHECK(compound_head(page)) argument
361 #define PF_NO_TAIL(page, enforce) ({ \ argument
362 VM_BUG_ON_PGFLAGS(enforce && PageTail(page), page); \
363 PF_POISONED_CHECK(compound_head(page)); })
364 #define PF_NO_COMPOUND(page, enforce) ({ \ argument
365 VM_BUG_ON_PGFLAGS(enforce && PageCompound(page), page); \
366 PF_POISONED_CHECK(page); })
367 #define PF_SECOND(page, enforce) ({ \ argument
368 VM_BUG_ON_PGFLAGS(!PageHead(page), page); \
369 PF_POISONED_CHECK(&page[1]); })
384 #define FOLIO_TEST_FLAG(name, page) \ argument
386 { return test_bit(PG_##name, const_folio_flags(folio, page)); }
388 #define FOLIO_SET_FLAG(name, page) \ argument
390 { set_bit(PG_##name, folio_flags(folio, page)); }
392 #define FOLIO_CLEAR_FLAG(name, page) \ argument
394 { clear_bit(PG_##name, folio_flags(folio, page)); }
396 #define __FOLIO_SET_FLAG(name, page) \ argument
398 { __set_bit(PG_##name, folio_flags(folio, page)); }
400 #define __FOLIO_CLEAR_FLAG(name, page) \ argument
402 { __clear_bit(PG_##name, folio_flags(folio, page)); }
404 #define FOLIO_TEST_SET_FLAG(name, page) \ argument
406 { return test_and_set_bit(PG_##name, folio_flags(folio, page)); }
408 #define FOLIO_TEST_CLEAR_FLAG(name, page) \ argument
410 { return test_and_clear_bit(PG_##name, folio_flags(folio, page)); }
412 #define FOLIO_FLAG(name, page) \ argument
413 FOLIO_TEST_FLAG(name, page) \
414 FOLIO_SET_FLAG(name, page) \
415 FOLIO_CLEAR_FLAG(name, page)
419 static __always_inline int Page##uname(const struct page *page) \
420 { return test_bit(PG_##lname, &policy(page, 0)->flags); }
424 static __always_inline void SetPage##uname(struct page *page) \
425 { set_bit(PG_##lname, &policy(page, 1)->flags); }
429 static __always_inline void ClearPage##uname(struct page *page) \
430 { clear_bit(PG_##lname, &policy(page, 1)->flags); }
434 static __always_inline void __SetPage##uname(struct page *page) \
435 { __set_bit(PG_##lname, &policy(page, 1)->flags); }
439 static __always_inline void __ClearPage##uname(struct page *page) \
440 { __clear_bit(PG_##lname, &policy(page, 1)->flags); }
444 static __always_inline int TestSetPage##uname(struct page *page) \
445 { return test_and_set_bit(PG_##lname, &policy(page, 1)->flags); }
449 static __always_inline int TestClearPage##uname(struct page *page) \
450 { return test_and_clear_bit(PG_##lname, &policy(page, 1)->flags); }
491 static inline int Page##uname(const struct page *page) { return 0; }
495 static inline void SetPage##uname(struct page *page) { }
499 static inline void ClearPage##uname(struct page *page) { }
503 static inline void __ClearPage##uname(struct page *page) { }
507 static inline int TestSetPage##uname(struct page *page) { return 0; }
511 static inline int TestClearPage##uname(struct page *page) { return 0; }
704 static __always_inline bool PageMappingFlags(const struct page *page) in PageMappingFlags() argument
706 return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) != 0; in PageMappingFlags()
714 static __always_inline bool PageAnon(const struct page *page) in PageAnon() argument
716 return folio_test_anon(page_folio(page)); in PageAnon()
725 static __always_inline bool __PageMovable(const struct page *page) in __PageMovable() argument
727 return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) == in __PageMovable()
744 static __always_inline bool PageKsm(const struct page *page) in PageKsm() argument
746 return folio_test_ksm(page_folio(page)); in PageKsm()
752 u64 stable_page_flags(const struct page *page);
800 static inline bool PageUptodate(const struct page *page) in PageUptodate() argument
802 return folio_test_uptodate(page_folio(page)); in PageUptodate()
822 static __always_inline void __SetPageUptodate(struct page *page) in __SetPageUptodate() argument
824 __folio_mark_uptodate((struct folio *)page); in __SetPageUptodate()
827 static __always_inline void SetPageUptodate(struct page *page) in SetPageUptodate() argument
829 folio_mark_uptodate((struct folio *)page); in SetPageUptodate()
835 void set_page_writeback(struct page *page);
847 static __always_inline int PageHead(const struct page *page) in PageHead() argument
849 PF_POISONED_CHECK(page); in PageHead()
850 return test_bit(PG_head, &page->flags) && !page_is_fake_head(page); in PageHead()
868 static __always_inline void set_compound_head(struct page *page, struct page *head) in set_compound_head() argument
870 WRITE_ONCE(page->compound_head, (unsigned long)head + 1); in set_compound_head()
873 static __always_inline void clear_compound_head(struct page *page) in clear_compound_head() argument
875 WRITE_ONCE(page->compound_head, 0); in clear_compound_head()
879 static inline void ClearPageCompound(struct page *page) in ClearPageCompound() argument
881 BUG_ON(!PageHead(page)); in ClearPageCompound()
882 ClearPageHead(page); in ClearPageCompound()
902 static inline int PageTransHuge(const struct page *page) in FOLIO_FLAG()
904 VM_BUG_ON_PAGE(PageTail(page), page); in FOLIO_FLAG()
905 return PageHead(page); in FOLIO_FLAG()
913 static inline int PageTransCompound(const struct page *page) in PageTransCompound() argument
915 return PageCompound(page); in PageTransCompound()
923 static inline int PageTransTail(const struct page *page) in PageTransTail() argument
925 return PageTail(page); in PageTransTail()
980 static inline bool page_has_type(const struct page *page) in page_has_type() argument
982 return page_mapcount_is_type(data_race(page->page_type)); in page_has_type()
988 return data_race(folio->page.page_type >> 24) == PGTY_##lname; \
994 VM_BUG_ON_FOLIO(data_race(folio->page.page_type) != UINT_MAX, \
996 folio->page.page_type = (unsigned int)PGTY_##lname << 24; \
1000 if (folio->page.page_type == UINT_MAX) \
1003 folio->page.page_type = UINT_MAX; \
1008 static __always_inline int Page##uname(const struct page *page) \
1010 return data_race(page->page_type >> 24) == PGTY_##lname; \
1012 static __always_inline void __SetPage##uname(struct page *page) \
1014 if (Page##uname(page)) \
1016 VM_BUG_ON_PAGE(data_race(page->page_type) != UINT_MAX, page); \
1017 page->page_type = (unsigned int)PGTY_##lname << 24; \
1019 static __always_inline void __ClearPage##uname(struct page *page) \
1021 if (page->page_type == UINT_MAX) \
1023 VM_BUG_ON_PAGE(!Page##uname(page), page); \
1024 page->page_type = UINT_MAX; \
1088 static inline bool PageSlab(const struct page *page) in PAGE_TYPE_OPS()
1090 return folio_test_slab(page_folio(page)); in PAGE_TYPE_OPS()
1116 static inline bool PageHuge(const struct page *page) in FOLIO_TYPE_OPS()
1118 return folio_test_hugetlb(page_folio(page)); in FOLIO_TYPE_OPS()
1126 static inline bool is_page_hwpoison(const struct page *page) in is_page_hwpoison() argument
1130 if (PageHWPoison(page)) in is_page_hwpoison()
1132 folio = page_folio(page); in is_page_hwpoison()
1133 return folio_test_hugetlb(folio) && PageHWPoison(&folio->page); in is_page_hwpoison()
1142 bool is_free_buddy_page(const struct page *page);
1146 static __always_inline int PageAnonExclusive(const struct page *page) in PageAnonExclusive() argument
1148 VM_BUG_ON_PGFLAGS(!PageAnon(page), page); in PageAnonExclusive()
1153 if (PageHuge(page)) in PageAnonExclusive()
1154 page = compound_head(page); in PageAnonExclusive()
1155 return test_bit(PG_anon_exclusive, &PF_ANY(page, 1)->flags); in PageAnonExclusive()
1158 static __always_inline void SetPageAnonExclusive(struct page *page) in SetPageAnonExclusive() argument
1160 VM_BUG_ON_PGFLAGS(!PageAnon(page) || PageKsm(page), page); in SetPageAnonExclusive()
1161 VM_BUG_ON_PGFLAGS(PageHuge(page) && !PageHead(page), page); in SetPageAnonExclusive()
1162 set_bit(PG_anon_exclusive, &PF_ANY(page, 1)->flags); in SetPageAnonExclusive()
1165 static __always_inline void ClearPageAnonExclusive(struct page *page) in ClearPageAnonExclusive() argument
1167 VM_BUG_ON_PGFLAGS(!PageAnon(page) || PageKsm(page), page); in ClearPageAnonExclusive()
1168 VM_BUG_ON_PGFLAGS(PageHuge(page) && !PageHead(page), page); in ClearPageAnonExclusive()
1169 clear_bit(PG_anon_exclusive, &PF_ANY(page, 1)->flags); in ClearPageAnonExclusive()
1172 static __always_inline void __ClearPageAnonExclusive(struct page *page) in __ClearPageAnonExclusive() argument
1174 VM_BUG_ON_PGFLAGS(!PageAnon(page), page); in __ClearPageAnonExclusive()
1175 VM_BUG_ON_PGFLAGS(PageHuge(page) && !PageHead(page), page); in __ClearPageAnonExclusive()
1176 __clear_bit(PG_anon_exclusive, &PF_ANY(page, 1)->flags); in __ClearPageAnonExclusive()