Lines Matching full:page
3 * Macros for manipulating and testing page->flags
18 * Various page->flags bits:
20 * PG_reserved is set for special pages. The "struct page" of such a page
25 * - Pages reserved or allocated early during boot (before the page allocator
27 * initial vmemmap, initial page tables, crashkernel, elfcorehdr, and much
29 * be given to the page allocator.
32 * - The zero page(s)
33 * - Pages not added to the page allocator when onlining a section because
49 * Consequently, PG_reserved for a page mapped into user space can indicate
50 * the zero page, the vDSO, MMIO pages or device memory.
53 * specific data (which is normally at page->private). It can be used by
60 * PG_locked also pins a page in pagecache, and blocks truncation of the file
63 * page_waitqueue(page) is a wait queue of all tasks waiting for the page
66 * PG_swapbacked is set when a page uses swap as a backing storage. This are
71 * PG_uptodate tells whether the page's contents is valid. When a read
72 * completes, the page becomes uptodate, unless a disk I/O error happened.
74 * PG_referenced, PG_reclaim are used for page reclaim for anonymous and
77 * PG_error is set to indicate that an I/O error occurred on this page.
79 * PG_arch_1 is an architecture specific page state bit. The generic code
80 * guarantees that this bit is cleared for a page when it first is entered into
81 * the page cache.
83 * PG_hwpoison indicates that a page got corrupted in hardware and contains
90 * locked- and dirty-page accounting.
92 * The page flags field is split into two parts, the main flags area
105 PG_locked, /* Page is locked. Don't touch. */
112 …PG_waiters, /* Page has waiters, check its waitqueue. Must be bit #7 and in the same byte as "PG_…
120 PG_writeback, /* Page is under writeback */
121 PG_head, /* A head page */
124 PG_swapbacked, /* Page is backed by RAM/swap */
125 PG_unevictable, /* Page is "unevictable" */
127 PG_mlocked, /* Page is vma mlocked */
130 PG_uncached, /* Page has been mapped as uncached */
133 PG_hwpoison, /* hardware poisoned page. Don't touch */
159 PG_swapcache = PG_owner_priv_1, /* Swap page: swp_entry_t in private */
161 /* Two page bits are conscripted by FS-Cache to maintain local caching
165 PG_fscache = PG_private_2, /* page backed by cache */
168 /* Pinned in Xen as a read-only pagetable page. */
172 /* Has a grant mapping of another (foreign) domain's page. */
180 /* Compound pages. Stored in first tail page's flags */
183 /* non-lru isolated movable page */
192 struct page; /* forward declaration */
194 static inline struct page *compound_head(struct page *page) in compound_head() argument
196 unsigned long head = READ_ONCE(page->compound_head); in compound_head()
199 return (struct page *) (head - 1); in compound_head()
200 return page; in compound_head()
203 static __always_inline int PageTail(struct page *page) in PageTail() argument
205 return READ_ONCE(page->compound_head) & 1; in PageTail()
208 static __always_inline int PageCompound(struct page *page) in PageCompound() argument
210 return test_bit(PG_head, &page->flags) || PageTail(page); in PageCompound()
214 static inline int PagePoisoned(const struct page *page) in PagePoisoned() argument
216 return page->flags == PAGE_POISON_PATTERN; in PagePoisoned()
220 void page_init_poison(struct page *page, size_t size);
222 static inline void page_init_poison(struct page *page, size_t size) in page_init_poison() argument
228 * Page flags policies wrt compound pages
231 * check if this struct page poisoned/uninitialized
234 * the page flag is relevant for small, head and tail pages.
237 * for compound page all operations related to the page flag applied to
238 * head page.
241 * for compound page, callers only ever operate on the head page.
244 * modifications of the page flag must be done on small or head pages,
248 * the page flag is not relevant for compound pages.
251 * the page flag is stored in the first tail page.
253 #define PF_POISONED_CHECK(page) ({ \ argument
254 VM_BUG_ON_PGFLAGS(PagePoisoned(page), page); \
255 page; })
256 #define PF_ANY(page, enforce) PF_POISONED_CHECK(page) argument
257 #define PF_HEAD(page, enforce) PF_POISONED_CHECK(compound_head(page)) argument
258 #define PF_ONLY_HEAD(page, enforce) ({ \ argument
259 VM_BUG_ON_PGFLAGS(PageTail(page), page); \
260 PF_POISONED_CHECK(page); })
261 #define PF_NO_TAIL(page, enforce) ({ \ argument
262 VM_BUG_ON_PGFLAGS(enforce && PageTail(page), page); \
263 PF_POISONED_CHECK(compound_head(page)); })
264 #define PF_NO_COMPOUND(page, enforce) ({ \ argument
265 VM_BUG_ON_PGFLAGS(enforce && PageCompound(page), page); \
266 PF_POISONED_CHECK(page); })
267 #define PF_SECOND(page, enforce) ({ \ argument
268 VM_BUG_ON_PGFLAGS(!PageHead(page), page); \
269 PF_POISONED_CHECK(&page[1]); })
272 * Macros to create function definitions for page flags
275 static __always_inline int Page##uname(struct page *page) \
276 { return test_bit(PG_##lname, &policy(page, 0)->flags); }
279 static __always_inline void SetPage##uname(struct page *page) \
280 { set_bit(PG_##lname, &policy(page, 1)->flags); }
283 static __always_inline void ClearPage##uname(struct page *page) \
284 { clear_bit(PG_##lname, &policy(page, 1)->flags); }
287 static __always_inline void __SetPage##uname(struct page *page) \
288 { __set_bit(PG_##lname, &policy(page, 1)->flags); }
291 static __always_inline void __ClearPage##uname(struct page *page) \
292 { __clear_bit(PG_##lname, &policy(page, 1)->flags); }
295 static __always_inline int TestSetPage##uname(struct page *page) \
296 { return test_and_set_bit(PG_##lname, &policy(page, 1)->flags); }
299 static __always_inline int TestClearPage##uname(struct page *page) \
300 { return test_and_clear_bit(PG_##lname, &policy(page, 1)->flags); }
317 static inline int Page##uname(const struct page *page) { return 0; }
320 static inline void SetPage##uname(struct page *page) { }
323 static inline void ClearPage##uname(struct page *page) { }
326 static inline void __ClearPage##uname(struct page *page) { }
329 static inline int TestSetPage##uname(struct page *page) { return 0; }
332 static inline int TestClearPage##uname(struct page *page) { return 0; }
381 * Private page markings that may be used by the filesystem that owns the page in PAGEFLAG()
393 * risky: they bypass page accounting. in PAGEFLAG()
416 static __always_inline int PageSwapCache(struct page *page) in PAGEFLAG()
419 page = compound_head(page); in PAGEFLAG()
421 return PageSwapBacked(page) && test_bit(PG_swapcache, &page->flags); in PAGEFLAG()
453 extern bool take_page_off_buddy(struct page *page);
485 * On an anonymous page mapped into a user virtual memory area, in TESTPAGEFLAG()
486 * page->mapping points to its anon_vma, not to a struct address_space; in TESTPAGEFLAG()
489 * On an anonymous page in a VM_MERGEABLE area, if CONFIG_KSM is enabled, in TESTPAGEFLAG()
491 * bit; and then page->mapping points, not to an anon_vma, but to a private in TESTPAGEFLAG()
492 * structure which KSM associates with that merged page. See ksm.h. in TESTPAGEFLAG()
495 * page and then page->mapping points a struct address_space. in TESTPAGEFLAG()
498 * address_space which maps the page from disk; whereas "page_mapped" in TESTPAGEFLAG()
499 * refers to user virtual address space into which the page is mapped. in TESTPAGEFLAG()
506 static __always_inline int PageMappingFlags(struct page *page) in TESTPAGEFLAG()
508 return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) != 0; in TESTPAGEFLAG()
511 static __always_inline int PageAnon(struct page *page) in PageAnon() argument
513 page = compound_head(page); in PageAnon()
514 return ((unsigned long)page->mapping & PAGE_MAPPING_ANON) != 0; in PageAnon()
517 static __always_inline int __PageMovable(struct page *page) in __PageMovable() argument
519 return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) == in __PageMovable()
525 * A KSM page is one of those write-protected "shared pages" or "merged pages"
526 * which KSM maps into multiple mms, wherever identical anonymous page content
527 * is found in VM_MERGEABLE vmas. It's a PageAnon page, pointing not to any
528 * anon_vma, but to that page's node of the stable tree.
530 static __always_inline int PageKsm(struct page *page) in PageKsm() argument
532 page = compound_head(page); in PageKsm()
533 return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) == in PageKsm()
540 u64 stable_page_flags(struct page *page);
542 static inline int PageUptodate(struct page *page) in PageUptodate() argument
545 page = compound_head(page); in PageUptodate()
546 ret = test_bit(PG_uptodate, &(page)->flags); in PageUptodate()
548 * Must ensure that the data we read out of the page is loaded in PageUptodate()
549 * _after_ we've loaded page->flags to check for PageUptodate. in PageUptodate()
550 * We can skip the barrier if the page is not uptodate, because in PageUptodate()
561 static __always_inline void __SetPageUptodate(struct page *page) in __SetPageUptodate() argument
563 VM_BUG_ON_PAGE(PageTail(page), page); in __SetPageUptodate()
565 __set_bit(PG_uptodate, &page->flags); in __SetPageUptodate()
568 static __always_inline void SetPageUptodate(struct page *page) in SetPageUptodate() argument
570 VM_BUG_ON_PAGE(PageTail(page), page); in SetPageUptodate()
573 * so that all previous stores issued in order to bring the page in SetPageUptodate()
577 set_bit(PG_uptodate, &page->flags); in SetPageUptodate()
582 int test_clear_page_writeback(struct page *page);
583 int __test_set_page_writeback(struct page *page, bool keep_write);
585 #define test_set_page_writeback(page) \ argument
586 __test_set_page_writeback(page, false)
587 #define test_set_page_writeback_keepwrite(page) \ argument
588 __test_set_page_writeback(page, true)
590 static inline void set_page_writeback(struct page *page) in set_page_writeback() argument
592 test_set_page_writeback(page); in set_page_writeback()
595 static inline void set_page_writeback_keepwrite(struct page *page) in set_page_writeback_keepwrite() argument
597 test_set_page_writeback_keepwrite(page); in set_page_writeback_keepwrite()
602 static __always_inline void set_compound_head(struct page *page, struct page *head) in __PAGEFLAG()
604 WRITE_ONCE(page->compound_head, (unsigned long)head + 1); in __PAGEFLAG()
607 static __always_inline void clear_compound_head(struct page *page) in clear_compound_head() argument
609 WRITE_ONCE(page->compound_head, 0); in clear_compound_head()
613 static inline void ClearPageCompound(struct page *page) in ClearPageCompound() argument
615 BUG_ON(!PageHead(page)); in ClearPageCompound()
616 ClearPageHead(page); in ClearPageCompound()
623 int PageHuge(struct page *page);
624 int PageHeadHuge(struct page *page);
625 bool page_huge_active(struct page *page);
630 static inline bool page_huge_active(struct page *page) in TESTPAGEFLAG_FALSE()
646 static inline int PageTransHuge(struct page *page) in PageTransHuge() argument
648 VM_BUG_ON_PAGE(PageTail(page), page); in PageTransHuge()
649 return PageHead(page); in PageTransHuge()
657 static inline int PageTransCompound(struct page *page) in PageTransCompound() argument
659 return PageCompound(page); in PageTransCompound()
664 * guarantees the primary MMU has the entire compound page mapped
666 * can also map the entire compound page. This allows the secondary
667 * MMUs to call get_user_pages() only once for each compound page and
668 * to immediately map the entire compound page with a single secondary
675 * MMU notifier, otherwise it may result in page->_mapcount check false
678 * We have to treat page cache THP differently since every subpage of it
683 static inline int PageTransCompoundMap(struct page *page) in PageTransCompoundMap() argument
685 struct page *head; in PageTransCompoundMap()
687 if (!PageTransCompound(page)) in PageTransCompoundMap()
690 if (PageAnon(page)) in PageTransCompoundMap()
691 return atomic_read(&page->_mapcount) < 0; in PageTransCompoundMap()
693 head = compound_head(page); in PageTransCompoundMap()
695 return atomic_read(&page->_mapcount) == in PageTransCompoundMap()
704 static inline int PageTransTail(struct page *page) in PageTransTail() argument
706 return PageTail(page); in PageTransTail()
710 * PageDoubleMap indicates that the compound page is mapped with PTEs as well
714 * per small page mapcount accounting (and its overhead from atomic operations)
717 * For the page PageDoubleMap means ->_mapcount in all sub-pages is offset up
739 * mistaken for a page type value. in PAGEFLAG()
751 #define PageType(page, flag) \ in PAGEFLAG() argument
752 ((page->page_type & (PAGE_TYPE_BASE | flag)) == PAGE_TYPE_BASE) in PAGEFLAG()
754 static inline int page_has_type(struct page *page) in PAGEFLAG()
756 return (int)page->page_type < PAGE_MAPCOUNT_RESERVE; in PAGEFLAG()
760 static __always_inline int Page##uname(struct page *page) \
762 return PageType(page, PG_##lname); \
764 static __always_inline void __SetPage##uname(struct page *page) \
766 VM_BUG_ON_PAGE(!PageType(page, 0), page); \
767 page->page_type &= ~PG_##lname; \
769 static __always_inline void __ClearPage##uname(struct page *page) \
771 VM_BUG_ON_PAGE(!Page##uname(page), page); \
772 page->page_type |= PG_##lname; \
776 * PageBuddy() indicates that the page is free and in the buddy system
782 * PageOffline() indicates that the page is logically offline although the
802 * pages allocated with __GFP_ACCOUNT. It gets cleared on page free.
807 * Marks pages in use as page tables.
816 extern bool is_free_buddy_page(struct page *page);
824 static inline int PageSlabPfmemalloc(struct page *page) in PageSlabPfmemalloc() argument
826 VM_BUG_ON_PAGE(!PageSlab(page), page); in PageSlabPfmemalloc()
827 return PageActive(page); in PageSlabPfmemalloc()
830 static inline void SetPageSlabPfmemalloc(struct page *page) in SetPageSlabPfmemalloc() argument
832 VM_BUG_ON_PAGE(!PageSlab(page), page); in SetPageSlabPfmemalloc()
833 SetPageActive(page); in SetPageSlabPfmemalloc()
836 static inline void __ClearPageSlabPfmemalloc(struct page *page) in __ClearPageSlabPfmemalloc() argument
838 VM_BUG_ON_PAGE(!PageSlab(page), page); in __ClearPageSlabPfmemalloc()
839 __ClearPageActive(page); in __ClearPageSlabPfmemalloc()
842 static inline void ClearPageSlabPfmemalloc(struct page *page) in ClearPageSlabPfmemalloc() argument
844 VM_BUG_ON_PAGE(!PageSlab(page), page); in ClearPageSlabPfmemalloc()
845 ClearPageActive(page); in ClearPageSlabPfmemalloc()
855 * Flags checked when a page is freed. Pages being freed should not have
873 * Flags checked when a page is prepped for return by the page allocator.
875 * there has been a kernel bug or struct page corruption.
877 * __PG_HWPOISON is exceptional because it needs to be kept beyond page's
878 * alloc-free cycle to prevent from reusing the page.
886 * page_has_private - Determine if page has private stuff
887 * @page: The page to be checked
889 * Determine if a page has private stuff, indicating that release routines
892 static inline int page_has_private(struct page *page) in page_has_private() argument
894 return !!(page->flags & PAGE_FLAGS_PRIVATE); in page_has_private()