Lines Matching full:page
3 * Macros for manipulating and testing page->flags
18 * Various page->flags bits:
20 * PG_reserved is set for special pages. The "struct page" of such a page
25 * - Pages reserved or allocated early during boot (before the page allocator
27 * initial vmemmap, initial page tables, crashkernel, elfcorehdr, and much
29 * be given to the page allocator.
32 * - The zero page(s)
33 * - Pages not added to the page allocator when onlining a section because
49 * Consequently, PG_reserved for a page mapped into user space can indicate
50 * the zero page, the vDSO, MMIO pages or device memory.
53 * specific data (which is normally at page->private). It can be used by
60 * PG_locked also pins a page in pagecache, and blocks truncation of the file
63 * page_waitqueue(page) is a wait queue of all tasks waiting for the page
66 * PG_swapbacked is set when a page uses swap as a backing storage. This are
71 * PG_uptodate tells whether the page's contents is valid. When a read
72 * completes, the page becomes uptodate, unless a disk I/O error happened.
74 * PG_referenced, PG_reclaim are used for page reclaim for anonymous and
77 * PG_error is set to indicate that an I/O error occurred on this page.
79 * PG_arch_1 is an architecture specific page state bit. The generic code
80 * guarantees that this bit is cleared for a page when it first is entered into
81 * the page cache.
83 * PG_hwpoison indicates that a page got corrupted in hardware and contains
90 * locked- and dirty-page accounting.
92 * The page flags field is split into two parts, the main flags area
105 PG_locked, /* Page is locked. Don't touch. */
112 …PG_waiters, /* Page has waiters, check its waitqueue. Must be bit #7 and in the same byte as "PG_…
120 PG_writeback, /* Page is under writeback */
121 PG_head, /* A head page */
124 PG_swapbacked, /* Page is backed by RAM/swap */
125 PG_unevictable, /* Page is "unevictable" */
127 PG_mlocked, /* Page is vma mlocked */
130 PG_uncached, /* Page has been mapped as uncached */
133 PG_hwpoison, /* hardware poisoned page. Don't touch */
155 PG_swapcache = PG_owner_priv_1, /* Swap page: swp_entry_t in private */
157 /* Two page bits are conscripted by FS-Cache to maintain local caching
161 PG_fscache = PG_private_2, /* page backed by cache */
164 /* Pinned in Xen as a read-only pagetable page. */
168 /* Has a grant mapping of another (foreign) domain's page. */
176 /* Compound pages. Stored in first tail page's flags */
179 /* non-lru isolated movable page */
188 struct page; /* forward declaration */
190 static inline struct page *compound_head(struct page *page) in compound_head() argument
192 unsigned long head = READ_ONCE(page->compound_head); in compound_head()
195 return (struct page *) (head - 1); in compound_head()
196 return page; in compound_head()
199 static __always_inline int PageTail(struct page *page) in PageTail() argument
201 return READ_ONCE(page->compound_head) & 1; in PageTail()
204 static __always_inline int PageCompound(struct page *page) in PageCompound() argument
206 return test_bit(PG_head, &page->flags) || PageTail(page); in PageCompound()
210 static inline int PagePoisoned(const struct page *page) in PagePoisoned() argument
212 return page->flags == PAGE_POISON_PATTERN; in PagePoisoned()
216 void page_init_poison(struct page *page, size_t size);
218 static inline void page_init_poison(struct page *page, size_t size) in page_init_poison() argument
224 * Page flags policies wrt compound pages
227 * check if this struct page poisoned/uninitialized
230 * the page flag is relevant for small, head and tail pages.
233 * for compound page all operations related to the page flag applied to
234 * head page.
237 * for compound page, callers only ever operate on the head page.
240 * modifications of the page flag must be done on small or head pages,
244 * the page flag is not relevant for compound pages.
247 * the page flag is stored in the first tail page.
249 #define PF_POISONED_CHECK(page) ({ \ argument
250 VM_BUG_ON_PGFLAGS(PagePoisoned(page), page); \
251 page; })
252 #define PF_ANY(page, enforce) PF_POISONED_CHECK(page) argument
253 #define PF_HEAD(page, enforce) PF_POISONED_CHECK(compound_head(page)) argument
254 #define PF_ONLY_HEAD(page, enforce) ({ \ argument
255 VM_BUG_ON_PGFLAGS(PageTail(page), page); \
256 PF_POISONED_CHECK(page); })
257 #define PF_NO_TAIL(page, enforce) ({ \ argument
258 VM_BUG_ON_PGFLAGS(enforce && PageTail(page), page); \
259 PF_POISONED_CHECK(compound_head(page)); })
260 #define PF_NO_COMPOUND(page, enforce) ({ \ argument
261 VM_BUG_ON_PGFLAGS(enforce && PageCompound(page), page); \
262 PF_POISONED_CHECK(page); })
263 #define PF_SECOND(page, enforce) ({ \ argument
264 VM_BUG_ON_PGFLAGS(!PageHead(page), page); \
265 PF_POISONED_CHECK(&page[1]); })
268 * Macros to create function definitions for page flags
271 static __always_inline int Page##uname(struct page *page) \
272 { return test_bit(PG_##lname, &policy(page, 0)->flags); }
275 static __always_inline void SetPage##uname(struct page *page) \
276 { set_bit(PG_##lname, &policy(page, 1)->flags); }
279 static __always_inline void ClearPage##uname(struct page *page) \
280 { clear_bit(PG_##lname, &policy(page, 1)->flags); }
283 static __always_inline void __SetPage##uname(struct page *page) \
284 { __set_bit(PG_##lname, &policy(page, 1)->flags); }
287 static __always_inline void __ClearPage##uname(struct page *page) \
288 { __clear_bit(PG_##lname, &policy(page, 1)->flags); }
291 static __always_inline int TestSetPage##uname(struct page *page) \
292 { return test_and_set_bit(PG_##lname, &policy(page, 1)->flags); }
295 static __always_inline int TestClearPage##uname(struct page *page) \
296 { return test_and_clear_bit(PG_##lname, &policy(page, 1)->flags); }
313 static inline int Page##uname(const struct page *page) { return 0; }
316 static inline void SetPage##uname(struct page *page) { }
319 static inline void ClearPage##uname(struct page *page) { }
322 static inline void __ClearPage##uname(struct page *page) { }
325 static inline int TestSetPage##uname(struct page *page) { return 0; }
328 static inline int TestClearPage##uname(struct page *page) { return 0; }
369 * Private page markings that may be used by the filesystem that owns the page in PAGEFLAG()
381 * risky: they bypass page accounting. in PAGEFLAG()
404 static __always_inline int PageSwapCache(struct page *page) in PAGEFLAG()
407 page = compound_head(page); in PAGEFLAG()
409 return PageSwapBacked(page) && test_bit(PG_swapcache, &page->flags); in PAGEFLAG()
441 extern bool take_page_off_buddy(struct page *page);
473 * On an anonymous page mapped into a user virtual memory area, in TESTPAGEFLAG()
474 * page->mapping points to its anon_vma, not to a struct address_space; in TESTPAGEFLAG()
477 * On an anonymous page in a VM_MERGEABLE area, if CONFIG_KSM is enabled, in TESTPAGEFLAG()
479 * bit; and then page->mapping points, not to an anon_vma, but to a private in TESTPAGEFLAG()
480 * structure which KSM associates with that merged page. See ksm.h. in TESTPAGEFLAG()
483 * page and then page->mapping points a struct address_space. in TESTPAGEFLAG()
486 * address_space which maps the page from disk; whereas "page_mapped" in TESTPAGEFLAG()
487 * refers to user virtual address space into which the page is mapped. in TESTPAGEFLAG()
494 static __always_inline int PageMappingFlags(struct page *page) in TESTPAGEFLAG()
496 return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) != 0; in TESTPAGEFLAG()
499 static __always_inline int PageAnon(struct page *page) in PageAnon() argument
501 page = compound_head(page); in PageAnon()
502 return ((unsigned long)page->mapping & PAGE_MAPPING_ANON) != 0; in PageAnon()
505 static __always_inline int __PageMovable(struct page *page) in __PageMovable() argument
507 return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) == in __PageMovable()
513 * A KSM page is one of those write-protected "shared pages" or "merged pages"
514 * which KSM maps into multiple mms, wherever identical anonymous page content
515 * is found in VM_MERGEABLE vmas. It's a PageAnon page, pointing not to any
516 * anon_vma, but to that page's node of the stable tree.
518 static __always_inline int PageKsm(struct page *page) in PageKsm() argument
520 page = compound_head(page); in PageKsm()
521 return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) == in PageKsm()
528 u64 stable_page_flags(struct page *page);
530 static inline int PageUptodate(struct page *page) in PageUptodate() argument
533 page = compound_head(page); in PageUptodate()
534 ret = test_bit(PG_uptodate, &(page)->flags); in PageUptodate()
536 * Must ensure that the data we read out of the page is loaded in PageUptodate()
537 * _after_ we've loaded page->flags to check for PageUptodate. in PageUptodate()
538 * We can skip the barrier if the page is not uptodate, because in PageUptodate()
549 static __always_inline void __SetPageUptodate(struct page *page) in __SetPageUptodate() argument
551 VM_BUG_ON_PAGE(PageTail(page), page); in __SetPageUptodate()
553 __set_bit(PG_uptodate, &page->flags); in __SetPageUptodate()
556 static __always_inline void SetPageUptodate(struct page *page) in SetPageUptodate() argument
558 VM_BUG_ON_PAGE(PageTail(page), page); in SetPageUptodate()
561 * so that all previous stores issued in order to bring the page in SetPageUptodate()
565 set_bit(PG_uptodate, &page->flags); in SetPageUptodate()
570 int test_clear_page_writeback(struct page *page);
571 int __test_set_page_writeback(struct page *page, bool keep_write);
573 #define test_set_page_writeback(page) \ argument
574 __test_set_page_writeback(page, false)
575 #define test_set_page_writeback_keepwrite(page) \ argument
576 __test_set_page_writeback(page, true)
578 static inline void set_page_writeback(struct page *page) in set_page_writeback() argument
580 test_set_page_writeback(page); in set_page_writeback()
583 static inline void set_page_writeback_keepwrite(struct page *page) in set_page_writeback_keepwrite() argument
585 test_set_page_writeback_keepwrite(page); in set_page_writeback_keepwrite()
590 static __always_inline void set_compound_head(struct page *page, struct page *head) in __PAGEFLAG()
592 WRITE_ONCE(page->compound_head, (unsigned long)head + 1); in __PAGEFLAG()
595 static __always_inline void clear_compound_head(struct page *page) in clear_compound_head() argument
597 WRITE_ONCE(page->compound_head, 0); in clear_compound_head()
601 static inline void ClearPageCompound(struct page *page) in ClearPageCompound() argument
603 BUG_ON(!PageHead(page)); in ClearPageCompound()
604 ClearPageHead(page); in ClearPageCompound()
611 int PageHuge(struct page *page);
612 int PageHeadHuge(struct page *page);
613 bool page_huge_active(struct page *page);
618 static inline bool page_huge_active(struct page *page) in TESTPAGEFLAG_FALSE()
634 static inline int PageTransHuge(struct page *page) in PageTransHuge() argument
636 VM_BUG_ON_PAGE(PageTail(page), page); in PageTransHuge()
637 return PageHead(page); in PageTransHuge()
645 static inline int PageTransCompound(struct page *page) in PageTransCompound() argument
647 return PageCompound(page); in PageTransCompound()
652 * guarantees the primary MMU has the entire compound page mapped
654 * can also map the entire compound page. This allows the secondary
655 * MMUs to call get_user_pages() only once for each compound page and
656 * to immediately map the entire compound page with a single secondary
663 * MMU notifier, otherwise it may result in page->_mapcount check false
666 * We have to treat page cache THP differently since every subpage of it
671 static inline int PageTransCompoundMap(struct page *page) in PageTransCompoundMap() argument
673 struct page *head; in PageTransCompoundMap()
675 if (!PageTransCompound(page)) in PageTransCompoundMap()
678 if (PageAnon(page)) in PageTransCompoundMap()
679 return atomic_read(&page->_mapcount) < 0; in PageTransCompoundMap()
681 head = compound_head(page); in PageTransCompoundMap()
683 return atomic_read(&page->_mapcount) == in PageTransCompoundMap()
692 static inline int PageTransTail(struct page *page) in PageTransTail() argument
694 return PageTail(page); in PageTransTail()
698 * PageDoubleMap indicates that the compound page is mapped with PTEs as well
702 * per small page mapcount accounting (and its overhead from atomic operations)
705 * For the page PageDoubleMap means ->_mapcount in all sub-pages is offset up
727 * mistaken for a page type value. in PAGEFLAG()
739 #define PageType(page, flag) \ in PAGEFLAG() argument
740 ((page->page_type & (PAGE_TYPE_BASE | flag)) == PAGE_TYPE_BASE) in PAGEFLAG()
742 static inline int page_has_type(struct page *page) in PAGEFLAG()
744 return (int)page->page_type < PAGE_MAPCOUNT_RESERVE; in PAGEFLAG()
748 static __always_inline int Page##uname(struct page *page) \
750 return PageType(page, PG_##lname); \
752 static __always_inline void __SetPage##uname(struct page *page) \
754 VM_BUG_ON_PAGE(!PageType(page, 0), page); \
755 page->page_type &= ~PG_##lname; \
757 static __always_inline void __ClearPage##uname(struct page *page) \
759 VM_BUG_ON_PAGE(!Page##uname(page), page); \
760 page->page_type |= PG_##lname; \
764 * PageBuddy() indicates that the page is free and in the buddy system
770 * PageOffline() indicates that the page is logically offline although the
790 * pages allocated with __GFP_ACCOUNT. It gets cleared on page free.
795 * Marks pages in use as page tables.
804 extern bool is_free_buddy_page(struct page *page);
812 static inline int PageSlabPfmemalloc(struct page *page) in PageSlabPfmemalloc() argument
814 VM_BUG_ON_PAGE(!PageSlab(page), page); in PageSlabPfmemalloc()
815 return PageActive(page); in PageSlabPfmemalloc()
818 static inline void SetPageSlabPfmemalloc(struct page *page) in SetPageSlabPfmemalloc() argument
820 VM_BUG_ON_PAGE(!PageSlab(page), page); in SetPageSlabPfmemalloc()
821 SetPageActive(page); in SetPageSlabPfmemalloc()
824 static inline void __ClearPageSlabPfmemalloc(struct page *page) in __ClearPageSlabPfmemalloc() argument
826 VM_BUG_ON_PAGE(!PageSlab(page), page); in __ClearPageSlabPfmemalloc()
827 __ClearPageActive(page); in __ClearPageSlabPfmemalloc()
830 static inline void ClearPageSlabPfmemalloc(struct page *page) in ClearPageSlabPfmemalloc() argument
832 VM_BUG_ON_PAGE(!PageSlab(page), page); in ClearPageSlabPfmemalloc()
833 ClearPageActive(page); in ClearPageSlabPfmemalloc()
843 * Flags checked when a page is freed. Pages being freed should not have
854 * Flags checked when a page is prepped for return by the page allocator.
856 * there has been a kernel bug or struct page corruption.
858 * __PG_HWPOISON is exceptional because it needs to be kept beyond page's
859 * alloc-free cycle to prevent from reusing the page.
867 * page_has_private - Determine if page has private stuff
868 * @page: The page to be checked
870 * Determine if a page has private stuff, indicating that release routines
873 static inline int page_has_private(struct page *page) in page_has_private() argument
875 return !!(page->flags & PAGE_FLAGS_PRIVATE); in page_has_private()