Lines Matching full:page
13 * Handles page cache pages in various states. The tricky part
14 * here is that we can access any page asynchronously in respect to
26 * - The case actually shows up as a frequent (top 10) page state in
27 * tools/vm/page-types when running a real workload.
38 #include <linux/page-flags.h>
39 #include <linux/kernel-page-flags.h>
58 #include <linux/page-isolation.h>
68 static bool page_handle_poison(struct page *page, bool hugepage_or_freepage, bool release) in page_handle_poison() argument
75 if (dissolve_free_huge_page(page) || !take_page_off_buddy(page)) in page_handle_poison()
77 * We could fail to take off the target page from buddy in page_handle_poison()
78 * for example due to racy page allocaiton, but that's in page_handle_poison()
79 * acceptable because soft-offlined page is not broken in page_handle_poison()
86 SetPageHWPoison(page); in page_handle_poison()
88 put_page(page); in page_handle_poison()
89 page_ref_inc(page); in page_handle_poison()
108 static int hwpoison_filter_dev(struct page *p) in hwpoison_filter_dev()
138 static int hwpoison_filter_flags(struct page *p) in hwpoison_filter_flags()
157 * can only guarantee that the page either belongs to the memcg tasks, or is
158 * a freed page.
163 static int hwpoison_filter_task(struct page *p) in hwpoison_filter_task()
174 static int hwpoison_filter_task(struct page *p) { return 0; } in hwpoison_filter_task()
177 int hwpoison_filter(struct page *p) in hwpoison_filter()
194 int hwpoison_filter(struct page *p) in hwpoison_filter()
203 * Kill all processes that have a poisoned page mapped and then isolate
204 * the page.
207 * Find all processes having the page mapped and kill them.
208 * But we keep a page reference around so that the page is not
210 * Then stash the page away
232 * Send all the processes who have the page mapped a signal.
266 * When a unknown page type is encountered drain as many buffers as possible
267 * in the hope to turn the page into a LRU or free page, which we can handle.
269 void shake_page(struct page *p, int access) in shake_page()
292 static unsigned long dev_pagemap_mapping_shift(struct page *page, in dev_pagemap_mapping_shift() argument
295 unsigned long address = vma_address(page, vma); in dev_pagemap_mapping_shift()
335 static void add_to_kill(struct task_struct *tsk, struct page *p, in add_to_kill()
357 * ZONE_DEVICE. Indeed, when a devdax page is mmapped N times in add_to_kill()
359 * contain mappings for the page, but at least one VMA does. in add_to_kill()
361 * has a mapping for the page. in add_to_kill()
397 …pr_err("Memory failure: %#lx: forcibly killing %s:%d because of failure to unmap corrupted page\n", in kill_procs()
444 * to be signaled when some page under the process is hwpoisoned.
470 * Collect processes when the error hit an anonymous page.
472 static void collect_procs_anon(struct page *page, struct list_head *to_kill, in collect_procs_anon() argument
480 av = page_lock_anon_vma_read(page); in collect_procs_anon()
484 pgoff = page_to_pgoff(page); in collect_procs_anon()
495 if (!page_mapped_in_vma(page, vma)) in collect_procs_anon()
498 add_to_kill(t, page, vma, to_kill); in collect_procs_anon()
506 * Collect processes when the error hit a file mapped page.
508 static void collect_procs_file(struct page *page, struct list_head *to_kill, in collect_procs_file() argument
513 struct address_space *mapping = page->mapping; in collect_procs_file()
518 pgoff = page_to_pgoff(page); in collect_procs_file()
528 * the page but the corrupted page is not necessarily in collect_procs_file()
534 add_to_kill(t, page, vma, to_kill); in collect_procs_file()
542 * Collect the processes who have the corrupted page mapped to kill.
544 static void collect_procs(struct page *page, struct list_head *tokill, in collect_procs() argument
547 if (!page->mapping) in collect_procs()
550 if (PageAnon(page)) in collect_procs()
551 collect_procs_anon(page, tokill, force_early); in collect_procs()
553 collect_procs_file(page, tokill, force_early); in collect_procs()
564 [MF_MSG_KERNEL] = "reserved kernel page",
565 [MF_MSG_KERNEL_HIGH_ORDER] = "high-order kernel page",
566 [MF_MSG_SLAB] = "kernel slab page",
567 [MF_MSG_DIFFERENT_COMPOUND] = "different compound page after locking",
568 [MF_MSG_POISONED_HUGE] = "huge page already hardware poisoned",
569 [MF_MSG_HUGE] = "huge page",
570 [MF_MSG_FREE_HUGE] = "free huge page",
571 [MF_MSG_NON_PMD_HUGE] = "non-pmd-sized huge page",
572 [MF_MSG_UNMAP_FAILED] = "unmapping failed page",
573 [MF_MSG_DIRTY_SWAPCACHE] = "dirty swapcache page",
574 [MF_MSG_CLEAN_SWAPCACHE] = "clean swapcache page",
575 [MF_MSG_DIRTY_MLOCKED_LRU] = "dirty mlocked LRU page",
576 [MF_MSG_CLEAN_MLOCKED_LRU] = "clean mlocked LRU page",
577 [MF_MSG_DIRTY_UNEVICTABLE_LRU] = "dirty unevictable LRU page",
578 [MF_MSG_CLEAN_UNEVICTABLE_LRU] = "clean unevictable LRU page",
579 [MF_MSG_DIRTY_LRU] = "dirty LRU page",
580 [MF_MSG_CLEAN_LRU] = "clean LRU page",
581 [MF_MSG_TRUNCATED_LRU] = "already truncated LRU page",
582 [MF_MSG_BUDDY] = "free buddy page",
583 [MF_MSG_BUDDY_2ND] = "free buddy page (2nd try)",
584 [MF_MSG_DAX] = "dax page",
586 [MF_MSG_UNKNOWN] = "unknown page",
590 * XXX: It is possible that a page is isolated from LRU cache,
591 * and then kept in swap cache or failed to remove from page cache.
592 * The page count will stop it from being freed by unpoison.
595 static int delete_from_lru_cache(struct page *p) in delete_from_lru_cache()
599 * Clear sensible page flags, so that the buddy system won't in delete_from_lru_cache()
600 * complain when the page is unpoison-and-freed. in delete_from_lru_cache()
606 * Poisoned page might never drop its ref count to 0 so we have in delete_from_lru_cache()
612 * drop the page count elevated by isolate_lru_page() in delete_from_lru_cache()
620 static int truncate_error_page(struct page *p, unsigned long pfn, in truncate_error_page()
629 pr_info("Memory failure: %#lx: Failed to punch page: %d\n", in truncate_error_page()
654 * Error hit kernel page.
658 static int me_kernel(struct page *p, unsigned long pfn) in me_kernel()
664 * Page in unknown state. Do nothing.
666 static int me_unknown(struct page *p, unsigned long pfn) in me_unknown()
668 pr_err("Memory failure: %#lx: Unknown page state\n", pfn); in me_unknown()
673 * Clean (or cleaned) page cache page.
675 static int me_pagecache_clean(struct page *p, unsigned long pfn) in me_pagecache_clean()
689 * Now truncate the page in the page cache. This is really in me_pagecache_clean()
698 * Page has been teared down in the meanwhile in me_pagecache_clean()
712 * Dirty pagecache page
713 * Issues: when the error hit a hole page the error is not properly
716 static int me_pagecache_dirty(struct page *p, unsigned long pfn) in me_pagecache_dirty()
735 * and then through the PageError flag in the page. in me_pagecache_dirty()
742 * when the page is reread or dropped. If an in me_pagecache_dirty()
745 * and the page is dropped between then the error in me_pagecache_dirty()
750 * report through AS_EIO) or when the page is dropped in me_pagecache_dirty()
766 * Dirty swap cache page is tricky to handle. The page could live both in page
767 * cache and swap cache(ie. page is freshly swapped in). So it could be
775 * a later page fault, we know the application is accessing
779 * Clean swap cache pages can be directly isolated. A later page fault will
782 static int me_swapcache_dirty(struct page *p, unsigned long pfn) in me_swapcache_dirty()
794 static int me_swapcache_clean(struct page *p, unsigned long pfn) in me_swapcache_clean()
807 * - Error on hugepage is contained in hugepage unit (not in raw page unit.)
808 * To narrow down kill region to one page, we need to break up pmd.
810 static int me_huge_page(struct page *p, unsigned long pfn) in me_huge_page()
813 struct page *hpage = compound_head(p); in me_huge_page()
840 * Various page states we can handle.
842 * A page state is defined by its current page->flags bits.
845 * This is quite tricky because we can access page at any time
865 int (*action)(struct page *p, unsigned long pfn);
874 * Could in theory check if slab page is free or if we can drop
911 * setting PG_dirty outside page lock. See also comment above set_page_dirty().
922 static int page_action(struct page_state *ps, struct page *p, in page_action()
940 /* Could do more checks here if page looks ok */ in page_action()
942 * Could adjust zone counters here to correct for the missing page. in page_action()
950 * @page: raw error page (hit by memory error)
955 static int get_hwpoison_page(struct page *page) in get_hwpoison_page() argument
957 struct page *head = compound_head(page); in get_hwpoison_page()
964 * tries to touch the "partially handled" page. in get_hwpoison_page()
968 page_to_pfn(page)); in get_hwpoison_page()
974 if (head == compound_head(page)) in get_hwpoison_page()
978 page_to_pfn(page)); in get_hwpoison_page()
989 static bool hwpoison_user_mappings(struct page *p, unsigned long pfn, in hwpoison_user_mappings()
990 int flags, struct page **hpagep) in hwpoison_user_mappings()
997 struct page *hpage = *hpagep; in hwpoison_user_mappings()
1022 pr_err("Memory failure: %#lx: keeping poisoned page in swap cache\n", in hwpoison_user_mappings()
1028 * Propagate the dirty bit from PTEs to struct page first, because we in hwpoison_user_mappings()
1029 * need this to decide if we should kill or just drop the page. in hwpoison_user_mappings()
1031 * be called inside page lock (it's recommended but not enforced). in hwpoison_user_mappings()
1041 pr_info("Memory failure: %#lx: corrupted page was clean: dropped without side effects\n", in hwpoison_user_mappings()
1047 * First collect all the processes that have the page in hwpoison_user_mappings()
1074 pr_info("Memory failure: %#lx: could not lock mapping for mapped huge page\n", pfn); in hwpoison_user_mappings()
1082 pr_err("Memory failure: %#lx: failed to unmap page (mapcount=%d)\n", in hwpoison_user_mappings()
1086 * try_to_unmap() might put mlocked page in lru cache, so call in hwpoison_user_mappings()
1094 * struct page and all unmaps done we can decide if in hwpoison_user_mappings()
1095 * killing is needed or not. Only kill when the page in hwpoison_user_mappings()
1108 static int identify_page_state(unsigned long pfn, struct page *p, in identify_page_state()
1114 * The first check uses the current page flags which may not have any in identify_page_state()
1115 * relevant information. The second check with the saved page flags is in identify_page_state()
1116 * carried out only if the first check can't determine the page status. in identify_page_state()
1131 static int try_to_split_thp_page(struct page *page, const char *msg) in try_to_split_thp_page() argument
1133 lock_page(page); in try_to_split_thp_page()
1134 if (!PageAnon(page) || unlikely(split_huge_page(page))) { in try_to_split_thp_page()
1135 unsigned long pfn = page_to_pfn(page); in try_to_split_thp_page()
1137 unlock_page(page); in try_to_split_thp_page()
1138 if (!PageAnon(page)) in try_to_split_thp_page()
1142 put_page(page); in try_to_split_thp_page()
1145 unlock_page(page); in try_to_split_thp_page()
1152 struct page *p = pfn_to_page(pfn); in memory_failure_hugetlb()
1153 struct page *head = compound_head(p); in memory_failure_hugetlb()
1201 * - other mm code walking over page table is aware of pud-aligned in memory_failure_hugetlb()
1225 struct page *page = pfn_to_page(pfn); in memory_failure_dev_pagemap() local
1238 put_page(page); in memory_failure_dev_pagemap()
1249 * lock_page(), but dax pages do not use the page lock. This in memory_failure_dev_pagemap()
1253 cookie = dax_lock_page(page); in memory_failure_dev_pagemap()
1257 if (hwpoison_filter(page)) { in memory_failure_dev_pagemap()
1271 * Use this flag as an indication that the dax page has been in memory_failure_dev_pagemap()
1274 SetPageHWPoison(page); in memory_failure_dev_pagemap()
1278 * different physical page at a given virtual address, so all in memory_failure_dev_pagemap()
1283 collect_procs(page, &tokill, flags & MF_ACTION_REQUIRED); in memory_failure_dev_pagemap()
1295 start = (page->index << PAGE_SHIFT) & ~(size - 1); in memory_failure_dev_pagemap()
1296 unmap_mapping_range(page->mapping, start, size, 0); in memory_failure_dev_pagemap()
1301 dax_unlock_page(page, cookie); in memory_failure_dev_pagemap()
1310 * memory_failure - Handle memory failure of a page.
1311 * @pfn: Page Number of the corrupted page
1316 * of a page. It tries its best to recover, which includes
1328 struct page *p; in memory_failure()
1329 struct page *hpage; in memory_failure()
1330 struct page *orig_head; in memory_failure()
1337 panic("Memory failure on page %lx", pfn); in memory_failure()
1371 * 1) it's a free page, and therefore in safe hand: in memory_failure()
1373 * 2) it's part of a non-compound high order page. in memory_failure()
1375 * R/W the page; let's pray that the page has been in memory_failure()
1377 * In fact it's dangerous to directly bump up page count from 0, in memory_failure()
1405 * walked by the page reclaim code, however that's not a big loss. in memory_failure()
1420 * The page could have changed compound pages during the locking. in memory_failure()
1430 * We use page flags to determine what action should be taken, but in memory_failure()
1432 * example is an mlocked page, where PG_mlocked is cleared by in memory_failure()
1433 * page_remove_rmap() in try_to_unmap_one(). So to determine page status in memory_failure()
1434 * correctly, we save a copy of the page flags at this time. in memory_failure()
1439 * unpoison always clear PG_hwpoison inside page lock in memory_failure()
1457 * __munlock_pagevec may clear a writeback page's LRU flag without in memory_failure()
1458 * page_lock. We need wait writeback completion for this page or it in memory_failure()
1472 * Abort on fail: __delete_from_page_cache() assumes unmapped page. in memory_failure()
1517 * memory_failure_queue - Schedule handling memory failure of a page.
1518 * @pfn: Page Number of the corrupted page
1522 * when it detects hardware memory corruption of a page. It schedules
1523 * the recovering of error page, including dropping pages, killing
1610 * unpoison_memory - Unpoison a previously poisoned page
1611 * @pfn: Page number of the to be unpoisoned page
1613 * Software-unpoison a page that has been poisoned by
1623 struct page *page; in unpoison_memory() local
1624 struct page *p; in unpoison_memory()
1633 page = compound_head(p); in unpoison_memory()
1636 unpoison_pr_info("Unpoison: Page was already unpoisoned %#lx\n", in unpoison_memory()
1641 if (page_count(page) > 1) { in unpoison_memory()
1642 unpoison_pr_info("Unpoison: Someone grabs the hwpoison page %#lx\n", in unpoison_memory()
1647 if (page_mapped(page)) { in unpoison_memory()
1648 unpoison_pr_info("Unpoison: Someone maps the hwpoison page %#lx\n", in unpoison_memory()
1653 if (page_mapping(page)) { in unpoison_memory()
1654 unpoison_pr_info("Unpoison: the hwpoison page has non-NULL mapping %#lx\n", in unpoison_memory()
1661 * worked by memory_failure() and the page lock is not held yet. in unpoison_memory()
1664 if (!PageHuge(page) && PageTransHuge(page)) { in unpoison_memory()
1673 unpoison_pr_info("Unpoison: Software-unpoisoned free page %#lx\n", in unpoison_memory()
1678 lock_page(page); in unpoison_memory()
1680 * This test is racy because PG_hwpoison is set outside of page lock. in unpoison_memory()
1682 * the PG_hwpoison page will be caught and isolated on the entrance to in unpoison_memory()
1683 * the free buddy page pool. in unpoison_memory()
1685 if (TestClearPageHWPoison(page)) { in unpoison_memory()
1686 unpoison_pr_info("Unpoison: Software-unpoisoned page %#lx\n", in unpoison_memory()
1691 unlock_page(page); in unpoison_memory()
1693 put_page(page); in unpoison_memory()
1695 put_page(page); in unpoison_memory()
1702 * Safely get reference count of an arbitrary page.
1703 * Returns 0 for a free page, -EIO for a zero refcount page
1704 * that is not free, and 1 for any other page type.
1705 * For 1 the page is returned with increased page count, otherwise not.
1707 static int __get_any_page(struct page *p, unsigned long pfn, int flags) in __get_any_page()
1715 * When the target page is a free hugepage, just remove it in __get_any_page()
1720 pr_info("%s: %#lx free huge page\n", __func__, pfn); in __get_any_page()
1723 pr_info("%s: %#lx free buddy page\n", __func__, pfn); in __get_any_page()
1729 pr_info("%s: %#lx: unknown zero refcount page type %lx\n", in __get_any_page()
1734 /* Not a free page */ in __get_any_page()
1740 static int get_any_page(struct page *page, unsigned long pfn, int flags) in get_any_page() argument
1742 int ret = __get_any_page(page, pfn, flags); in get_any_page()
1745 ret = __get_any_page(page, pfn, flags); in get_any_page()
1747 if (ret == 1 && !PageHuge(page) && in get_any_page()
1748 !PageLRU(page) && !__PageMovable(page)) { in get_any_page()
1752 put_page(page); in get_any_page()
1753 shake_page(page, 1); in get_any_page()
1758 ret = __get_any_page(page, pfn, 0); in get_any_page()
1759 if (ret == 1 && !PageLRU(page)) { in get_any_page()
1760 /* Drop page reference which is from __get_any_page() */ in get_any_page()
1761 put_page(page); in get_any_page()
1762 pr_info("soft_offline: %#lx: unknown non LRU page type %lx (%pGp)\n", in get_any_page()
1763 pfn, page->flags, &page->flags); in get_any_page()
1770 static bool isolate_page(struct page *page, struct list_head *pagelist) in isolate_page() argument
1773 bool lru = PageLRU(page); in isolate_page()
1775 if (PageHuge(page)) { in isolate_page()
1776 isolated = isolate_huge_page(page, pagelist); in isolate_page()
1779 isolated = !isolate_lru_page(page); in isolate_page()
1781 isolated = !isolate_movable_page(page, ISOLATE_UNEVICTABLE); in isolate_page()
1784 list_add(&page->lru, pagelist); in isolate_page()
1788 inc_node_page_state(page, NR_ISOLATED_ANON + in isolate_page()
1789 page_is_file_lru(page)); in isolate_page()
1792 * If we succeed to isolate the page, we grabbed another refcount on in isolate_page()
1793 * the page, so we can safely drop the one we got from get_any_pages(). in isolate_page()
1794 * If we failed to isolate the page, it means that we cannot go further in isolate_page()
1798 put_page(page); in isolate_page()
1804 * If the page is a non-dirty unmapped page-cache page, it simply invalidates.
1805 * If the page is mapped, it migrates the contents over.
1807 static int __soft_offline_page(struct page *page) in __soft_offline_page() argument
1810 unsigned long pfn = page_to_pfn(page); in __soft_offline_page()
1811 struct page *hpage = compound_head(page); in __soft_offline_page()
1812 char const *msg_page[] = {"page", "hugepage"}; in __soft_offline_page()
1813 bool huge = PageHuge(page); in __soft_offline_page()
1821 * Check PageHWPoison again inside page lock because PageHWPoison in __soft_offline_page()
1822 * is set by memory_failure() outside page lock. Note that in __soft_offline_page()
1823 * memory_failure() also double-checks PageHWPoison inside page lock, in __soft_offline_page()
1826 lock_page(page); in __soft_offline_page()
1827 if (!PageHuge(page)) in __soft_offline_page()
1828 wait_on_page_writeback(page); in __soft_offline_page()
1829 if (PageHWPoison(page)) { in __soft_offline_page()
1830 unlock_page(page); in __soft_offline_page()
1831 put_page(page); in __soft_offline_page()
1832 pr_info("soft offline: %#lx page already poisoned\n", pfn); in __soft_offline_page()
1836 if (!PageHuge(page)) in __soft_offline_page()
1839 * non dirty unmapped page cache pages. in __soft_offline_page()
1841 ret = invalidate_inode_page(page); in __soft_offline_page()
1842 unlock_page(page); in __soft_offline_page()
1850 page_handle_poison(page, false, true); in __soft_offline_page()
1860 if (!page_handle_poison(page, huge, release)) in __soft_offline_page()
1867 pfn, msg_page[huge], ret, page->flags, &page->flags); in __soft_offline_page()
1872 pr_info("soft offline: %#lx: %s isolation failed, page count %d, type %lx (%pGp)\n", in __soft_offline_page()
1873 pfn, msg_page[huge], page_count(page), page->flags, &page->flags); in __soft_offline_page()
1879 static int soft_offline_in_use_page(struct page *page) in soft_offline_in_use_page() argument
1881 struct page *hpage = compound_head(page); in soft_offline_in_use_page()
1883 if (!PageHuge(page) && PageTransHuge(hpage)) in soft_offline_in_use_page()
1884 if (try_to_split_thp_page(page, "soft offline") < 0) in soft_offline_in_use_page()
1886 return __soft_offline_page(page); in soft_offline_in_use_page()
1889 static int soft_offline_free_page(struct page *page) in soft_offline_free_page() argument
1893 if (!page_handle_poison(page, true, false)) in soft_offline_free_page()
1899 static void put_ref_page(struct page *page) in put_ref_page() argument
1901 if (page) in put_ref_page()
1902 put_page(page); in put_ref_page()
1906 * soft_offline_page - Soft offline a page.
1912 * Soft offline a page, by migration or invalidation,
1914 * a page is not corrupted yet (so it's still valid to access),
1931 struct page *page, *ref_page = NULL; in soft_offline_page() local
1941 page = pfn_to_online_page(pfn); in soft_offline_page()
1942 if (!page) { in soft_offline_page()
1947 if (PageHWPoison(page)) { in soft_offline_page()
1948 pr_info("soft offline: %#lx page already poisoned\n", pfn); in soft_offline_page()
1955 ret = get_any_page(page, pfn, flags); in soft_offline_page()
1959 ret = soft_offline_in_use_page(page); in soft_offline_page()
1961 if (soft_offline_free_page(page) && try_again) { in soft_offline_page()