Lines Matching refs:page
472 struct page *page; in break_ksm() local
477 page = follow_page(vma, addr, in break_ksm()
479 if (IS_ERR_OR_NULL(page)) in break_ksm()
481 if (PageKsm(page)) in break_ksm()
486 put_page(page); in break_ksm()
552 static struct page *get_mergeable_page(struct rmap_item *rmap_item) in get_mergeable_page()
557 struct page *page; in get_mergeable_page() local
564 page = follow_page(vma, addr, FOLL_GET); in get_mergeable_page()
565 if (IS_ERR_OR_NULL(page)) in get_mergeable_page()
567 if (PageAnon(page)) { in get_mergeable_page()
568 flush_anon_page(vma, page, addr); in get_mergeable_page()
569 flush_dcache_page(page); in get_mergeable_page()
571 put_page(page); in get_mergeable_page()
573 page = NULL; in get_mergeable_page()
576 return page; in get_mergeable_page()
694 static struct page *get_ksm_page(struct stable_node *stable_node, in get_ksm_page()
697 struct page *page; in get_ksm_page() local
705 page = pfn_to_page(kpfn); in get_ksm_page()
706 if (READ_ONCE(page->mapping) != expected_mapping) in get_ksm_page()
719 while (!get_page_unless_zero(page)) { in get_ksm_page()
728 if (!PageSwapCache(page)) in get_ksm_page()
733 if (READ_ONCE(page->mapping) != expected_mapping) { in get_ksm_page()
734 put_page(page); in get_ksm_page()
739 if (!trylock_page(page)) { in get_ksm_page()
740 put_page(page); in get_ksm_page()
744 lock_page(page); in get_ksm_page()
747 if (READ_ONCE(page->mapping) != expected_mapping) { in get_ksm_page()
748 unlock_page(page); in get_ksm_page()
749 put_page(page); in get_ksm_page()
753 return page; in get_ksm_page()
777 struct page *page; in remove_rmap_item_from_tree() local
780 page = get_ksm_page(stable_node, GET_KSM_PAGE_LOCK); in remove_rmap_item_from_tree()
781 if (!page) in remove_rmap_item_from_tree()
785 unlock_page(page); in remove_rmap_item_from_tree()
786 put_page(page); in remove_rmap_item_from_tree()
860 static inline struct stable_node *page_stable_node(struct page *page) in page_stable_node() argument
862 return PageKsm(page) ? page_rmapping(page) : NULL; in page_stable_node()
865 static inline void set_page_stable_node(struct page *page, in set_page_stable_node() argument
868 page->mapping = (void *)((unsigned long)stable_node | PAGE_MAPPING_KSM); in set_page_stable_node()
877 struct page *page; in remove_stable_node() local
880 page = get_ksm_page(stable_node, GET_KSM_PAGE_LOCK); in remove_stable_node()
881 if (!page) { in remove_stable_node()
894 if (!page_mapped(page)) { in remove_stable_node()
903 set_page_stable_node(page, NULL); in remove_stable_node()
908 unlock_page(page); in remove_stable_node()
909 put_page(page); in remove_stable_node()
1023 static u32 calc_checksum(struct page *page) in calc_checksum() argument
1026 void *addr = kmap_atomic(page); in calc_checksum()
1032 static int write_protect_page(struct vm_area_struct *vma, struct page *page, in write_protect_page() argument
1037 .page = page, in write_protect_page()
1044 pvmw.address = page_address_in_vma(page, vma); in write_protect_page()
1048 BUG_ON(PageTransCompound(page)); in write_protect_page()
1065 swapped = PageSwapCache(page); in write_protect_page()
1066 flush_cache_page(vma, pvmw.address, page_to_pfn(page)); in write_protect_page()
1086 if (page_mapcount(page) + 1 + swapped != page_count(page)) { in write_protect_page()
1091 set_page_dirty(page); in write_protect_page()
1119 static int replace_page(struct vm_area_struct *vma, struct page *page, in replace_page() argument
1120 struct page *kpage, pte_t orig_pte) in replace_page()
1131 addr = page_address_in_vma(page, vma); in replace_page()
1179 page_remove_rmap(page, false); in replace_page()
1180 if (!page_mapped(page)) in replace_page()
1181 try_to_free_swap(page); in replace_page()
1182 put_page(page); in replace_page()
1202 struct page *page, struct page *kpage) in try_to_merge_one_page() argument
1207 if (page == kpage) /* ksm page forked */ in try_to_merge_one_page()
1210 if (!PageAnon(page)) in try_to_merge_one_page()
1220 if (!trylock_page(page)) in try_to_merge_one_page()
1223 if (PageTransCompound(page)) { in try_to_merge_one_page()
1224 if (split_huge_page(page)) in try_to_merge_one_page()
1234 if (write_protect_page(vma, page, &orig_pte) == 0) { in try_to_merge_one_page()
1241 set_page_stable_node(page, NULL); in try_to_merge_one_page()
1242 mark_page_accessed(page); in try_to_merge_one_page()
1247 if (!PageDirty(page)) in try_to_merge_one_page()
1248 SetPageDirty(page); in try_to_merge_one_page()
1250 } else if (pages_identical(page, kpage)) in try_to_merge_one_page()
1251 err = replace_page(vma, page, kpage, orig_pte); in try_to_merge_one_page()
1255 munlock_vma_page(page); in try_to_merge_one_page()
1257 unlock_page(page); in try_to_merge_one_page()
1260 page = kpage; /* for final unlock */ in try_to_merge_one_page()
1265 unlock_page(page); in try_to_merge_one_page()
1277 struct page *page, struct page *kpage) in try_to_merge_with_ksm_page() argument
1288 err = try_to_merge_one_page(vma, page, kpage); in try_to_merge_with_ksm_page()
1313 static struct page *try_to_merge_two_pages(struct rmap_item *rmap_item, in try_to_merge_two_pages()
1314 struct page *page, in try_to_merge_two_pages() argument
1316 struct page *tree_page) in try_to_merge_two_pages()
1320 err = try_to_merge_with_ksm_page(rmap_item, page, NULL); in try_to_merge_two_pages()
1323 tree_page, page); in try_to_merge_two_pages()
1331 return err ? NULL : page; in try_to_merge_two_pages()
1354 static struct page *stable_node_dup(struct stable_node **_stable_node_dup, in stable_node_dup()
1361 struct page *_tree_page, *tree_page = NULL; in stable_node_dup()
1500 static struct page *__stable_node_chain(struct stable_node **_stable_node_dup, in __stable_node_chain()
1522 static __always_inline struct page *chain_prune(struct stable_node **s_n_d, in chain_prune()
1529 static __always_inline struct page *chain(struct stable_node **s_n_d, in chain()
1534 struct page *tree_page; in chain()
1551 static struct page *stable_tree_search(struct page *page) in stable_tree_search() argument
1560 page_node = page_stable_node(page); in stable_tree_search()
1563 get_page(page); in stable_tree_search()
1564 return page; in stable_tree_search()
1567 nid = get_kpfn_nid(page_to_pfn(page)); in stable_tree_search()
1574 struct page *tree_page; in stable_tree_search()
1631 ret = memcmp_pages(page, tree_page); in stable_tree_search()
1648 if (page_mapcount(page) > 1) in stable_tree_search()
1707 get_page(page); in stable_tree_search()
1708 return page; in stable_tree_search()
1733 get_page(page); in stable_tree_search()
1735 page = NULL; in stable_tree_search()
1738 page = NULL; in stable_tree_search()
1749 get_page(page); in stable_tree_search()
1751 page = NULL; in stable_tree_search()
1753 page = NULL; in stable_tree_search()
1758 return page; in stable_tree_search()
1803 static struct stable_node *stable_tree_insert(struct page *kpage) in stable_tree_insert()
1821 struct page *tree_page; in stable_tree_insert()
1924 struct page *page, in unstable_tree_search_insert() argument
1925 struct page **tree_pagep) in unstable_tree_search_insert()
1932 nid = get_kpfn_nid(page_to_pfn(page)); in unstable_tree_search_insert()
1938 struct page *tree_page; in unstable_tree_search_insert()
1950 if (page == tree_page) { in unstable_tree_search_insert()
1955 ret = memcmp_pages(page, tree_page); in unstable_tree_search_insert()
2035 static void cmp_and_merge_page(struct page *page, struct rmap_item *rmap_item) in cmp_and_merge_page() argument
2039 struct page *tree_page = NULL; in cmp_and_merge_page()
2041 struct page *kpage; in cmp_and_merge_page()
2046 stable_node = page_stable_node(page); in cmp_and_merge_page()
2067 kpage = stable_tree_search(page); in cmp_and_merge_page()
2068 if (kpage == page && rmap_item->head == stable_node) { in cmp_and_merge_page()
2079 err = try_to_merge_with_ksm_page(rmap_item, page, kpage); in cmp_and_merge_page()
2100 checksum = calc_checksum(page); in cmp_and_merge_page()
2115 err = try_to_merge_one_page(vma, page, in cmp_and_merge_page()
2126 unstable_tree_search_insert(rmap_item, page, &tree_page); in cmp_and_merge_page()
2130 kpage = try_to_merge_two_pages(rmap_item, page, in cmp_and_merge_page()
2142 split = PageTransCompound(page) in cmp_and_merge_page()
2143 && compound_head(page) == compound_head(tree_page); in cmp_and_merge_page()
2180 if (!trylock_page(page)) in cmp_and_merge_page()
2182 split_huge_page(page); in cmp_and_merge_page()
2183 unlock_page(page); in cmp_and_merge_page()
2216 static struct rmap_item *scan_get_next_rmap_item(struct page **page) in scan_get_next_rmap_item() argument
2249 struct page *page; in scan_get_next_rmap_item() local
2253 page = get_ksm_page(stable_node, in scan_get_next_rmap_item()
2255 if (page) in scan_get_next_rmap_item()
2256 put_page(page); in scan_get_next_rmap_item()
2297 *page = follow_page(vma, ksm_scan.address, FOLL_GET); in scan_get_next_rmap_item()
2298 if (IS_ERR_OR_NULL(*page)) { in scan_get_next_rmap_item()
2303 if (PageAnon(*page)) { in scan_get_next_rmap_item()
2304 flush_anon_page(vma, *page, ksm_scan.address); in scan_get_next_rmap_item()
2305 flush_dcache_page(*page); in scan_get_next_rmap_item()
2313 put_page(*page); in scan_get_next_rmap_item()
2317 put_page(*page); in scan_get_next_rmap_item()
2382 struct page *uninitialized_var(page); in ksm_do_scan()
2386 rmap_item = scan_get_next_rmap_item(&page); in ksm_do_scan()
2389 cmp_and_merge_page(page, rmap_item); in ksm_do_scan()
2390 put_page(page); in ksm_do_scan()
2559 struct page *ksm_might_need_to_copy(struct page *page, in ksm_might_need_to_copy() argument
2562 struct anon_vma *anon_vma = page_anon_vma(page); in ksm_might_need_to_copy()
2563 struct page *new_page; in ksm_might_need_to_copy()
2565 if (PageKsm(page)) { in ksm_might_need_to_copy()
2566 if (page_stable_node(page) && in ksm_might_need_to_copy()
2568 return page; /* no need to copy it */ in ksm_might_need_to_copy()
2570 return page; /* no need to copy it */ in ksm_might_need_to_copy()
2572 page->index == linear_page_index(vma, address)) { in ksm_might_need_to_copy()
2573 return page; /* still no need to copy it */ in ksm_might_need_to_copy()
2575 if (!PageUptodate(page)) in ksm_might_need_to_copy()
2576 return page; /* let do_swap_page report the error */ in ksm_might_need_to_copy()
2580 copy_user_highpage(new_page, page, address, vma); in ksm_might_need_to_copy()
2590 void rmap_walk_ksm(struct page *page, struct rmap_walk_control *rwc) in rmap_walk_ksm() argument
2596 VM_BUG_ON_PAGE(!PageKsm(page), page); in rmap_walk_ksm()
2602 VM_BUG_ON_PAGE(!PageLocked(page), page); in rmap_walk_ksm()
2604 stable_node = page_stable_node(page); in rmap_walk_ksm()
2639 if (!rwc->rmap_one(page, vma, addr, rwc->arg)) { in rmap_walk_ksm()
2643 if (rwc->done && rwc->done(page)) { in rmap_walk_ksm()
2654 bool reuse_ksm_page(struct page *page, in reuse_ksm_page() argument
2659 if (WARN_ON(is_zero_pfn(page_to_pfn(page))) || in reuse_ksm_page()
2660 WARN_ON(!page_mapped(page)) || in reuse_ksm_page()
2661 WARN_ON(!PageLocked(page))) { in reuse_ksm_page()
2662 dump_page(page, "reuse_ksm_page"); in reuse_ksm_page()
2667 if (PageSwapCache(page) || !page_stable_node(page)) in reuse_ksm_page()
2670 if (!page_ref_freeze(page, 1)) in reuse_ksm_page()
2673 page_move_anon_rmap(page, vma); in reuse_ksm_page()
2674 page->index = linear_page_index(vma, address); in reuse_ksm_page()
2675 page_ref_unfreeze(page, 1); in reuse_ksm_page()
2680 void ksm_migrate_page(struct page *newpage, struct page *oldpage) in ksm_migrate_page()