Lines Matching refs:page
366 struct page *page; in break_ksm() local
371 page = follow_page(vma, addr, FOLL_GET | FOLL_MIGRATION); in break_ksm()
372 if (IS_ERR_OR_NULL(page)) in break_ksm()
374 if (PageKsm(page)) in break_ksm()
379 put_page(page); in break_ksm()
445 static struct page *page_trans_compound_anon(struct page *page) in page_trans_compound_anon() argument
447 if (PageTransCompound(page)) { in page_trans_compound_anon()
448 struct page *head = compound_head(page); in page_trans_compound_anon()
459 static struct page *get_mergeable_page(struct rmap_item *rmap_item) in get_mergeable_page()
464 struct page *page; in get_mergeable_page() local
471 page = follow_page(vma, addr, FOLL_GET); in get_mergeable_page()
472 if (IS_ERR_OR_NULL(page)) in get_mergeable_page()
474 if (PageAnon(page) || page_trans_compound_anon(page)) { in get_mergeable_page()
475 flush_anon_page(vma, page, addr); in get_mergeable_page()
476 flush_dcache_page(page); in get_mergeable_page()
478 put_page(page); in get_mergeable_page()
479 out: page = NULL; in get_mergeable_page()
482 return page; in get_mergeable_page()
537 static struct page *get_ksm_page(struct stable_node *stable_node, bool lock_it) in get_ksm_page()
539 struct page *page; in get_ksm_page() local
547 page = pfn_to_page(kpfn); in get_ksm_page()
555 if (ACCESS_ONCE(page->mapping) != expected_mapping) in get_ksm_page()
567 while (!get_page_unless_zero(page)) { in get_ksm_page()
576 if (!PageSwapCache(page)) in get_ksm_page()
581 if (ACCESS_ONCE(page->mapping) != expected_mapping) { in get_ksm_page()
582 put_page(page); in get_ksm_page()
587 lock_page(page); in get_ksm_page()
588 if (ACCESS_ONCE(page->mapping) != expected_mapping) { in get_ksm_page()
589 unlock_page(page); in get_ksm_page()
590 put_page(page); in get_ksm_page()
594 return page; in get_ksm_page()
618 struct page *page; in remove_rmap_item_from_tree() local
621 page = get_ksm_page(stable_node, true); in remove_rmap_item_from_tree()
622 if (!page) in remove_rmap_item_from_tree()
626 unlock_page(page); in remove_rmap_item_from_tree()
627 put_page(page); in remove_rmap_item_from_tree()
705 struct page *page; in remove_stable_node() local
708 page = get_ksm_page(stable_node, true); in remove_stable_node()
709 if (!page) { in remove_stable_node()
716 if (WARN_ON_ONCE(page_mapped(page))) { in remove_stable_node()
731 set_page_stable_node(page, NULL); in remove_stable_node()
736 unlock_page(page); in remove_stable_node()
737 put_page(page); in remove_stable_node()
829 static u32 calc_checksum(struct page *page) in calc_checksum() argument
832 void *addr = kmap_atomic(page); in calc_checksum()
838 static int memcmp_pages(struct page *page1, struct page *page2) in memcmp_pages()
851 static inline int pages_identical(struct page *page1, struct page *page2) in pages_identical()
856 static int write_protect_page(struct vm_area_struct *vma, struct page *page, in write_protect_page() argument
868 addr = page_address_in_vma(page, vma); in write_protect_page()
872 BUG_ON(PageTransCompound(page)); in write_protect_page()
878 ptep = page_check_address(page, mm, addr, &ptl, 0); in write_protect_page()
885 swapped = PageSwapCache(page); in write_protect_page()
886 flush_cache_page(vma, addr, page_to_pfn(page)); in write_protect_page()
901 if (page_mapcount(page) + 1 + swapped != page_count(page)) { in write_protect_page()
906 set_page_dirty(page); in write_protect_page()
930 static int replace_page(struct vm_area_struct *vma, struct page *page, in replace_page() argument
931 struct page *kpage, pte_t orig_pte) in replace_page()
942 addr = page_address_in_vma(page, vma); in replace_page()
967 page_remove_rmap(page); in replace_page()
968 if (!page_mapped(page)) in replace_page()
969 try_to_free_swap(page); in replace_page()
970 put_page(page); in replace_page()
980 static int page_trans_compound_anon_split(struct page *page) in page_trans_compound_anon_split() argument
983 struct page *transhuge_head = page_trans_compound_anon(page); in page_trans_compound_anon_split()
1017 struct page *page, struct page *kpage) in try_to_merge_one_page() argument
1022 if (page == kpage) /* ksm page forked */ in try_to_merge_one_page()
1027 if (PageTransCompound(page) && page_trans_compound_anon_split(page)) in try_to_merge_one_page()
1029 BUG_ON(PageTransCompound(page)); in try_to_merge_one_page()
1030 if (!PageAnon(page)) in try_to_merge_one_page()
1040 if (!trylock_page(page)) in try_to_merge_one_page()
1048 if (write_protect_page(vma, page, &orig_pte) == 0) { in try_to_merge_one_page()
1055 set_page_stable_node(page, NULL); in try_to_merge_one_page()
1056 mark_page_accessed(page); in try_to_merge_one_page()
1058 } else if (pages_identical(page, kpage)) in try_to_merge_one_page()
1059 err = replace_page(vma, page, kpage, orig_pte); in try_to_merge_one_page()
1063 munlock_vma_page(page); in try_to_merge_one_page()
1065 unlock_page(page); in try_to_merge_one_page()
1068 page = kpage; /* for final unlock */ in try_to_merge_one_page()
1072 unlock_page(page); in try_to_merge_one_page()
1084 struct page *page, struct page *kpage) in try_to_merge_with_ksm_page() argument
1097 err = try_to_merge_one_page(vma, page, kpage); in try_to_merge_with_ksm_page()
1122 static struct page *try_to_merge_two_pages(struct rmap_item *rmap_item, in try_to_merge_two_pages()
1123 struct page *page, in try_to_merge_two_pages() argument
1125 struct page *tree_page) in try_to_merge_two_pages()
1129 err = try_to_merge_with_ksm_page(rmap_item, page, NULL); in try_to_merge_two_pages()
1132 tree_page, page); in try_to_merge_two_pages()
1140 return err ? NULL : page; in try_to_merge_two_pages()
1152 static struct page *stable_tree_search(struct page *page) in stable_tree_search() argument
1161 page_node = page_stable_node(page); in stable_tree_search()
1164 get_page(page); in stable_tree_search()
1165 return page; in stable_tree_search()
1168 nid = get_kpfn_nid(page_to_pfn(page)); in stable_tree_search()
1175 struct page *tree_page; in stable_tree_search()
1184 ret = memcmp_pages(page, tree_page); in stable_tree_search()
1227 get_page(page); in stable_tree_search()
1228 return page; in stable_tree_search()
1235 get_page(page); in stable_tree_search()
1238 page = NULL; in stable_tree_search()
1242 return page; in stable_tree_search()
1252 static struct stable_node *stable_tree_insert(struct page *kpage) in stable_tree_insert()
1267 struct page *tree_page; in stable_tree_insert()
1324 struct page *page, in unstable_tree_search_insert() argument
1325 struct page **tree_pagep) in unstable_tree_search_insert()
1332 nid = get_kpfn_nid(page_to_pfn(page)); in unstable_tree_search_insert()
1338 struct page *tree_page; in unstable_tree_search_insert()
1350 if (page == tree_page) { in unstable_tree_search_insert()
1355 ret = memcmp_pages(page, tree_page); in unstable_tree_search_insert()
1416 static void cmp_and_merge_page(struct page *page, struct rmap_item *rmap_item) in cmp_and_merge_page() argument
1419 struct page *tree_page = NULL; in cmp_and_merge_page()
1421 struct page *kpage; in cmp_and_merge_page()
1425 stable_node = page_stable_node(page); in cmp_and_merge_page()
1440 kpage = stable_tree_search(page); in cmp_and_merge_page()
1441 if (kpage == page && rmap_item->head == stable_node) { in cmp_and_merge_page()
1449 err = try_to_merge_with_ksm_page(rmap_item, page, kpage); in cmp_and_merge_page()
1469 checksum = calc_checksum(page); in cmp_and_merge_page()
1476 unstable_tree_search_insert(rmap_item, page, &tree_page); in cmp_and_merge_page()
1478 kpage = try_to_merge_two_pages(rmap_item, page, in cmp_and_merge_page()
1536 static struct rmap_item *scan_get_next_rmap_item(struct page **page) in scan_get_next_rmap_item() argument
1570 struct page *page; in scan_get_next_rmap_item() local
1575 page = get_ksm_page(stable_node, false); in scan_get_next_rmap_item()
1576 if (page) in scan_get_next_rmap_item()
1577 put_page(page); in scan_get_next_rmap_item()
1618 *page = follow_page(vma, ksm_scan.address, FOLL_GET); in scan_get_next_rmap_item()
1619 if (IS_ERR_OR_NULL(*page)) { in scan_get_next_rmap_item()
1624 if (PageAnon(*page) || in scan_get_next_rmap_item()
1625 page_trans_compound_anon(*page)) { in scan_get_next_rmap_item()
1626 flush_anon_page(vma, *page, ksm_scan.address); in scan_get_next_rmap_item()
1627 flush_dcache_page(*page); in scan_get_next_rmap_item()
1635 put_page(*page); in scan_get_next_rmap_item()
1639 put_page(*page); in scan_get_next_rmap_item()
1697 struct page *uninitialized_var(page); in ksm_do_scan()
1701 rmap_item = scan_get_next_rmap_item(&page); in ksm_do_scan()
1704 cmp_and_merge_page(page, rmap_item); in ksm_do_scan()
1705 put_page(page); in ksm_do_scan()
1863 struct page *ksm_might_need_to_copy(struct page *page, in ksm_might_need_to_copy() argument
1866 struct anon_vma *anon_vma = page_anon_vma(page); in ksm_might_need_to_copy()
1867 struct page *new_page; in ksm_might_need_to_copy()
1869 if (PageKsm(page)) { in ksm_might_need_to_copy()
1870 if (page_stable_node(page) && in ksm_might_need_to_copy()
1872 return page; /* no need to copy it */ in ksm_might_need_to_copy()
1874 return page; /* no need to copy it */ in ksm_might_need_to_copy()
1876 page->index == linear_page_index(vma, address)) { in ksm_might_need_to_copy()
1877 return page; /* still no need to copy it */ in ksm_might_need_to_copy()
1879 if (!PageUptodate(page)) in ksm_might_need_to_copy()
1880 return page; /* let do_swap_page report the error */ in ksm_might_need_to_copy()
1884 copy_user_highpage(new_page, page, address, vma); in ksm_might_need_to_copy()
1894 int rmap_walk_ksm(struct page *page, struct rmap_walk_control *rwc) in rmap_walk_ksm() argument
1901 VM_BUG_ON_PAGE(!PageKsm(page), page); in rmap_walk_ksm()
1907 VM_BUG_ON_PAGE(!PageLocked(page), page); in rmap_walk_ksm()
1909 stable_node = page_stable_node(page); in rmap_walk_ksm()
1937 ret = rwc->rmap_one(page, vma, in rmap_walk_ksm()
1943 if (rwc->done && rwc->done(page)) { in rmap_walk_ksm()
1957 void ksm_migrate_page(struct page *newpage, struct page *oldpage) in ksm_migrate_page()