364 static int break_ksm(struct vm_area_struct *vma, unsigned long addr)  in break_ksm()  argument
371 		page = follow_page(vma, addr, FOLL_GET | FOLL_MIGRATION);  in break_ksm()
375 			ret = handle_mm_fault(vma->vm_mm, vma, addr,  in break_ksm()
415 	struct vm_area_struct *vma;  in find_mergeable_vma()  local
418 	vma = find_vma(mm, addr);  in find_mergeable_vma()
419 	if (!vma || vma->vm_start > addr)  in find_mergeable_vma()
421 	if (!(vma->vm_flags & VM_MERGEABLE) || !vma->anon_vma)  in find_mergeable_vma()
423 	return vma;  in find_mergeable_vma()
430 	struct vm_area_struct *vma;  in break_cow()  local
439 	vma = find_mergeable_vma(mm, addr);  in break_cow()
440 	if (vma)  in break_cow()
441 		break_ksm(vma, addr);  in break_cow()
463 	struct vm_area_struct *vma;  in get_mergeable_page()  local
467 	vma = find_mergeable_vma(mm, addr);  in get_mergeable_page()
468 	if (!vma)  in get_mergeable_page()
471 	page = follow_page(vma, addr, FOLL_GET);  in get_mergeable_page()
475 		flush_anon_page(vma, page, addr);  in get_mergeable_page()
682 static int unmerge_ksm_pages(struct vm_area_struct *vma,  in unmerge_ksm_pages()  argument
689 		if (ksm_test_exit(vma->vm_mm))  in unmerge_ksm_pages()
694 			err = break_ksm(vma, addr);  in unmerge_ksm_pages()
772 	struct vm_area_struct *vma;  in unmerge_and_remove_all_rmap_items()  local
784 		for (vma = mm->mmap; vma; vma = vma->vm_next) {  in unmerge_and_remove_all_rmap_items()
787 			if (!(vma->vm_flags & VM_MERGEABLE) || !vma->anon_vma)  in unmerge_and_remove_all_rmap_items()
789 			err = unmerge_ksm_pages(vma,  in unmerge_and_remove_all_rmap_items()
790 						vma->vm_start, vma->vm_end);  in unmerge_and_remove_all_rmap_items()
856 static int write_protect_page(struct vm_area_struct *vma, struct page *page,  in write_protect_page()  argument
859 	struct mm_struct *mm = vma->vm_mm;  in write_protect_page()
868 	addr = page_address_in_vma(page, vma);  in write_protect_page()
886 		flush_cache_page(vma, addr, page_to_pfn(page));  in write_protect_page()
896 		entry = ptep_clear_flush(vma, addr, ptep);  in write_protect_page()
930 static int replace_page(struct vm_area_struct *vma, struct page *page,  in replace_page()  argument
933 	struct mm_struct *mm = vma->vm_mm;  in replace_page()
942 	addr = page_address_in_vma(page, vma);  in replace_page()
961 	page_add_anon_rmap(kpage, vma, addr);  in replace_page()
963 	flush_cache_page(vma, addr, pte_pfn(*ptep));  in replace_page()
964 	ptep_clear_flush(vma, addr, ptep);  in replace_page()
965 	set_pte_at_notify(mm, addr, ptep, mk_pte(kpage, vma->vm_page_prot));  in replace_page()
1016 static int try_to_merge_one_page(struct vm_area_struct *vma,  in try_to_merge_one_page()  argument
1025 	if (!(vma->vm_flags & VM_MERGEABLE))  in try_to_merge_one_page()
1048 	if (write_protect_page(vma, page, &orig_pte) == 0) {  in try_to_merge_one_page()
1059 			err = replace_page(vma, page, kpage, orig_pte);  in try_to_merge_one_page()
1062 	if ((vma->vm_flags & VM_LOCKED) && kpage && !err) {  in try_to_merge_one_page()
1087 	struct vm_area_struct *vma;  in try_to_merge_with_ksm_page()  local
1093 	vma = find_vma(mm, rmap_item->address);  in try_to_merge_with_ksm_page()
1094 	if (!vma || vma->vm_start > rmap_item->address)  in try_to_merge_with_ksm_page()
1097 	err = try_to_merge_one_page(vma, page, kpage);  in try_to_merge_with_ksm_page()
1105 	rmap_item->anon_vma = vma->anon_vma;  in try_to_merge_with_ksm_page()
1106 	get_anon_vma(vma->anon_vma);  in try_to_merge_with_ksm_page()
1540 	struct vm_area_struct *vma;  in scan_get_next_rmap_item()  local
1603 		vma = NULL;  in scan_get_next_rmap_item()
1605 		vma = find_vma(mm, ksm_scan.address);  in scan_get_next_rmap_item()
1607 	for (; vma; vma = vma->vm_next) {  in scan_get_next_rmap_item()
1608 		if (!(vma->vm_flags & VM_MERGEABLE))  in scan_get_next_rmap_item()
1610 		if (ksm_scan.address < vma->vm_start)  in scan_get_next_rmap_item()
1611 			ksm_scan.address = vma->vm_start;  in scan_get_next_rmap_item()
1612 		if (!vma->anon_vma)  in scan_get_next_rmap_item()
1613 			ksm_scan.address = vma->vm_end;  in scan_get_next_rmap_item()
1615 		while (ksm_scan.address < vma->vm_end) {  in scan_get_next_rmap_item()
1618 			*page = follow_page(vma, ksm_scan.address, FOLL_GET);  in scan_get_next_rmap_item()
1626 				flush_anon_page(vma, *page, ksm_scan.address);  in scan_get_next_rmap_item()
1739 int ksm_madvise(struct vm_area_struct *vma, unsigned long start,  in ksm_madvise()  argument
1742 	struct mm_struct *mm = vma->vm_mm;  in ksm_madvise()
1773 		if (vma->anon_vma) {  in ksm_madvise()
1774 			err = unmerge_ksm_pages(vma, start, end);  in ksm_madvise()
1864 			struct vm_area_struct *vma, unsigned long address)  in ksm_might_need_to_copy()  argument
1875 	} else if (anon_vma->root == vma->anon_vma->root &&  in ksm_might_need_to_copy()
1876 		 page->index == linear_page_index(vma, address)) {  in ksm_might_need_to_copy()
1882 	new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, address);  in ksm_might_need_to_copy()
1884 		copy_user_highpage(new_page, page, address, vma);  in ksm_might_need_to_copy()
1916 		struct vm_area_struct *vma;  in rmap_walk_ksm()  local
1921 			vma = vmac->vma;  in rmap_walk_ksm()
1922 			if (rmap_item->address < vma->vm_start ||  in rmap_walk_ksm()
1923 			    rmap_item->address >= vma->vm_end)  in rmap_walk_ksm()
1931 			if ((rmap_item->mm == vma->vm_mm) == search_new_forks)  in rmap_walk_ksm()
1934 			if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg))  in rmap_walk_ksm()
1937 			ret = rwc->rmap_one(page, vma,  in rmap_walk_ksm()