21 static int mcopy_atomic_pte(struct mm_struct *dst_mm,  in mcopy_atomic_pte()  argument
69 	if (mem_cgroup_try_charge(page, dst_mm, GFP_KERNEL, &memcg, false))  in mcopy_atomic_pte()
76 	dst_pte = pte_offset_map_lock(dst_mm, dst_pmd, dst_addr, &ptl);  in mcopy_atomic_pte()
90 	inc_mm_counter(dst_mm, MM_ANONPAGES);  in mcopy_atomic_pte()
95 	set_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte);  in mcopy_atomic_pte()
112 static int mfill_zeropage_pte(struct mm_struct *dst_mm,  in mfill_zeropage_pte()  argument
125 	dst_pte = pte_offset_map_lock(dst_mm, dst_pmd, dst_addr, &ptl);  in mfill_zeropage_pte()
138 	set_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte);  in mfill_zeropage_pte()
173 static __always_inline ssize_t __mcopy_atomic_hugetlb(struct mm_struct *dst_mm,  in __mcopy_atomic_hugetlb()  argument
200 		up_read(&dst_mm->mmap_sem);  in __mcopy_atomic_hugetlb()
224 		dst_vma = find_vma(dst_mm, dst_start);  in __mcopy_atomic_hugetlb()
276 		dst_pte = huge_pte_alloc(dst_mm, dst_addr, huge_page_size(h));  in __mcopy_atomic_hugetlb()
289 		err = hugetlb_mcopy_atomic_pte(dst_mm, dst_pte, dst_vma,  in __mcopy_atomic_hugetlb()
298 			up_read(&dst_mm->mmap_sem);  in __mcopy_atomic_hugetlb()
308 			down_read(&dst_mm->mmap_sem);  in __mcopy_atomic_hugetlb()
328 	up_read(&dst_mm->mmap_sem);  in __mcopy_atomic_hugetlb()
385 extern ssize_t __mcopy_atomic_hugetlb(struct mm_struct *dst_mm,
393 static __always_inline ssize_t mfill_atomic_pte(struct mm_struct *dst_mm,  in mfill_atomic_pte()  argument
415 			err = mcopy_atomic_pte(dst_mm, dst_pmd, dst_vma,  in mfill_atomic_pte()
418 			err = mfill_zeropage_pte(dst_mm, dst_pmd,  in mfill_atomic_pte()
422 			err = shmem_mcopy_atomic_pte(dst_mm, dst_pmd,  in mfill_atomic_pte()
426 			err = shmem_mfill_zeropage_pte(dst_mm, dst_pmd,  in mfill_atomic_pte()
433 static __always_inline ssize_t __mcopy_atomic(struct mm_struct *dst_mm,  in __mcopy_atomic()  argument
462 	down_read(&dst_mm->mmap_sem);  in __mcopy_atomic()
478 	dst_vma = find_vma(dst_mm, dst_start);  in __mcopy_atomic()
506 		return  __mcopy_atomic_hugetlb(dst_mm, dst_vma, dst_start,  in __mcopy_atomic()
527 		dst_pmd = mm_alloc_pmd(dst_mm, dst_addr);  in __mcopy_atomic()
543 		    unlikely(__pte_alloc(dst_mm, dst_pmd))) {  in __mcopy_atomic()
556 		err = mfill_atomic_pte(dst_mm, dst_pmd, dst_vma, dst_addr,  in __mcopy_atomic()
563 			up_read(&dst_mm->mmap_sem);  in __mcopy_atomic()
592 	up_read(&dst_mm->mmap_sem);  in __mcopy_atomic()
602 ssize_t mcopy_atomic(struct mm_struct *dst_mm, unsigned long dst_start,  in mcopy_atomic()  argument
606 	return __mcopy_atomic(dst_mm, dst_start, src_start, len, false,  in mcopy_atomic()
610 ssize_t mfill_zeropage(struct mm_struct *dst_mm, unsigned long start,  in mfill_zeropage()  argument
613 	return __mcopy_atomic(dst_mm, start, 0, len, true, mmap_changing);  in mfill_zeropage()