583 static vm_fault_t __do_huge_pmd_anonymous_page(struct vm_fault *vmf,  in __do_huge_pmd_anonymous_page()  argument
586 	struct vm_area_struct *vma = vmf->vma;  in __do_huge_pmd_anonymous_page()
589 	unsigned long haddr = vmf->address & HPAGE_PMD_MASK;  in __do_huge_pmd_anonymous_page()
606 	clear_huge_page(page, vmf->address, HPAGE_PMD_NR);  in __do_huge_pmd_anonymous_page()
614 	vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);  in __do_huge_pmd_anonymous_page()
615 	if (unlikely(!pmd_none(*vmf->pmd))) {  in __do_huge_pmd_anonymous_page()
628 			spin_unlock(vmf->ptl);  in __do_huge_pmd_anonymous_page()
632 			ret2 = handle_userfault(vmf, VM_UFFD_MISSING);  in __do_huge_pmd_anonymous_page()
642 		pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, pgtable);  in __do_huge_pmd_anonymous_page()
643 		set_pmd_at(vma->vm_mm, haddr, vmf->pmd, entry);  in __do_huge_pmd_anonymous_page()
646 		spin_unlock(vmf->ptl);  in __do_huge_pmd_anonymous_page()
653 	spin_unlock(vmf->ptl);  in __do_huge_pmd_anonymous_page()
715 vm_fault_t do_huge_pmd_anonymous_page(struct vm_fault *vmf)  in do_huge_pmd_anonymous_page()  argument
717 	struct vm_area_struct *vma = vmf->vma;  in do_huge_pmd_anonymous_page()
720 	unsigned long haddr = vmf->address & HPAGE_PMD_MASK;  in do_huge_pmd_anonymous_page()
728 	if (!(vmf->flags & FAULT_FLAG_WRITE) &&  in do_huge_pmd_anonymous_page()
744 		vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);  in do_huge_pmd_anonymous_page()
747 		if (pmd_none(*vmf->pmd)) {  in do_huge_pmd_anonymous_page()
750 				spin_unlock(vmf->ptl);  in do_huge_pmd_anonymous_page()
752 				spin_unlock(vmf->ptl);  in do_huge_pmd_anonymous_page()
753 				ret = handle_userfault(vmf, VM_UFFD_MISSING);  in do_huge_pmd_anonymous_page()
757 						   haddr, vmf->pmd, zero_page);  in do_huge_pmd_anonymous_page()
758 				spin_unlock(vmf->ptl);  in do_huge_pmd_anonymous_page()
762 			spin_unlock(vmf->ptl);  in do_huge_pmd_anonymous_page()
774 	return __do_huge_pmd_anonymous_page(vmf, page, gfp);  in do_huge_pmd_anonymous_page()
824 vm_fault_t vmf_insert_pfn_pmd(struct vm_fault *vmf, pfn_t pfn, bool write)  in vmf_insert_pfn_pmd()  argument
826 	unsigned long addr = vmf->address & PMD_MASK;  in vmf_insert_pfn_pmd()
827 	struct vm_area_struct *vma = vmf->vma;  in vmf_insert_pfn_pmd()
853 	insert_pfn_pmd(vma, addr, vmf->pmd, pfn, pgprot, write, pgtable);  in vmf_insert_pfn_pmd()
902 vm_fault_t vmf_insert_pfn_pud(struct vm_fault *vmf, pfn_t pfn, bool write)  in vmf_insert_pfn_pud()  argument
904 	unsigned long addr = vmf->address & PUD_MASK;  in vmf_insert_pfn_pud()
905 	struct vm_area_struct *vma = vmf->vma;  in vmf_insert_pfn_pud()
924 	insert_pfn_pud(vma, addr, vmf->pud, pfn, pgprot, write);  in vmf_insert_pfn_pud()
1163 void huge_pud_set_accessed(struct vm_fault *vmf, pud_t orig_pud)  in huge_pud_set_accessed()  argument
1167 	bool write = vmf->flags & FAULT_FLAG_WRITE;  in huge_pud_set_accessed()
1169 	vmf->ptl = pud_lock(vmf->vma->vm_mm, vmf->pud);  in huge_pud_set_accessed()
1170 	if (unlikely(!pud_same(*vmf->pud, orig_pud)))  in huge_pud_set_accessed()
1176 	haddr = vmf->address & HPAGE_PUD_MASK;  in huge_pud_set_accessed()
1177 	if (pudp_set_access_flags(vmf->vma, haddr, vmf->pud, entry, write))  in huge_pud_set_accessed()
1178 		update_mmu_cache_pud(vmf->vma, vmf->address, vmf->pud);  in huge_pud_set_accessed()
1181 	spin_unlock(vmf->ptl);  in huge_pud_set_accessed()
1185 void huge_pmd_set_accessed(struct vm_fault *vmf, pmd_t orig_pmd)  in huge_pmd_set_accessed()  argument
1189 	bool write = vmf->flags & FAULT_FLAG_WRITE;  in huge_pmd_set_accessed()
1191 	vmf->ptl = pmd_lock(vmf->vma->vm_mm, vmf->pmd);  in huge_pmd_set_accessed()
1192 	if (unlikely(!pmd_same(*vmf->pmd, orig_pmd)))  in huge_pmd_set_accessed()
1198 	haddr = vmf->address & HPAGE_PMD_MASK;  in huge_pmd_set_accessed()
1199 	if (pmdp_set_access_flags(vmf->vma, haddr, vmf->pmd, entry, write))  in huge_pmd_set_accessed()
1200 		update_mmu_cache_pmd(vmf->vma, vmf->address, vmf->pmd);  in huge_pmd_set_accessed()
1203 	spin_unlock(vmf->ptl);  in huge_pmd_set_accessed()
1206 static vm_fault_t do_huge_pmd_wp_page_fallback(struct vm_fault *vmf,  in do_huge_pmd_wp_page_fallback()  argument
1209 	struct vm_area_struct *vma = vmf->vma;  in do_huge_pmd_wp_page_fallback()
1210 	unsigned long haddr = vmf->address & HPAGE_PMD_MASK;  in do_huge_pmd_wp_page_fallback()
1228 					       vmf->address, page_to_nid(page));  in do_huge_pmd_wp_page_fallback()
1259 	vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);  in do_huge_pmd_wp_page_fallback()
1260 	if (unlikely(!pmd_same(*vmf->pmd, orig_pmd)))  in do_huge_pmd_wp_page_fallback()
1272 	pmdp_huge_clear_flush_notify(vma, haddr, vmf->pmd);  in do_huge_pmd_wp_page_fallback()
1274 	pgtable = pgtable_trans_huge_withdraw(vma->vm_mm, vmf->pmd);  in do_huge_pmd_wp_page_fallback()
1283 		page_add_new_anon_rmap(pages[i], vmf->vma, haddr, false);  in do_huge_pmd_wp_page_fallback()
1286 		vmf->pte = pte_offset_map(&_pmd, haddr);  in do_huge_pmd_wp_page_fallback()
1287 		VM_BUG_ON(!pte_none(*vmf->pte));  in do_huge_pmd_wp_page_fallback()
1288 		set_pte_at(vma->vm_mm, haddr, vmf->pte, entry);  in do_huge_pmd_wp_page_fallback()
1289 		pte_unmap(vmf->pte);  in do_huge_pmd_wp_page_fallback()
1294 	pmd_populate(vma->vm_mm, vmf->pmd, pgtable);  in do_huge_pmd_wp_page_fallback()
1296 	spin_unlock(vmf->ptl);  in do_huge_pmd_wp_page_fallback()
1311 	spin_unlock(vmf->ptl);  in do_huge_pmd_wp_page_fallback()
1323 vm_fault_t do_huge_pmd_wp_page(struct vm_fault *vmf, pmd_t orig_pmd)  in do_huge_pmd_wp_page()  argument
1325 	struct vm_area_struct *vma = vmf->vma;  in do_huge_pmd_wp_page()
1328 	unsigned long haddr = vmf->address & HPAGE_PMD_MASK;  in do_huge_pmd_wp_page()
1333 	vmf->ptl = pmd_lockptr(vma->vm_mm, vmf->pmd);  in do_huge_pmd_wp_page()
1337 	spin_lock(vmf->ptl);  in do_huge_pmd_wp_page()
1338 	if (unlikely(!pmd_same(*vmf->pmd, orig_pmd)))  in do_huge_pmd_wp_page()
1349 		spin_unlock(vmf->ptl);  in do_huge_pmd_wp_page()
1351 		spin_lock(vmf->ptl);  in do_huge_pmd_wp_page()
1352 		if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) {  in do_huge_pmd_wp_page()
1363 		if (pmdp_set_access_flags(vma, haddr, vmf->pmd, entry,  1))  in do_huge_pmd_wp_page()
1364 			update_mmu_cache_pmd(vma, vmf->address, vmf->pmd);  in do_huge_pmd_wp_page()
1371 	spin_unlock(vmf->ptl);  in do_huge_pmd_wp_page()
1384 			split_huge_pmd(vma, vmf->pmd, vmf->address);  in do_huge_pmd_wp_page()
1387 			ret = do_huge_pmd_wp_page_fallback(vmf, orig_pmd, page);  in do_huge_pmd_wp_page()
1389 				split_huge_pmd(vma, vmf->pmd, vmf->address);  in do_huge_pmd_wp_page()
1401 		split_huge_pmd(vma, vmf->pmd, vmf->address);  in do_huge_pmd_wp_page()
1413 		clear_huge_page(new_page, vmf->address, HPAGE_PMD_NR);  in do_huge_pmd_wp_page()
1415 		copy_user_huge_page(new_page, page, vmf->address,  in do_huge_pmd_wp_page()
1423 	spin_lock(vmf->ptl);  in do_huge_pmd_wp_page()
1426 	if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) {  in do_huge_pmd_wp_page()
1427 		spin_unlock(vmf->ptl);  in do_huge_pmd_wp_page()
1435 		pmdp_huge_clear_flush_notify(vma, haddr, vmf->pmd);  in do_huge_pmd_wp_page()
1439 		set_pmd_at(vma->vm_mm, haddr, vmf->pmd, entry);  in do_huge_pmd_wp_page()
1440 		update_mmu_cache_pmd(vma, vmf->address, vmf->pmd);  in do_huge_pmd_wp_page()
1450 	spin_unlock(vmf->ptl);  in do_huge_pmd_wp_page()
1460 	spin_unlock(vmf->ptl);  in do_huge_pmd_wp_page()
1543 vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf, pmd_t pmd)  in do_huge_pmd_numa_page()  argument
1545 	struct vm_area_struct *vma = vmf->vma;  in do_huge_pmd_numa_page()
1548 	unsigned long haddr = vmf->address & HPAGE_PMD_MASK;  in do_huge_pmd_numa_page()
1556 	vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);  in do_huge_pmd_numa_page()
1557 	if (unlikely(!pmd_same(pmd, *vmf->pmd)))  in do_huge_pmd_numa_page()
1565 	if (unlikely(pmd_trans_migrating(*vmf->pmd))) {  in do_huge_pmd_numa_page()
1566 		page = pmd_page(*vmf->pmd);  in do_huge_pmd_numa_page()
1569 		spin_unlock(vmf->ptl);  in do_huge_pmd_numa_page()
1605 		spin_unlock(vmf->ptl);  in do_huge_pmd_numa_page()
1615 	spin_unlock(vmf->ptl);  in do_huge_pmd_numa_page()
1619 	spin_lock(vmf->ptl);  in do_huge_pmd_numa_page()
1620 	if (unlikely(!pmd_same(pmd, *vmf->pmd))) {  in do_huge_pmd_numa_page()
1664 	spin_unlock(vmf->ptl);  in do_huge_pmd_numa_page()
1667 				vmf->pmd, pmd, vmf->address, page, target_nid);  in do_huge_pmd_numa_page()
1682 	set_pmd_at(vma->vm_mm, haddr, vmf->pmd, pmd);  in do_huge_pmd_numa_page()
1683 	update_mmu_cache_pmd(vma, vmf->address, vmf->pmd);  in do_huge_pmd_numa_page()
1686 	spin_unlock(vmf->ptl);  in do_huge_pmd_numa_page()