Lines Matching refs:vmf
581 static vm_fault_t __do_huge_pmd_anonymous_page(struct vm_fault *vmf, in __do_huge_pmd_anonymous_page() argument
584 struct vm_area_struct *vma = vmf->vma; in __do_huge_pmd_anonymous_page()
586 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; in __do_huge_pmd_anonymous_page()
605 clear_huge_page(page, vmf->address, HPAGE_PMD_NR); in __do_huge_pmd_anonymous_page()
613 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); in __do_huge_pmd_anonymous_page()
614 if (unlikely(!pmd_none(*vmf->pmd))) { in __do_huge_pmd_anonymous_page()
627 spin_unlock(vmf->ptl); in __do_huge_pmd_anonymous_page()
630 ret2 = handle_userfault(vmf, VM_UFFD_MISSING); in __do_huge_pmd_anonymous_page()
639 pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, pgtable); in __do_huge_pmd_anonymous_page()
640 set_pmd_at(vma->vm_mm, haddr, vmf->pmd, entry); in __do_huge_pmd_anonymous_page()
643 spin_unlock(vmf->ptl); in __do_huge_pmd_anonymous_page()
650 spin_unlock(vmf->ptl); in __do_huge_pmd_anonymous_page()
711 vm_fault_t do_huge_pmd_anonymous_page(struct vm_fault *vmf) in do_huge_pmd_anonymous_page() argument
713 struct vm_area_struct *vma = vmf->vma; in do_huge_pmd_anonymous_page()
716 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; in do_huge_pmd_anonymous_page()
724 if (!(vmf->flags & FAULT_FLAG_WRITE) && in do_huge_pmd_anonymous_page()
739 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); in do_huge_pmd_anonymous_page()
741 if (pmd_none(*vmf->pmd)) { in do_huge_pmd_anonymous_page()
744 spin_unlock(vmf->ptl); in do_huge_pmd_anonymous_page()
747 spin_unlock(vmf->ptl); in do_huge_pmd_anonymous_page()
749 ret = handle_userfault(vmf, VM_UFFD_MISSING); in do_huge_pmd_anonymous_page()
753 haddr, vmf->pmd, zero_page); in do_huge_pmd_anonymous_page()
754 spin_unlock(vmf->ptl); in do_huge_pmd_anonymous_page()
757 spin_unlock(vmf->ptl); in do_huge_pmd_anonymous_page()
769 return __do_huge_pmd_anonymous_page(vmf, page, gfp); in do_huge_pmd_anonymous_page()
832 vm_fault_t vmf_insert_pfn_pmd_prot(struct vm_fault *vmf, pfn_t pfn, in vmf_insert_pfn_pmd_prot() argument
835 unsigned long addr = vmf->address & PMD_MASK; in vmf_insert_pfn_pmd_prot()
836 struct vm_area_struct *vma = vmf->vma; in vmf_insert_pfn_pmd_prot()
861 insert_pfn_pmd(vma, addr, vmf->pmd, pfn, pgprot, write, pgtable); in vmf_insert_pfn_pmd_prot()
923 vm_fault_t vmf_insert_pfn_pud_prot(struct vm_fault *vmf, pfn_t pfn, in vmf_insert_pfn_pud_prot() argument
926 unsigned long addr = vmf->address & PUD_MASK; in vmf_insert_pfn_pud_prot()
927 struct vm_area_struct *vma = vmf->vma; in vmf_insert_pfn_pud_prot()
945 insert_pfn_pud(vma, addr, vmf->pud, pfn, pgprot, write); in vmf_insert_pfn_pud_prot()
1228 void huge_pud_set_accessed(struct vm_fault *vmf, pud_t orig_pud) in huge_pud_set_accessed() argument
1232 bool write = vmf->flags & FAULT_FLAG_WRITE; in huge_pud_set_accessed()
1234 vmf->ptl = pud_lock(vmf->vma->vm_mm, vmf->pud); in huge_pud_set_accessed()
1235 if (unlikely(!pud_same(*vmf->pud, orig_pud))) in huge_pud_set_accessed()
1241 haddr = vmf->address & HPAGE_PUD_MASK; in huge_pud_set_accessed()
1242 if (pudp_set_access_flags(vmf->vma, haddr, vmf->pud, entry, write)) in huge_pud_set_accessed()
1243 update_mmu_cache_pud(vmf->vma, vmf->address, vmf->pud); in huge_pud_set_accessed()
1246 spin_unlock(vmf->ptl); in huge_pud_set_accessed()
1250 void huge_pmd_set_accessed(struct vm_fault *vmf, pmd_t orig_pmd) in huge_pmd_set_accessed() argument
1254 bool write = vmf->flags & FAULT_FLAG_WRITE; in huge_pmd_set_accessed()
1256 vmf->ptl = pmd_lock(vmf->vma->vm_mm, vmf->pmd); in huge_pmd_set_accessed()
1257 if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) in huge_pmd_set_accessed()
1263 haddr = vmf->address & HPAGE_PMD_MASK; in huge_pmd_set_accessed()
1264 if (pmdp_set_access_flags(vmf->vma, haddr, vmf->pmd, entry, write)) in huge_pmd_set_accessed()
1265 update_mmu_cache_pmd(vmf->vma, vmf->address, vmf->pmd); in huge_pmd_set_accessed()
1268 spin_unlock(vmf->ptl); in huge_pmd_set_accessed()
1271 vm_fault_t do_huge_pmd_wp_page(struct vm_fault *vmf, pmd_t orig_pmd) in do_huge_pmd_wp_page() argument
1273 struct vm_area_struct *vma = vmf->vma; in do_huge_pmd_wp_page()
1275 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; in do_huge_pmd_wp_page()
1277 vmf->ptl = pmd_lockptr(vma->vm_mm, vmf->pmd); in do_huge_pmd_wp_page()
1283 spin_lock(vmf->ptl); in do_huge_pmd_wp_page()
1285 if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) { in do_huge_pmd_wp_page()
1286 spin_unlock(vmf->ptl); in do_huge_pmd_wp_page()
1296 spin_unlock(vmf->ptl); in do_huge_pmd_wp_page()
1298 spin_lock(vmf->ptl); in do_huge_pmd_wp_page()
1299 if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) { in do_huge_pmd_wp_page()
1300 spin_unlock(vmf->ptl); in do_huge_pmd_wp_page()
1316 if (pmdp_set_access_flags(vma, haddr, vmf->pmd, entry, 1)) in do_huge_pmd_wp_page()
1317 update_mmu_cache_pmd(vma, vmf->address, vmf->pmd); in do_huge_pmd_wp_page()
1319 spin_unlock(vmf->ptl); in do_huge_pmd_wp_page()
1324 spin_unlock(vmf->ptl); in do_huge_pmd_wp_page()
1326 __split_huge_pmd(vma, vmf->pmd, vmf->address, false, NULL); in do_huge_pmd_wp_page()
1411 vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf, pmd_t pmd) in do_huge_pmd_numa_page() argument
1413 struct vm_area_struct *vma = vmf->vma; in do_huge_pmd_numa_page()
1416 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; in do_huge_pmd_numa_page()
1424 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); in do_huge_pmd_numa_page()
1425 if (unlikely(!pmd_same(pmd, *vmf->pmd))) in do_huge_pmd_numa_page()
1433 if (unlikely(pmd_trans_migrating(*vmf->pmd))) { in do_huge_pmd_numa_page()
1434 page = pmd_page(*vmf->pmd); in do_huge_pmd_numa_page()
1437 spin_unlock(vmf->ptl); in do_huge_pmd_numa_page()
1473 spin_unlock(vmf->ptl); in do_huge_pmd_numa_page()
1483 spin_unlock(vmf->ptl); in do_huge_pmd_numa_page()
1487 spin_lock(vmf->ptl); in do_huge_pmd_numa_page()
1488 if (unlikely(!pmd_same(pmd, *vmf->pmd))) { in do_huge_pmd_numa_page()
1532 spin_unlock(vmf->ptl); in do_huge_pmd_numa_page()
1535 vmf->pmd, pmd, vmf->address, page, target_nid); in do_huge_pmd_numa_page()
1550 set_pmd_at(vma->vm_mm, haddr, vmf->pmd, pmd); in do_huge_pmd_numa_page()
1551 update_mmu_cache_pmd(vma, vmf->address, vmf->pmd); in do_huge_pmd_numa_page()
1554 spin_unlock(vmf->ptl); in do_huge_pmd_numa_page()