Lines Matching refs:pmd
183 unsigned long address, pmd_t *pmd, unsigned int flags, in follow_page_pte() argument
203 if (unlikely(pmd_bad(*pmd))) in follow_page_pte()
206 ptep = pte_offset_map_lock(mm, pmd, address, &ptl); in follow_page_pte()
223 migration_entry_wait(mm, pmd, address); in follow_page_pte()
333 pmd_t *pmd, pmdval; in follow_pmd_mask() local
338 pmd = pmd_offset(pudp, address); in follow_pmd_mask()
343 pmdval = READ_ONCE(*pmd); in follow_pmd_mask()
367 pmd_migration_entry_wait(mm, pmd); in follow_pmd_mask()
368 pmdval = READ_ONCE(*pmd); in follow_pmd_mask()
378 ptl = pmd_lock(mm, pmd); in follow_pmd_mask()
379 page = follow_devmap_pmd(vma, address, pmd, flags, &ctx->pgmap); in follow_pmd_mask()
385 return follow_page_pte(vma, address, pmd, flags, &ctx->pgmap); in follow_pmd_mask()
391 ptl = pmd_lock(mm, pmd); in follow_pmd_mask()
392 if (unlikely(pmd_none(*pmd))) { in follow_pmd_mask()
396 if (unlikely(!pmd_present(*pmd))) { in follow_pmd_mask()
400 pmd_migration_entry_wait(mm, pmd); in follow_pmd_mask()
403 if (unlikely(!pmd_trans_huge(*pmd))) { in follow_pmd_mask()
405 return follow_page_pte(vma, address, pmd, flags, &ctx->pgmap); in follow_pmd_mask()
409 page = pmd_page(*pmd); in follow_pmd_mask()
413 split_huge_pmd(vma, pmd, address); in follow_pmd_mask()
414 if (pmd_trans_unstable(pmd)) in follow_pmd_mask()
426 if (pmd_none(*pmd)) in follow_pmd_mask()
430 split_huge_pmd(vma, pmd, address); in follow_pmd_mask()
431 ret = pte_alloc(mm, pmd) ? -ENOMEM : 0; in follow_pmd_mask()
435 follow_page_pte(vma, address, pmd, flags, &ctx->pgmap); in follow_pmd_mask()
437 page = follow_trans_huge_pmd(vma, address, pmd, flags); in follow_pmd_mask()
587 pmd_t *pmd; in get_gate_page() local
606 pmd = pmd_offset(pud, address); in get_gate_page()
607 if (!pmd_present(*pmd)) in get_gate_page()
609 VM_BUG_ON(pmd_trans_huge(*pmd)); in get_gate_page()
610 pte = pte_offset_map(pmd, address); in get_gate_page()
1837 static int gup_pte_range(pmd_t pmd, unsigned long addr, unsigned long end, in gup_pte_range() argument
1844 ptem = ptep = pte_offset_map(&pmd, addr); in gup_pte_range()
1910 static int gup_pte_range(pmd_t pmd, unsigned long addr, unsigned long end, in gup_pte_range() argument
2206 pmd_t pmd = READ_ONCE(*pmdp); in gup_pmd_range() local
2209 if (!pmd_present(pmd)) in gup_pmd_range()
2212 if (unlikely(pmd_trans_huge(pmd) || pmd_huge(pmd) || in gup_pmd_range()
2213 pmd_devmap(pmd))) { in gup_pmd_range()
2219 if (pmd_protnone(pmd)) in gup_pmd_range()
2222 if (!gup_huge_pmd(pmd, pmdp, addr, next, flags, in gup_pmd_range()
2226 } else if (unlikely(is_hugepd(__hugepd(pmd_val(pmd))))) { in gup_pmd_range()
2231 if (!gup_huge_pd(__hugepd(pmd_val(pmd)), addr, in gup_pmd_range()
2234 } else if (!gup_pte_range(pmd, addr, next, flags, pages, nr)) in gup_pmd_range()