223 static int hmm_vma_do_fault(struct mm_walk *walk, unsigned long addr,  in hmm_vma_do_fault()  argument
240 	ret = handle_mm_fault(vma, addr, flags);  in hmm_vma_do_fault()
255 static int hmm_pfns_bad(unsigned long addr,  in hmm_pfns_bad()  argument
264 	i = (addr - range->start) >> PAGE_SHIFT;  in hmm_pfns_bad()
265 	for (; addr < end; addr += PAGE_SIZE, i++)  in hmm_pfns_bad()
283 static int hmm_vma_walk_hole_(unsigned long addr, unsigned long end,  in hmm_vma_walk_hole_()  argument
292 	hmm_vma_walk->last = addr;  in hmm_vma_walk_hole_()
293 	i = (addr - range->start) >> PAGE_SHIFT;  in hmm_vma_walk_hole_()
298 	for (; addr < end; addr += PAGE_SIZE, i++) {  in hmm_vma_walk_hole_()
303 			ret = hmm_vma_do_fault(walk, addr, write_fault,  in hmm_vma_walk_hole_()
378 static int hmm_vma_walk_hole(unsigned long addr, unsigned long end,  in hmm_vma_walk_hole()  argument
387 	i = (addr - range->start) >> PAGE_SHIFT;  in hmm_vma_walk_hole()
388 	npages = (end - addr) >> PAGE_SHIFT;  in hmm_vma_walk_hole()
392 	return hmm_vma_walk_hole_(addr, end, fault, write_fault, walk);  in hmm_vma_walk_hole()
405 static int hmm_vma_handle_pmd(struct mm_walk *walk, unsigned long addr,  in hmm_vma_handle_pmd()  argument
414 	npages = (end - addr) >> PAGE_SHIFT;  in hmm_vma_handle_pmd()
420 		return hmm_vma_walk_hole_(addr, end, fault, write_fault, walk);  in hmm_vma_handle_pmd()
422 	pfn = pmd_pfn(pmd) + ((addr & ~PMD_MASK) >> PAGE_SHIFT);  in hmm_vma_handle_pmd()
423 	for (i = 0; addr < end; addr += PAGE_SIZE, i++, pfn++) {  in hmm_vma_handle_pmd()
441 int hmm_vma_handle_pmd(struct mm_walk *walk, unsigned long addr,
454 static int hmm_vma_handle_pte(struct mm_walk *walk, unsigned long addr,  in hmm_vma_handle_pte()  argument
510 				hmm_vma_walk->last = addr;  in hmm_vma_handle_pte()
511 				migration_entry_wait(walk->mm, pmdp, addr);  in hmm_vma_handle_pte()
549 	return hmm_vma_walk_hole_(addr, end, fault, write_fault, walk);  in hmm_vma_handle_pte()
560 	unsigned long addr = start, i;  in hmm_vma_walk_pmd()  local
574 		i = (addr - range->start) >> PAGE_SHIFT;  in hmm_vma_walk_pmd()
575 		npages = (end - addr) >> PAGE_SHIFT;  in hmm_vma_walk_pmd()
581 			hmm_vma_walk->last = addr;  in hmm_vma_walk_pmd()
604 		i = (addr - range->start) >> PAGE_SHIFT;  in hmm_vma_walk_pmd()
605 		return hmm_vma_handle_pmd(walk, addr, end, &pfns[i], pmd);  in hmm_vma_walk_pmd()
617 	ptep = pte_offset_map(pmdp, addr);  in hmm_vma_walk_pmd()
618 	i = (addr - range->start) >> PAGE_SHIFT;  in hmm_vma_walk_pmd()
619 	for (; addr < end; addr += PAGE_SIZE, ptep++, i++) {  in hmm_vma_walk_pmd()
622 		r = hmm_vma_handle_pte(walk, addr, end, pmdp, ptep, &pfns[i]);  in hmm_vma_walk_pmd()
625 			hmm_vma_walk->last = addr;  in hmm_vma_walk_pmd()
641 	hmm_vma_walk->last = addr;  in hmm_vma_walk_pmd()
661 	unsigned long addr = start, next;  in hmm_vma_walk_pud()  local
679 		i = (addr - range->start) >> PAGE_SHIFT;  in hmm_vma_walk_pud()
680 		npages = (end - addr) >> PAGE_SHIFT;  in hmm_vma_walk_pud()
687 			return hmm_vma_walk_hole_(addr, end, fault,  in hmm_vma_walk_pud()
690 		pfn = pud_pfn(pud) + ((addr & ~PUD_MASK) >> PAGE_SHIFT);  in hmm_vma_walk_pud()
707 	split_huge_pud(walk->vma, pudp, addr);  in hmm_vma_walk_pud()
711 	pmdp = pmd_offset(pudp, addr);  in hmm_vma_walk_pud()
713 		next = pmd_addr_end(addr, end);  in hmm_vma_walk_pud()
714 		ret = hmm_vma_walk_pmd(pmdp, addr, next, walk);  in hmm_vma_walk_pud()
717 	} while (pmdp++, addr = next, addr != end);  in hmm_vma_walk_pud()
730 	unsigned long addr = start, i, pfn;  in hmm_vma_walk_hugetlb_entry()  local
756 	for (; addr < end; addr += PAGE_SIZE, i++, pfn++)  in hmm_vma_walk_hugetlb_entry()
765 		return hmm_vma_walk_hole_(addr, end, fault, write_fault, walk);  in hmm_vma_walk_hugetlb_entry()
775 			   unsigned long addr,  in hmm_pfns_clear()  argument
778 	for (; addr < end; addr += PAGE_SIZE, pfns++)  in hmm_pfns_clear()