1302 struct page *get_dump_page(unsigned long addr)  in get_dump_page()  argument
1307 	if (__get_user_pages(current, current->mm, addr, 1,  in get_dump_page()
1311 	flush_cache_page(vma, addr, page_to_pfn(page));  in get_dump_page()
1811 static int gup_pte_range(pmd_t pmd, unsigned long addr, unsigned long end,  in gup_pte_range()  argument
1818 	ptem = ptep = pte_offset_map(&pmd, addr);  in gup_pte_range()
1863 	} while (ptep++, addr += PAGE_SIZE, addr != end);  in gup_pte_range()
1884 static int gup_pte_range(pmd_t pmd, unsigned long addr, unsigned long end,  in gup_pte_range()  argument
1892 static int __gup_device_huge(unsigned long pfn, unsigned long addr,  in __gup_device_huge()  argument
1911 	} while (addr += PAGE_SIZE, addr != end);  in __gup_device_huge()
1918 static int __gup_device_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr,  in __gup_device_huge_pmd()  argument
1924 	fault_pfn = pmd_pfn(orig) + ((addr & ~PMD_MASK) >> PAGE_SHIFT);  in __gup_device_huge_pmd()
1925 	if (!__gup_device_huge(fault_pfn, addr, end, pages, nr))  in __gup_device_huge_pmd()
1935 static int __gup_device_huge_pud(pud_t orig, pud_t *pudp, unsigned long addr,  in __gup_device_huge_pud()  argument
1941 	fault_pfn = pud_pfn(orig) + ((addr & ~PUD_MASK) >> PAGE_SHIFT);  in __gup_device_huge_pud()
1942 	if (!__gup_device_huge(fault_pfn, addr, end, pages, nr))  in __gup_device_huge_pud()
1952 static int __gup_device_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr,  in __gup_device_huge_pmd()  argument
1959 static int __gup_device_huge_pud(pud_t pud, pud_t *pudp, unsigned long addr,  in __gup_device_huge_pud()  argument
1968 static unsigned long hugepte_addr_end(unsigned long addr, unsigned long end,  in hugepte_addr_end()  argument
1971 	unsigned long __boundary = (addr + sz) & ~(sz-1);  in hugepte_addr_end()
1975 static int gup_hugepte(pte_t *ptep, unsigned long sz, unsigned long addr,  in gup_hugepte()  argument
1984 	pte_end = (addr + sz) & ~(sz-1);  in gup_hugepte()
1999 	page = head + ((addr & (sz-1)) >> PAGE_SHIFT);  in gup_hugepte()
2006 	} while (addr += PAGE_SIZE, addr != end);  in gup_hugepte()
2026 static int gup_huge_pd(hugepd_t hugepd, unsigned long addr,  in gup_huge_pd()  argument
2034 	ptep = hugepte_offset(hugepd, addr, pdshift);  in gup_huge_pd()
2036 		next = hugepte_addr_end(addr, end, sz);  in gup_huge_pd()
2037 		if (!gup_hugepte(ptep, sz, addr, end, flags, pages, nr))  in gup_huge_pd()
2039 	} while (ptep++, addr = next, addr != end);  in gup_huge_pd()
2044 static inline int gup_huge_pd(hugepd_t hugepd, unsigned long addr,  in gup_huge_pd()  argument
2052 static int gup_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr,  in gup_huge_pmd()  argument
2065 		return __gup_device_huge_pmd(orig, pmdp, addr, end, pages, nr);  in gup_huge_pmd()
2069 	page = pmd_page(orig) + ((addr & ~PMD_MASK) >> PAGE_SHIFT);  in gup_huge_pmd()
2075 	} while (addr += PAGE_SIZE, addr != end);  in gup_huge_pmd()
2094 static int gup_huge_pud(pud_t orig, pud_t *pudp, unsigned long addr,  in gup_huge_pud()  argument
2106 		return __gup_device_huge_pud(orig, pudp, addr, end, pages, nr);  in gup_huge_pud()
2110 	page = pud_page(orig) + ((addr & ~PUD_MASK) >> PAGE_SHIFT);  in gup_huge_pud()
2116 	} while (addr += PAGE_SIZE, addr != end);  in gup_huge_pud()
2135 static int gup_huge_pgd(pgd_t orig, pgd_t *pgdp, unsigned long addr,  in gup_huge_pgd()  argument
2147 	page = pgd_page(orig) + ((addr & ~PGDIR_MASK) >> PAGE_SHIFT);  in gup_huge_pgd()
2153 	} while (addr += PAGE_SIZE, addr != end);  in gup_huge_pgd()
2172 static int gup_pmd_range(pud_t pud, unsigned long addr, unsigned long end,  in gup_pmd_range()  argument
2178 	pmdp = pmd_offset(&pud, addr);  in gup_pmd_range()
2182 		next = pmd_addr_end(addr, end);  in gup_pmd_range()
2196 			if (!gup_huge_pmd(pmd, pmdp, addr, next, flags,  in gup_pmd_range()
2205 			if (!gup_huge_pd(__hugepd(pmd_val(pmd)), addr,  in gup_pmd_range()
2208 		} else if (!gup_pte_range(pmd, addr, next, flags, pages, nr))  in gup_pmd_range()
2210 	} while (pmdp++, addr = next, addr != end);  in gup_pmd_range()
2215 static int gup_pud_range(p4d_t p4d, unsigned long addr, unsigned long end,  in gup_pud_range()  argument
2221 	pudp = pud_offset(&p4d, addr);  in gup_pud_range()
2225 		next = pud_addr_end(addr, end);  in gup_pud_range()
2229 			if (!gup_huge_pud(pud, pudp, addr, next, flags,  in gup_pud_range()
2233 			if (!gup_huge_pd(__hugepd(pud_val(pud)), addr,  in gup_pud_range()
2236 		} else if (!gup_pmd_range(pud, addr, next, flags, pages, nr))  in gup_pud_range()
2238 	} while (pudp++, addr = next, addr != end);  in gup_pud_range()
2243 static int gup_p4d_range(pgd_t pgd, unsigned long addr, unsigned long end,  in gup_p4d_range()  argument
2249 	p4dp = p4d_offset(&pgd, addr);  in gup_p4d_range()
2253 		next = p4d_addr_end(addr, end);  in gup_p4d_range()
2258 			if (!gup_huge_pd(__hugepd(p4d_val(p4d)), addr,  in gup_p4d_range()
2261 		} else if (!gup_pud_range(p4d, addr, next, flags, pages, nr))  in gup_p4d_range()
2263 	} while (p4dp++, addr = next, addr != end);  in gup_p4d_range()
2268 static void gup_pgd_range(unsigned long addr, unsigned long end,  in gup_pgd_range()  argument
2274 	pgdp = pgd_offset(current->mm, addr);  in gup_pgd_range()
2278 		next = pgd_addr_end(addr, end);  in gup_pgd_range()
2282 			if (!gup_huge_pgd(pgd, pgdp, addr, next, flags,  in gup_pgd_range()
2286 			if (!gup_huge_pd(__hugepd(pgd_val(pgd)), addr,  in gup_pgd_range()
2289 		} else if (!gup_p4d_range(pgd, addr, next, flags, pages, nr))  in gup_pgd_range()
2291 	} while (pgdp++, addr = next, addr != end);  in gup_pgd_range()
2294 static inline void gup_pgd_range(unsigned long addr, unsigned long end,  in gup_pgd_range()  argument
2401 	unsigned long addr, len, end;  in get_user_pages_fast()  local
2408 	addr = start;  in get_user_pages_fast()
2420 		gup_pgd_range(addr, end, gup_flags, pages, &nr);  in get_user_pages_fast()