Lines Matching refs:page
54 void put_user_pages_dirty_lock(struct page **pages, unsigned long npages, in put_user_pages_dirty_lock()
71 struct page *page = compound_head(pages[index]); in put_user_pages_dirty_lock() local
92 if (!PageDirty(page)) in put_user_pages_dirty_lock()
93 set_page_dirty_lock(page); in put_user_pages_dirty_lock()
94 put_user_page(page); in put_user_pages_dirty_lock()
108 void put_user_pages(struct page **pages, unsigned long npages) in put_user_pages()
123 static struct page *no_page_table(struct vm_area_struct *vma, in no_page_table()
173 static struct page *follow_page_pte(struct vm_area_struct *vma, in follow_page_pte()
178 struct page *page; in follow_page_pte() local
213 page = vm_normal_page(vma, address, pte); in follow_page_pte()
214 if (!page && pte_devmap(pte) && (flags & FOLL_GET)) { in follow_page_pte()
221 page = pte_page(pte); in follow_page_pte()
224 } else if (unlikely(!page)) { in follow_page_pte()
227 page = ERR_PTR(-EFAULT); in follow_page_pte()
232 page = pte_page(pte); in follow_page_pte()
237 page = ERR_PTR(ret); in follow_page_pte()
242 if (flags & FOLL_SPLIT && PageTransCompound(page)) { in follow_page_pte()
244 get_page(page); in follow_page_pte()
246 lock_page(page); in follow_page_pte()
247 ret = split_huge_page(page); in follow_page_pte()
248 unlock_page(page); in follow_page_pte()
249 put_page(page); in follow_page_pte()
256 if (unlikely(!try_get_page(page))) { in follow_page_pte()
257 page = ERR_PTR(-ENOMEM); in follow_page_pte()
263 !pte_dirty(pte) && !PageDirty(page)) in follow_page_pte()
264 set_page_dirty(page); in follow_page_pte()
270 mark_page_accessed(page); in follow_page_pte()
274 if (PageTransCompound(page)) in follow_page_pte()
286 if (page->mapping && trylock_page(page)) { in follow_page_pte()
294 mlock_vma_page(page); in follow_page_pte()
295 unlock_page(page); in follow_page_pte()
300 return page; in follow_page_pte()
308 static struct page *follow_pmd_mask(struct vm_area_struct *vma, in follow_pmd_mask()
315 struct page *page; in follow_pmd_mask() local
327 page = follow_huge_pmd(mm, address, pmd, flags); in follow_pmd_mask()
328 if (page) in follow_pmd_mask()
329 return page; in follow_pmd_mask()
333 page = follow_huge_pd(vma, address, in follow_pmd_mask()
336 if (page) in follow_pmd_mask()
337 return page; in follow_pmd_mask()
359 page = follow_devmap_pmd(vma, address, pmd, flags, &ctx->pgmap); in follow_pmd_mask()
361 if (page) in follow_pmd_mask()
362 return page; in follow_pmd_mask()
389 page = pmd_page(*pmd); in follow_pmd_mask()
390 if (is_huge_zero_page(page)) { in follow_pmd_mask()
397 if (unlikely(!try_get_page(page))) { in follow_pmd_mask()
402 lock_page(page); in follow_pmd_mask()
403 ret = split_huge_page(page); in follow_pmd_mask()
404 unlock_page(page); in follow_pmd_mask()
405 put_page(page); in follow_pmd_mask()
417 page = follow_trans_huge_pmd(vma, address, pmd, flags); in follow_pmd_mask()
420 return page; in follow_pmd_mask()
423 static struct page *follow_pud_mask(struct vm_area_struct *vma, in follow_pud_mask()
430 struct page *page; in follow_pud_mask() local
437 page = follow_huge_pud(mm, address, pud, flags); in follow_pud_mask()
438 if (page) in follow_pud_mask()
439 return page; in follow_pud_mask()
443 page = follow_huge_pd(vma, address, in follow_pud_mask()
446 if (page) in follow_pud_mask()
447 return page; in follow_pud_mask()
452 page = follow_devmap_pud(vma, address, pud, flags, &ctx->pgmap); in follow_pud_mask()
454 if (page) in follow_pud_mask()
455 return page; in follow_pud_mask()
463 static struct page *follow_p4d_mask(struct vm_area_struct *vma, in follow_p4d_mask()
469 struct page *page; in follow_p4d_mask() local
479 page = follow_huge_pd(vma, address, in follow_p4d_mask()
482 if (page) in follow_p4d_mask()
483 return page; in follow_p4d_mask()
508 static struct page *follow_page_mask(struct vm_area_struct *vma, in follow_page_mask()
513 struct page *page; in follow_page_mask() local
519 page = follow_huge_addr(mm, address, flags & FOLL_WRITE); in follow_page_mask()
520 if (!IS_ERR(page)) { in follow_page_mask()
522 return page; in follow_page_mask()
531 page = follow_huge_pgd(mm, address, pgd, flags); in follow_page_mask()
532 if (page) in follow_page_mask()
533 return page; in follow_page_mask()
537 page = follow_huge_pd(vma, address, in follow_page_mask()
540 if (page) in follow_page_mask()
541 return page; in follow_page_mask()
548 struct page *follow_page(struct vm_area_struct *vma, unsigned long address, in follow_page()
552 struct page *page; in follow_page() local
554 page = follow_page_mask(vma, address, foll_flags, &ctx); in follow_page()
557 return page; in follow_page()
562 struct page **page) in get_gate_page() argument
594 if (!page) in get_gate_page()
596 *page = vm_normal_page(*vma, address, *pte); in get_gate_page()
597 if (!*page) { in get_gate_page()
600 *page = pte_page(*pte); in get_gate_page()
602 if (unlikely(!try_get_page(*page))) { in get_gate_page()
781 unsigned int gup_flags, struct page **pages, in __get_user_pages()
804 struct page *page; in __get_user_pages() local
843 page = follow_page_mask(vma, start, foll_flags, &ctx); in __get_user_pages()
844 if (!page) { in __get_user_pages()
861 } else if (PTR_ERR(page) == -EEXIST) { in __get_user_pages()
867 } else if (IS_ERR(page)) { in __get_user_pages()
868 ret = PTR_ERR(page); in __get_user_pages()
872 pages[i] = page; in __get_user_pages()
873 flush_anon_page(vma, page, start); in __get_user_pages()
874 flush_dcache_page(page); in __get_user_pages()
1002 struct page **pages, in __get_user_pages_locked()
1151 unsigned int gup_flags, struct page **pages, in get_user_pages_remote()
1302 struct page *get_dump_page(unsigned long addr) in get_dump_page()
1305 struct page *page; in get_dump_page() local
1308 FOLL_FORCE | FOLL_DUMP | FOLL_GET, &page, &vma, in get_dump_page()
1311 flush_cache_page(vma, addr, page_to_pfn(page)); in get_dump_page()
1312 return page; in get_dump_page()
1318 unsigned long nr_pages, struct page **pages, in __get_user_pages_locked()
1382 static struct page *new_non_cma_page(struct page *page, unsigned long private) in new_non_cma_page() argument
1388 int nid = page_to_nid(page); in new_non_cma_page()
1398 if (PageHighMem(page)) in new_non_cma_page()
1402 if (PageHuge(page)) { in new_non_cma_page()
1403 struct hstate *h = page_hstate(page); in new_non_cma_page()
1411 if (PageTransHuge(page)) { in new_non_cma_page()
1412 struct page *thp; in new_non_cma_page()
1437 struct page **pages, in check_and_migrate_cma_pages()
1450 struct page *head = compound_head(pages[i]); in check_and_migrate_cma_pages()
1524 struct page **pages, in check_and_migrate_cma_pages()
1540 struct page **pages, in __gup_longterm_locked()
1591 struct page **pages, in __gup_longterm_locked()
1608 unsigned int gup_flags, struct page **pages, in get_user_pages()
1638 unsigned int gup_flags, struct page **pages, in get_user_pages_locked()
1672 struct page **pages, unsigned int gup_flags) in get_user_pages_unlocked()
1786 struct page **pages) in undo_dev_pagemap()
1789 struct page *page = pages[--(*nr)]; in undo_dev_pagemap() local
1791 ClearPageReferenced(page); in undo_dev_pagemap()
1792 put_page(page); in undo_dev_pagemap()
1800 static inline struct page *try_get_compound_head(struct page *page, int refs) in try_get_compound_head() argument
1802 struct page *head = compound_head(page); in try_get_compound_head()
1812 unsigned int flags, struct page **pages, int *nr) in gup_pte_range()
1821 struct page *head, *page; in gup_pte_range() local
1846 page = pte_page(pte); in gup_pte_range()
1848 head = try_get_compound_head(page, 1); in gup_pte_range()
1857 VM_BUG_ON_PAGE(compound_head(page) != head, page); in gup_pte_range()
1859 SetPageReferenced(page); in gup_pte_range()
1860 pages[*nr] = page; in gup_pte_range()
1885 unsigned int flags, struct page **pages, int *nr) in gup_pte_range()
1893 unsigned long end, struct page **pages, int *nr) in __gup_device_huge()
1899 struct page *page = pfn_to_page(pfn); in __gup_device_huge() local
1906 SetPageReferenced(page); in __gup_device_huge()
1907 pages[*nr] = page; in __gup_device_huge()
1908 get_page(page); in __gup_device_huge()
1919 unsigned long end, struct page **pages, int *nr) in __gup_device_huge_pmd()
1936 unsigned long end, struct page **pages, int *nr) in __gup_device_huge_pud()
1953 unsigned long end, struct page **pages, int *nr) in __gup_device_huge_pmd()
1960 unsigned long end, struct page **pages, int *nr) in __gup_device_huge_pud()
1977 struct page **pages, int *nr) in gup_hugepte()
1980 struct page *head, *page; in gup_hugepte() local
1999 page = head + ((addr & (sz-1)) >> PAGE_SHIFT); in gup_hugepte()
2001 VM_BUG_ON(compound_head(page) != head); in gup_hugepte()
2002 pages[*nr] = page; in gup_hugepte()
2004 page++; in gup_hugepte()
2028 struct page **pages, int *nr) in gup_huge_pd()
2046 struct page **pages, int *nr) in gup_huge_pd()
2054 struct page **pages, int *nr) in gup_huge_pmd()
2056 struct page *head, *page; in gup_huge_pmd() local
2069 page = pmd_page(orig) + ((addr & ~PMD_MASK) >> PAGE_SHIFT); in gup_huge_pmd()
2071 pages[*nr] = page; in gup_huge_pmd()
2073 page++; in gup_huge_pmd()
2095 unsigned long end, unsigned int flags, struct page **pages, int *nr) in gup_huge_pud()
2097 struct page *head, *page; in gup_huge_pud() local
2110 page = pud_page(orig) + ((addr & ~PUD_MASK) >> PAGE_SHIFT); in gup_huge_pud()
2112 pages[*nr] = page; in gup_huge_pud()
2114 page++; in gup_huge_pud()
2137 struct page **pages, int *nr) in gup_huge_pgd()
2140 struct page *head, *page; in gup_huge_pgd() local
2147 page = pgd_page(orig) + ((addr & ~PGDIR_MASK) >> PAGE_SHIFT); in gup_huge_pgd()
2149 pages[*nr] = page; in gup_huge_pgd()
2151 page++; in gup_huge_pgd()
2173 unsigned int flags, struct page **pages, int *nr) in gup_pmd_range()
2216 unsigned int flags, struct page **pages, int *nr) in gup_pud_range()
2244 unsigned int flags, struct page **pages, int *nr) in gup_p4d_range()
2269 unsigned int flags, struct page **pages, int *nr) in gup_pgd_range()
2295 unsigned int flags, struct page **pages, int *nr) in gup_pgd_range()
2321 struct page **pages) in __get_user_pages_fast()
2360 unsigned int gup_flags, struct page **pages) in __gup_longterm_unlocked()
2399 unsigned int gup_flags, struct page **pages) in get_user_pages_fast()