Lines Matching refs:page
19 static struct page *no_page_table(struct vm_area_struct *vma, in no_page_table()
45 static struct page *follow_page_pte(struct vm_area_struct *vma, in follow_page_pte()
49 struct page *page; in follow_page_pte() local
84 page = vm_normal_page(vma, address, pte); in follow_page_pte()
85 if (unlikely(!page)) { in follow_page_pte()
89 page = pte_page(pte); in follow_page_pte()
93 get_page_foll(page); in follow_page_pte()
96 !pte_dirty(pte) && !PageDirty(page)) in follow_page_pte()
97 set_page_dirty(page); in follow_page_pte()
103 mark_page_accessed(page); in follow_page_pte()
115 if (page->mapping && trylock_page(page)) { in follow_page_pte()
123 mlock_vma_page(page); in follow_page_pte()
124 unlock_page(page); in follow_page_pte()
128 return page; in follow_page_pte()
153 struct page *follow_page_mask(struct vm_area_struct *vma, in follow_page_mask()
161 struct page *page; in follow_page_mask() local
166 page = follow_huge_addr(mm, address, flags & FOLL_WRITE); in follow_page_mask()
167 if (!IS_ERR(page)) { in follow_page_mask()
169 return page; in follow_page_mask()
180 page = follow_huge_pud(mm, address, pud, flags); in follow_page_mask()
181 if (page) in follow_page_mask()
182 return page; in follow_page_mask()
192 page = follow_huge_pmd(mm, address, pmd, flags); in follow_page_mask()
193 if (page) in follow_page_mask()
194 return page; in follow_page_mask()
210 page = follow_trans_huge_pmd(vma, address, in follow_page_mask()
214 return page; in follow_page_mask()
224 struct page **page) in get_gate_page() argument
250 if (!page) in get_gate_page()
252 *page = vm_normal_page(*vma, address, *pte); in get_gate_page()
253 if (!*page) { in get_gate_page()
256 *page = pte_page(*pte); in get_gate_page()
258 get_page(*page); in get_gate_page()
423 unsigned int gup_flags, struct page **pages, in __get_user_pages()
444 struct page *page; in __get_user_pages() local
479 page = follow_page_mask(vma, start, foll_flags, &page_mask); in __get_user_pages()
480 if (!page) { in __get_user_pages()
498 if (IS_ERR(page)) in __get_user_pages()
499 return i ? i : PTR_ERR(page); in __get_user_pages()
501 pages[i] = page; in __get_user_pages()
502 flush_anon_page(vma, page, start); in __get_user_pages()
503 flush_dcache_page(page); in __get_user_pages()
635 int force, struct page **pages, struct vm_area_struct **vmas) in get_user_pages()
666 struct page *get_dump_page(unsigned long addr) in get_dump_page()
669 struct page *page; in get_dump_page() local
672 FOLL_FORCE | FOLL_DUMP | FOLL_GET, &page, &vma, in get_dump_page()
675 flush_cache_page(vma, addr, page_to_pfn(page)); in get_dump_page()
676 return page; in get_dump_page()
720 int write, struct page **pages, int *nr) in gup_pte_range()
735 struct page *page; in gup_pte_range() local
746 page = pte_page(pte); in gup_pte_range()
748 if (!page_cache_get_speculative(page)) in gup_pte_range()
752 put_page(page); in gup_pte_range()
756 pages[*nr] = page; in gup_pte_range()
779 int write, struct page **pages, int *nr) in gup_pte_range()
786 unsigned long end, int write, struct page **pages, int *nr) in gup_huge_pmd()
788 struct page *head, *page, *tail; in gup_huge_pmd() local
796 page = head + ((addr & ~PMD_MASK) >> PAGE_SHIFT); in gup_huge_pmd()
797 tail = page; in gup_huge_pmd()
799 VM_BUG_ON_PAGE(compound_head(page) != head, page); in gup_huge_pmd()
800 pages[*nr] = page; in gup_huge_pmd()
802 page++; in gup_huge_pmd()
833 unsigned long end, int write, struct page **pages, int *nr) in gup_huge_pud()
835 struct page *head, *page, *tail; in gup_huge_pud() local
843 page = head + ((addr & ~PUD_MASK) >> PAGE_SHIFT); in gup_huge_pud()
844 tail = page; in gup_huge_pud()
846 VM_BUG_ON_PAGE(compound_head(page) != head, page); in gup_huge_pud()
847 pages[*nr] = page; in gup_huge_pud()
849 page++; in gup_huge_pud()
875 int write, struct page **pages, int *nr) in gup_pmd_range()
909 int write, struct page **pages, int *nr) in gup_pud_range()
937 struct page **pages) in __get_user_pages_fast()
997 struct page **pages) in get_user_pages_fast()