Lines Matching full:pages
33 static inline void sanity_check_pinned_pages(struct page **pages, in sanity_check_pinned_pages() argument
40 * We only pin anonymous pages if they are exclusive. Once pinned, we in sanity_check_pinned_pages()
44 * We'd like to verify that our pinned anonymous pages are still mapped in sanity_check_pinned_pages()
51 for (; npages; npages--, pages++) { in sanity_check_pinned_pages()
52 struct page *page = *pages; in sanity_check_pinned_pages()
267 * Pages that were pinned via pin_user_pages*() must be released via either
269 * that such pages can be separately tracked and uniquely handled. In
337 * unpin_user_pages_dirty_lock() - release and optionally dirty gup-pinned pages
338 * @pages: array of pages to be maybe marked dirty, and definitely released.
339 * @npages: number of pages in the @pages array.
340 * @make_dirty: whether to mark the pages dirty
345 * For each page in the @pages array, make that page (or its head page, if a
347 * listed as clean. In any case, releases all pages using unpin_user_page(),
358 void unpin_user_pages_dirty_lock(struct page **pages, unsigned long npages, in unpin_user_pages_dirty_lock() argument
366 unpin_user_pages(pages, npages); in unpin_user_pages_dirty_lock()
370 sanity_check_pinned_pages(pages, npages); in unpin_user_pages_dirty_lock()
372 folio = gup_folio_next(pages, npages, i, &nr); in unpin_user_pages_dirty_lock()
408 * @npages: number of consecutive pages to release.
409 * @make_dirty: whether to mark the pages dirty
411 * "gup-pinned page range" refers to a range of pages that has had one of the
415 * its head pages, if a compound page) dirty, if @make_dirty is true, and if the
443 static void unpin_user_pages_lockless(struct page **pages, unsigned long npages) in unpin_user_pages_lockless() argument
451 * fork() and some anonymous pages might now actually be shared -- in unpin_user_pages_lockless()
455 folio = gup_folio_next(pages, npages, i, &nr); in unpin_user_pages_lockless()
461 * unpin_user_pages() - release an array of gup-pinned pages.
462 * @pages: array of pages to be marked dirty and released.
463 * @npages: number of pages in the @pages array.
465 * For each page in the @pages array, release the page using unpin_user_page().
469 void unpin_user_pages(struct page **pages, unsigned long npages) in unpin_user_pages() argument
476 * If this WARN_ON() fires, then the system *might* be leaking pages (by in unpin_user_pages()
483 sanity_check_pinned_pages(pages, npages); in unpin_user_pages()
485 folio = gup_folio_next(pages, npages, i, &nr); in unpin_user_pages()
508 * has touched so far, we don't want to allocate unnecessary pages or in no_page_table()
606 * We only care about anon pages in can_follow_write_pte() and don't in follow_page_pte()
617 * Only return device mapping pages in the FOLL_GET or FOLL_PIN in follow_page_pte()
628 /* Avoid special (like zero) pages in core dumps */ in follow_page_pte()
795 * When getting pages from ZONE_DEVICE memory, the @ctx->pgmap caches
849 * to fail on PROT_NONE-mapped pages. in follow_page()
869 /* user gate pages are read-only */ in get_gate_page()
1066 * Anon pages in shared mappings are surprising: now in check_vma_flags()
1131 * __get_user_pages() - pin user pages in memory
1134 * @nr_pages: number of pages from start to pin
1136 * @pages: array that receives pointers to the pages pinned.
1138 * only intends to ensure the pages are faulted in.
1141 * Returns either number of pages pinned (which may be less than the
1145 * -- If nr_pages is >0, but no pages were pinned, returns -errno.
1146 * -- If nr_pages is >0, and some pages were pinned, returns the number of
1147 * pages pinned. Again, this may be less than nr_pages.
1150 * The caller is responsible for releasing returned @pages, via put_page().
1186 unsigned int gup_flags, struct page **pages, in __get_user_pages() argument
1198 VM_BUG_ON(!!pages != !!(gup_flags & (FOLL_GET | FOLL_PIN))); in __get_user_pages()
1211 pages ? &page : NULL); in __get_user_pages()
1228 * If we have a pending SIGKILL, don't keep faulting pages and in __get_user_pages()
1257 * struct page. If the caller expects **pages to be in __get_user_pages()
1261 if (pages) { in __get_user_pages()
1274 if (pages) { in __get_user_pages()
1286 * pages. in __get_user_pages()
1311 pages[i + j] = subpage; in __get_user_pages()
1467 struct page **pages, in __get_user_pages_locked() argument
1492 * is to set FOLL_GET if the caller wants pages[] filled in (but has in __get_user_pages_locked()
1496 * FOLL_PIN always expects pages to be non-null, but no need to assert in __get_user_pages_locked()
1499 if (pages && !(flags & FOLL_PIN)) in __get_user_pages_locked()
1504 ret = __get_user_pages(mm, start, nr_pages, flags, pages, in __get_user_pages_locked()
1535 * For the prefault case (!pages) we only update counts. in __get_user_pages_locked()
1537 if (likely(pages)) in __get_user_pages_locked()
1538 pages += ret; in __get_user_pages_locked()
1569 pages, locked); in __get_user_pages_locked()
1585 if (likely(pages)) in __get_user_pages_locked()
1586 pages++; in __get_user_pages_locked()
1602 * populate_vma_page_range() - populate a range of pages in the vma.
1608 * This takes care of mlocking the pages too if VM_LOCKED is set.
1610 * Return either number of pages pinned in the vma, or a negative error
1676 * This takes care of mlocking the pages, too, if VM_LOCKED is set.
1684 * Returns either number of processed pages in the vma, or a negative error
1731 * __mm_populate - populate and/or mlock pages within a range of address space.
1749 * We want to fault in pages for [nstart; end) address range. in __mm_populate()
1771 * Now fault in a range of pages. populate_vma_page_range() in __mm_populate()
1772 * double checks the vma flags, so that it won't mlock pages in __mm_populate()
1792 unsigned long nr_pages, struct page **pages, in __get_user_pages_locked() argument
1832 if (pages) { in __get_user_pages_locked()
1833 pages[i] = virt_to_page((void *)start); in __get_user_pages_locked()
1834 if (pages[i]) in __get_user_pages_locked()
1835 get_page(pages[i]); in __get_user_pages_locked()
1921 * already know that some or all of the pages in the address range aren't in
1926 * Note that we don't pin or otherwise hold the pages referenced that we fault
2026 * Returns the number of collected pages. Return value is always >= 0.
2031 struct page **pages) in collect_longterm_unpinnable_pages() argument
2038 struct folio *folio = page_folio(pages[i]); in collect_longterm_unpinnable_pages()
2075 * Unpins all pages and migrates device coherent pages and movable_page_list.
2076 * Returns -EAGAIN if all pages were successfully migrated or -errno for failure
2082 struct page **pages) in migrate_longterm_unpinnable_pages() argument
2088 struct folio *folio = page_folio(pages[i]); in migrate_longterm_unpinnable_pages()
2095 pages[i] = NULL; in migrate_longterm_unpinnable_pages()
2108 * We can't migrate pages with unexpected references, so drop in migrate_longterm_unpinnable_pages()
2110 * Migrating pages have been added to movable_page_list after in migrate_longterm_unpinnable_pages()
2114 unpin_user_page(pages[i]); in migrate_longterm_unpinnable_pages()
2115 pages[i] = NULL; in migrate_longterm_unpinnable_pages()
2138 if (pages[i]) in migrate_longterm_unpinnable_pages()
2139 unpin_user_page(pages[i]); in migrate_longterm_unpinnable_pages()
2146 * Check whether all pages are *allowed* to be pinned. Rather confusingly, all
2147 * pages in the range are required to be pinned via FOLL_PIN, before calling
2150 * If any pages in the range are not allowed to be pinned, then this routine
2151 * will migrate those pages away, unpin all the pages in the range and return
2158 * If everything is OK and all pages in the range are allowed to be pinned, then
2159 * this routine leaves all pages pinned and returns zero for success.
2162 struct page **pages) in check_and_migrate_movable_pages() argument
2168 nr_pages, pages); in check_and_migrate_movable_pages()
2173 pages); in check_and_migrate_movable_pages()
2177 struct page **pages) in check_and_migrate_movable_pages() argument
2190 struct page **pages, in __gup_longterm_locked() argument
2198 return __get_user_pages_locked(mm, start, nr_pages, pages, in __gup_longterm_locked()
2204 pages, locked, in __gup_longterm_locked()
2212 rc = check_and_migrate_movable_pages(nr_pinned_pages, pages); in __gup_longterm_locked()
2222 static bool is_valid_gup_args(struct page **pages, int *locked, in is_valid_gup_args() argument
2256 /* Pages input must be given if using GET/PIN */ in is_valid_gup_args()
2257 if (WARN_ON_ONCE((gup_flags & (FOLL_GET | FOLL_PIN)) && !pages)) in is_valid_gup_args()
2271 * get_user_pages_remote() - pin user pages in memory
2274 * @nr_pages: number of pages from start to pin
2276 * @pages: array that receives pointers to the pages pinned.
2278 * only intends to ensure the pages are faulted in.
2283 * Returns either number of pages pinned (which may be less than the
2287 * -- If nr_pages is >0, but no pages were pinned, returns -errno.
2288 * -- If nr_pages is >0, and some pages were pinned, returns the number of
2289 * pages pinned. Again, this may be less than nr_pages.
2291 * The caller is responsible for releasing returned @pages, via put_page().
2315 * via the user virtual addresses. The pages may be submitted for
2328 unsigned int gup_flags, struct page **pages, in get_user_pages_remote() argument
2333 if (!is_valid_gup_args(pages, locked, &gup_flags, in get_user_pages_remote()
2337 return __get_user_pages_locked(mm, start, nr_pages, pages, in get_user_pages_remote()
2346 unsigned int gup_flags, struct page **pages, in get_user_pages_remote() argument
2354 * get_user_pages() - pin user pages in memory
2356 * @nr_pages: number of pages from start to pin
2358 * @pages: array that receives pointers to the pages pinned.
2360 * only intends to ensure the pages are faulted in.
2368 unsigned int gup_flags, struct page **pages) in get_user_pages() argument
2372 if (!is_valid_gup_args(pages, NULL, &gup_flags, FOLL_TOUCH)) in get_user_pages()
2375 return __get_user_pages_locked(current->mm, start, nr_pages, pages, in get_user_pages()
2384 * get_user_pages(mm, ..., pages, NULL);
2389 * get_user_pages_unlocked(mm, ..., pages);
2396 struct page **pages, unsigned int gup_flags) in get_user_pages_unlocked() argument
2400 if (!is_valid_gup_args(pages, NULL, &gup_flags, in get_user_pages_unlocked()
2404 return __get_user_pages_locked(current->mm, start, nr_pages, pages, in get_user_pages_unlocked()
2412 * get_user_pages_fast attempts to pin user pages by walking the page
2414 * protected from page table pages being freed from under it, and should
2419 * pages are freed. This is unsuitable for architectures that do not need
2422 * Another way to achieve this is to batch up page table containing pages
2424 * pages. Disabling interrupts will allow the fast_gup walker to both block
2432 * free pages containing page tables or TLB flushing requires IPI broadcast.
2522 struct page **pages) in undo_dev_pagemap() argument
2525 struct page *page = pages[--(*nr)]; in undo_dev_pagemap()
2557 struct page **pages, int *nr) in gup_pte_range() argument
2572 * Always fallback to ordinary GUP on PROT_NONE-mapped pages: in gup_pte_range()
2573 * pte_access_permitted() better should reject these pages in gup_pte_range()
2590 undo_dev_pagemap(nr, nr_start, flags, pages); in gup_pte_range()
2638 pages[*nr] = page; in gup_pte_range()
2658 * get_user_pages_fast_only implementation that can pin pages. Thus it's still
2663 struct page **pages, int *nr) in gup_pte_range() argument
2672 struct page **pages, int *nr) in __gup_device_huge() argument
2682 undo_dev_pagemap(nr, nr_start, flags, pages); in __gup_device_huge()
2687 undo_dev_pagemap(nr, nr_start, flags, pages); in __gup_device_huge()
2692 pages[*nr] = page; in __gup_device_huge()
2694 undo_dev_pagemap(nr, nr_start, flags, pages); in __gup_device_huge()
2707 struct page **pages, int *nr) in __gup_device_huge_pmd() argument
2713 if (!__gup_device_huge(fault_pfn, addr, end, flags, pages, nr)) in __gup_device_huge_pmd()
2717 undo_dev_pagemap(nr, nr_start, flags, pages); in __gup_device_huge_pmd()
2725 struct page **pages, int *nr) in __gup_device_huge_pud() argument
2731 if (!__gup_device_huge(fault_pfn, addr, end, flags, pages, nr)) in __gup_device_huge_pud()
2735 undo_dev_pagemap(nr, nr_start, flags, pages); in __gup_device_huge_pud()
2743 struct page **pages, int *nr) in __gup_device_huge_pmd() argument
2751 struct page **pages, int *nr) in __gup_device_huge_pud() argument
2759 unsigned long end, struct page **pages) in record_subpages() argument
2764 pages[nr] = nth_page(page, nr); in record_subpages()
2779 struct page **pages, int *nr) in gup_hugepte() argument
2800 refs = record_subpages(page, addr, end, pages + *nr); in gup_hugepte()
2828 struct page **pages, int *nr) in gup_huge_pd() argument
2837 if (!gup_hugepte(ptep, sz, addr, end, flags, pages, nr)) in gup_huge_pd()
2846 struct page **pages, int *nr) in gup_huge_pd() argument
2854 struct page **pages, int *nr) in gup_huge_pmd() argument
2867 pages, nr); in gup_huge_pmd()
2871 refs = record_subpages(page, addr, end, pages + *nr); in gup_huge_pmd()
2898 struct page **pages, int *nr) in gup_huge_pud() argument
2911 pages, nr); in gup_huge_pud()
2915 refs = record_subpages(page, addr, end, pages + *nr); in gup_huge_pud()
2943 struct page **pages, int *nr) in gup_huge_pgd() argument
2955 refs = record_subpages(page, addr, end, pages + *nr); in gup_huge_pgd()
2982 unsigned int flags, struct page **pages, int *nr) in gup_pmd_range() argument
3002 pages, nr)) in gup_pmd_range()
3011 PMD_SHIFT, next, flags, pages, nr)) in gup_pmd_range()
3013 } else if (!gup_pte_range(pmd, pmdp, addr, next, flags, pages, nr)) in gup_pmd_range()
3021 unsigned int flags, struct page **pages, int *nr) in gup_pud_range() argument
3035 pages, nr)) in gup_pud_range()
3039 PUD_SHIFT, next, flags, pages, nr)) in gup_pud_range()
3041 } else if (!gup_pmd_range(pudp, pud, addr, next, flags, pages, nr)) in gup_pud_range()
3049 unsigned int flags, struct page **pages, int *nr) in gup_p4d_range() argument
3064 P4D_SHIFT, next, flags, pages, nr)) in gup_p4d_range()
3066 } else if (!gup_pud_range(p4dp, p4d, addr, next, flags, pages, nr)) in gup_p4d_range()
3074 unsigned int flags, struct page **pages, int *nr) in gup_pgd_range() argument
3088 pages, nr)) in gup_pgd_range()
3092 PGDIR_SHIFT, next, flags, pages, nr)) in gup_pgd_range()
3094 } else if (!gup_p4d_range(pgdp, pgd, addr, next, flags, pages, nr)) in gup_pgd_range()
3100 unsigned int flags, struct page **pages, int *nr) in gup_pgd_range() argument
3119 struct page **pages) in lockless_pages_from_mm() argument
3139 * With interrupts disabled, we block page table pages from being freed in lockless_pages_from_mm()
3147 gup_pgd_range(start, end, gup_flags, pages, &nr_pinned); in lockless_pages_from_mm()
3151 * When pinning pages for DMA there could be a concurrent write protect in lockless_pages_from_mm()
3156 unpin_user_pages_lockless(pages, nr_pinned); in lockless_pages_from_mm()
3159 sanity_check_pinned_pages(pages, nr_pinned); in lockless_pages_from_mm()
3168 struct page **pages) in internal_get_user_pages_fast() argument
3196 nr_pinned = lockless_pages_from_mm(start, end, gup_flags, pages); in internal_get_user_pages_fast()
3200 /* Slow path: try to get the remaining pages with get_user_pages */ in internal_get_user_pages_fast()
3202 pages += nr_pinned; in internal_get_user_pages_fast()
3204 pages, &locked, in internal_get_user_pages_fast()
3208 * The caller has to unpin the pages we already pinned so in internal_get_user_pages_fast()
3219 * get_user_pages_fast_only() - pin user pages in memory
3221 * @nr_pages: number of pages from start to pin
3223 * @pages: array that receives pointers to the pages pinned.
3230 * pages pinned.
3237 unsigned int gup_flags, struct page **pages) in get_user_pages_fast_only() argument
3246 if (!is_valid_gup_args(pages, NULL, &gup_flags, in get_user_pages_fast_only()
3250 return internal_get_user_pages_fast(start, nr_pages, gup_flags, pages); in get_user_pages_fast_only()
3255 * get_user_pages_fast() - pin user pages in memory
3257 * @nr_pages: number of pages from start to pin
3259 * @pages: array that receives pointers to the pages pinned.
3262 * Attempt to pin user pages in memory without taking mm->mmap_lock.
3266 * Returns number of pages pinned. This may be fewer than the number requested.
3267 * If nr_pages is 0 or negative, returns 0. If no pages were pinned, returns
3271 unsigned int gup_flags, struct page **pages) in get_user_pages_fast() argument
3279 if (!is_valid_gup_args(pages, NULL, &gup_flags, FOLL_GET)) in get_user_pages_fast()
3281 return internal_get_user_pages_fast(start, nr_pages, gup_flags, pages); in get_user_pages_fast()
3286 * pin_user_pages_fast() - pin user pages in memory without taking locks
3289 * @nr_pages: number of pages from start to pin
3291 * @pages: array that receives pointers to the pages pinned.
3298 * FOLL_PIN means that the pages must be released via unpin_user_page(). Please
3301 * Note that if a zero_page is amongst the returned pages, it will not have
3305 unsigned int gup_flags, struct page **pages) in pin_user_pages_fast() argument
3307 if (!is_valid_gup_args(pages, NULL, &gup_flags, FOLL_PIN)) in pin_user_pages_fast()
3309 return internal_get_user_pages_fast(start, nr_pages, gup_flags, pages); in pin_user_pages_fast()
3314 * pin_user_pages_remote() - pin pages of a remote process
3318 * @nr_pages: number of pages from start to pin
3320 * @pages: array that receives pointers to the pages pinned.
3330 * FOLL_PIN means that the pages must be released via unpin_user_page(). Please
3333 * Note that if a zero_page is amongst the returned pages, it will not have
3338 unsigned int gup_flags, struct page **pages, in pin_user_pages_remote() argument
3343 if (!is_valid_gup_args(pages, locked, &gup_flags, in pin_user_pages_remote()
3346 return __gup_longterm_locked(mm, start, nr_pages, pages, in pin_user_pages_remote()
3353 * pin_user_pages() - pin user pages in memory for use by other devices
3356 * @nr_pages: number of pages from start to pin
3358 * @pages: array that receives pointers to the pages pinned.
3364 * FOLL_PIN means that the pages must be released via unpin_user_page(). Please
3367 * Note that if a zero_page is amongst the returned pages, it will not have
3371 unsigned int gup_flags, struct page **pages) in pin_user_pages() argument
3375 if (!is_valid_gup_args(pages, NULL, &gup_flags, FOLL_PIN)) in pin_user_pages()
3378 pages, &locked, gup_flags); in pin_user_pages()
3387 * Note that if a zero_page is amongst the returned pages, it will not have
3391 struct page **pages, unsigned int gup_flags) in pin_user_pages_unlocked() argument
3395 if (!is_valid_gup_args(pages, NULL, &gup_flags, in pin_user_pages_unlocked()
3399 return __gup_longterm_locked(current->mm, start, nr_pages, pages, in pin_user_pages_unlocked()