• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 // SPDX-License-Identifier: GPL-2.0-only
2 #include <linux/kernel.h>
3 #include <linux/errno.h>
4 #include <linux/err.h>
5 #include <linux/spinlock.h>
6 
7 #include <linux/mm.h>
8 #include <linux/memremap.h>
9 #include <linux/pagemap.h>
10 #include <linux/rmap.h>
11 #include <linux/swap.h>
12 #include <linux/swapops.h>
13 
14 #include <linux/sched/signal.h>
15 #include <linux/rwsem.h>
16 #include <linux/hugetlb.h>
17 #include <linux/migrate.h>
18 #include <linux/mm_inline.h>
19 #include <linux/sched/mm.h>
20 
21 #include <asm/mmu_context.h>
22 #include <asm/tlbflush.h>
23 
24 #include "internal.h"
25 
26 struct follow_page_context {
27 	struct dev_pagemap *pgmap;
28 	unsigned int page_mask;
29 };
30 
hpage_pincount_add(struct page * page,int refs)31 static void hpage_pincount_add(struct page *page, int refs)
32 {
33 	VM_BUG_ON_PAGE(!hpage_pincount_available(page), page);
34 	VM_BUG_ON_PAGE(page != compound_head(page), page);
35 
36 	atomic_add(refs, compound_pincount_ptr(page));
37 }
38 
hpage_pincount_sub(struct page * page,int refs)39 static void hpage_pincount_sub(struct page *page, int refs)
40 {
41 	VM_BUG_ON_PAGE(!hpage_pincount_available(page), page);
42 	VM_BUG_ON_PAGE(page != compound_head(page), page);
43 
44 	atomic_sub(refs, compound_pincount_ptr(page));
45 }
46 
47 /* Equivalent to calling put_page() @refs times. */
put_page_refs(struct page * page,int refs)48 static void put_page_refs(struct page *page, int refs)
49 {
50 #ifdef CONFIG_DEBUG_VM
51 	if (VM_WARN_ON_ONCE_PAGE(page_ref_count(page) < refs, page))
52 		return;
53 #endif
54 
55 	/*
56 	 * Calling put_page() for each ref is unnecessarily slow. Only the last
57 	 * ref needs a put_page().
58 	 */
59 	if (refs > 1)
60 		page_ref_sub(page, refs - 1);
61 	put_page(page);
62 }
63 
64 /*
65  * Return the compound head page with ref appropriately incremented,
66  * or NULL if that failed.
67  */
try_get_compound_head(struct page * page,int refs)68 static inline struct page *try_get_compound_head(struct page *page, int refs)
69 {
70 	struct page *head = compound_head(page);
71 
72 	if (WARN_ON_ONCE(page_ref_count(head) < 0))
73 		return NULL;
74 	if (unlikely(!page_cache_add_speculative(head, refs)))
75 		return NULL;
76 
77 	/*
78 	 * At this point we have a stable reference to the head page; but it
79 	 * could be that between the compound_head() lookup and the refcount
80 	 * increment, the compound page was split, in which case we'd end up
81 	 * holding a reference on a page that has nothing to do with the page
82 	 * we were given anymore.
83 	 * So now that the head page is stable, recheck that the pages still
84 	 * belong together.
85 	 */
86 	if (unlikely(compound_head(page) != head)) {
87 		put_page_refs(head, refs);
88 		return NULL;
89 	}
90 
91 	return head;
92 }
93 
94 /*
95  * try_grab_compound_head() - attempt to elevate a page's refcount, by a
96  * flags-dependent amount.
97  *
98  * "grab" names in this file mean, "look at flags to decide whether to use
99  * FOLL_PIN or FOLL_GET behavior, when incrementing the page's refcount.
100  *
101  * Either FOLL_PIN or FOLL_GET (or neither) must be set, but not both at the
102  * same time. (That's true throughout the get_user_pages*() and
103  * pin_user_pages*() APIs.) Cases:
104  *
105  *    FOLL_GET: page's refcount will be incremented by 1.
106  *    FOLL_PIN: page's refcount will be incremented by GUP_PIN_COUNTING_BIAS.
107  *
108  * Return: head page (with refcount appropriately incremented) for success, or
109  * NULL upon failure. If neither FOLL_GET nor FOLL_PIN was set, that's
110  * considered failure, and furthermore, a likely bug in the caller, so a warning
111  * is also emitted.
112  */
try_grab_compound_head(struct page * page,int refs,unsigned int flags)113 static __maybe_unused struct page *try_grab_compound_head(struct page *page,
114 							  int refs,
115 							  unsigned int flags)
116 {
117 	if (flags & FOLL_GET)
118 		return try_get_compound_head(page, refs);
119 	else if (flags & FOLL_PIN) {
120 		int orig_refs = refs;
121 
122 		/*
123 		 * Can't do FOLL_LONGTERM + FOLL_PIN with CMA in the gup fast
124 		 * path, so fail and let the caller fall back to the slow path.
125 		 */
126 		if (unlikely(flags & FOLL_LONGTERM) &&
127 				is_migrate_cma_page(page))
128 			return NULL;
129 
130 		/*
131 		 * CAUTION: Don't use compound_head() on the page before this
132 		 * point, the result won't be stable.
133 		 */
134 		page = try_get_compound_head(page, refs);
135 		if (!page)
136 			return NULL;
137 
138 		/*
139 		 * When pinning a compound page of order > 1 (which is what
140 		 * hpage_pincount_available() checks for), use an exact count to
141 		 * track it, via hpage_pincount_add/_sub().
142 		 *
143 		 * However, be sure to *also* increment the normal page refcount
144 		 * field at least once, so that the page really is pinned.
145 		 */
146 		if (hpage_pincount_available(page))
147 			hpage_pincount_add(page, refs);
148 		else
149 			page_ref_add(page, refs * (GUP_PIN_COUNTING_BIAS - 1));
150 
151 		mod_node_page_state(page_pgdat(page), NR_FOLL_PIN_ACQUIRED,
152 				    orig_refs);
153 
154 		return page;
155 	}
156 
157 	WARN_ON_ONCE(1);
158 	return NULL;
159 }
160 
put_compound_head(struct page * page,int refs,unsigned int flags)161 static void put_compound_head(struct page *page, int refs, unsigned int flags)
162 {
163 	if (flags & FOLL_PIN) {
164 		mod_node_page_state(page_pgdat(page), NR_FOLL_PIN_RELEASED,
165 				    refs);
166 
167 		if (hpage_pincount_available(page))
168 			hpage_pincount_sub(page, refs);
169 		else
170 			refs *= GUP_PIN_COUNTING_BIAS;
171 	}
172 
173 	put_page_refs(page, refs);
174 }
175 
176 /**
177  * try_grab_page() - elevate a page's refcount by a flag-dependent amount
178  *
179  * This might not do anything at all, depending on the flags argument.
180  *
181  * "grab" names in this file mean, "look at flags to decide whether to use
182  * FOLL_PIN or FOLL_GET behavior, when incrementing the page's refcount.
183  *
184  * @page:    pointer to page to be grabbed
185  * @flags:   gup flags: these are the FOLL_* flag values.
186  *
187  * Either FOLL_PIN or FOLL_GET (or neither) may be set, but not both at the same
188  * time. Cases:
189  *
190  *    FOLL_GET: page's refcount will be incremented by 1.
191  *    FOLL_PIN: page's refcount will be incremented by GUP_PIN_COUNTING_BIAS.
192  *
193  * Return: true for success, or if no action was required (if neither FOLL_PIN
194  * nor FOLL_GET was set, nothing is done). False for failure: FOLL_GET or
195  * FOLL_PIN was set, but the page could not be grabbed.
196  */
try_grab_page(struct page * page,unsigned int flags)197 bool __must_check try_grab_page(struct page *page, unsigned int flags)
198 {
199 	WARN_ON_ONCE((flags & (FOLL_GET | FOLL_PIN)) == (FOLL_GET | FOLL_PIN));
200 
201 	if (flags & FOLL_GET)
202 		return try_get_page(page);
203 	else if (flags & FOLL_PIN) {
204 		int refs = 1;
205 
206 		page = compound_head(page);
207 
208 		if (WARN_ON_ONCE(page_ref_count(page) <= 0))
209 			return false;
210 
211 		if (hpage_pincount_available(page))
212 			hpage_pincount_add(page, 1);
213 		else
214 			refs = GUP_PIN_COUNTING_BIAS;
215 
216 		/*
217 		 * Similar to try_grab_compound_head(): even if using the
218 		 * hpage_pincount_add/_sub() routines, be sure to
219 		 * *also* increment the normal page refcount field at least
220 		 * once, so that the page really is pinned.
221 		 */
222 		page_ref_add(page, refs);
223 
224 		mod_node_page_state(page_pgdat(page), NR_FOLL_PIN_ACQUIRED, 1);
225 	}
226 
227 	return true;
228 }
229 
230 /**
231  * unpin_user_page() - release a dma-pinned page
232  * @page:            pointer to page to be released
233  *
234  * Pages that were pinned via pin_user_pages*() must be released via either
235  * unpin_user_page(), or one of the unpin_user_pages*() routines. This is so
236  * that such pages can be separately tracked and uniquely handled. In
237  * particular, interactions with RDMA and filesystems need special handling.
238  */
unpin_user_page(struct page * page)239 void unpin_user_page(struct page *page)
240 {
241 	put_compound_head(compound_head(page), 1, FOLL_PIN);
242 }
243 EXPORT_SYMBOL(unpin_user_page);
244 
245 /**
246  * unpin_user_pages_dirty_lock() - release and optionally dirty gup-pinned pages
247  * @pages:  array of pages to be maybe marked dirty, and definitely released.
248  * @npages: number of pages in the @pages array.
249  * @make_dirty: whether to mark the pages dirty
250  *
251  * "gup-pinned page" refers to a page that has had one of the get_user_pages()
252  * variants called on that page.
253  *
254  * For each page in the @pages array, make that page (or its head page, if a
255  * compound page) dirty, if @make_dirty is true, and if the page was previously
256  * listed as clean. In any case, releases all pages using unpin_user_page(),
257  * possibly via unpin_user_pages(), for the non-dirty case.
258  *
259  * Please see the unpin_user_page() documentation for details.
260  *
261  * set_page_dirty_lock() is used internally. If instead, set_page_dirty() is
262  * required, then the caller should a) verify that this is really correct,
263  * because _lock() is usually required, and b) hand code it:
264  * set_page_dirty_lock(), unpin_user_page().
265  *
266  */
unpin_user_pages_dirty_lock(struct page ** pages,unsigned long npages,bool make_dirty)267 void unpin_user_pages_dirty_lock(struct page **pages, unsigned long npages,
268 				 bool make_dirty)
269 {
270 	unsigned long index;
271 
272 	/*
273 	 * TODO: this can be optimized for huge pages: if a series of pages is
274 	 * physically contiguous and part of the same compound page, then a
275 	 * single operation to the head page should suffice.
276 	 */
277 
278 	if (!make_dirty) {
279 		unpin_user_pages(pages, npages);
280 		return;
281 	}
282 
283 	for (index = 0; index < npages; index++) {
284 		struct page *page = compound_head(pages[index]);
285 		/*
286 		 * Checking PageDirty at this point may race with
287 		 * clear_page_dirty_for_io(), but that's OK. Two key
288 		 * cases:
289 		 *
290 		 * 1) This code sees the page as already dirty, so it
291 		 * skips the call to set_page_dirty(). That could happen
292 		 * because clear_page_dirty_for_io() called
293 		 * page_mkclean(), followed by set_page_dirty().
294 		 * However, now the page is going to get written back,
295 		 * which meets the original intention of setting it
296 		 * dirty, so all is well: clear_page_dirty_for_io() goes
297 		 * on to call TestClearPageDirty(), and write the page
298 		 * back.
299 		 *
300 		 * 2) This code sees the page as clean, so it calls
301 		 * set_page_dirty(). The page stays dirty, despite being
302 		 * written back, so it gets written back again in the
303 		 * next writeback cycle. This is harmless.
304 		 */
305 		if (!PageDirty(page))
306 			set_page_dirty_lock(page);
307 		unpin_user_page(page);
308 	}
309 }
310 EXPORT_SYMBOL(unpin_user_pages_dirty_lock);
311 
312 /**
313  * unpin_user_pages() - release an array of gup-pinned pages.
314  * @pages:  array of pages to be marked dirty and released.
315  * @npages: number of pages in the @pages array.
316  *
317  * For each page in the @pages array, release the page using unpin_user_page().
318  *
319  * Please see the unpin_user_page() documentation for details.
320  */
unpin_user_pages(struct page ** pages,unsigned long npages)321 void unpin_user_pages(struct page **pages, unsigned long npages)
322 {
323 	unsigned long index;
324 
325 	/*
326 	 * If this WARN_ON() fires, then the system *might* be leaking pages (by
327 	 * leaving them pinned), but probably not. More likely, gup/pup returned
328 	 * a hard -ERRNO error to the caller, who erroneously passed it here.
329 	 */
330 	if (WARN_ON(IS_ERR_VALUE(npages)))
331 		return;
332 	/*
333 	 * TODO: this can be optimized for huge pages: if a series of pages is
334 	 * physically contiguous and part of the same compound page, then a
335 	 * single operation to the head page should suffice.
336 	 */
337 	for (index = 0; index < npages; index++)
338 		unpin_user_page(pages[index]);
339 }
340 EXPORT_SYMBOL(unpin_user_pages);
341 
342 #ifdef CONFIG_MMU
no_page_table(struct vm_area_struct * vma,unsigned int flags)343 static struct page *no_page_table(struct vm_area_struct *vma,
344 		unsigned int flags)
345 {
346 	/*
347 	 * When core dumping an enormous anonymous area that nobody
348 	 * has touched so far, we don't want to allocate unnecessary pages or
349 	 * page tables.  Return error instead of NULL to skip handle_mm_fault,
350 	 * then get_dump_page() will return NULL to leave a hole in the dump.
351 	 * But we can only make this optimization where a hole would surely
352 	 * be zero-filled if handle_mm_fault() actually did handle it.
353 	 */
354 	if ((flags & FOLL_DUMP) &&
355 			(vma_is_anonymous(vma) || !vma->vm_ops->fault))
356 		return ERR_PTR(-EFAULT);
357 	return NULL;
358 }
359 
follow_pfn_pte(struct vm_area_struct * vma,unsigned long address,pte_t * pte,unsigned int flags)360 static int follow_pfn_pte(struct vm_area_struct *vma, unsigned long address,
361 		pte_t *pte, unsigned int flags)
362 {
363 	/* No page to get reference */
364 	if (flags & FOLL_GET)
365 		return -EFAULT;
366 
367 	if (flags & FOLL_TOUCH) {
368 		pte_t entry = *pte;
369 
370 		if (flags & FOLL_WRITE)
371 			entry = pte_mkdirty(entry);
372 		entry = pte_mkyoung(entry);
373 
374 		if (!pte_same(*pte, entry)) {
375 			set_pte_at(vma->vm_mm, address, pte, entry);
376 			update_mmu_cache(vma, address, pte);
377 		}
378 	}
379 
380 	/* Proper page table entry exists, but no corresponding struct page */
381 	return -EEXIST;
382 }
383 
384 /*
385  * FOLL_FORCE can write to even unwritable pte's, but only
386  * after we've gone through a COW cycle and they are dirty.
387  */
can_follow_write_pte(pte_t pte,unsigned int flags)388 static inline bool can_follow_write_pte(pte_t pte, unsigned int flags)
389 {
390 	return pte_write(pte) ||
391 		((flags & FOLL_FORCE) && (flags & FOLL_COW) && pte_dirty(pte));
392 }
393 
follow_page_pte(struct vm_area_struct * vma,unsigned long address,pmd_t * pmd,unsigned int flags,struct dev_pagemap ** pgmap)394 static struct page *follow_page_pte(struct vm_area_struct *vma,
395 		unsigned long address, pmd_t *pmd, unsigned int flags,
396 		struct dev_pagemap **pgmap)
397 {
398 	struct mm_struct *mm = vma->vm_mm;
399 	struct page *page;
400 	spinlock_t *ptl;
401 	pte_t *ptep, pte;
402 	int ret;
403 
404 	/* FOLL_GET and FOLL_PIN are mutually exclusive. */
405 	if (WARN_ON_ONCE((flags & (FOLL_PIN | FOLL_GET)) ==
406 			 (FOLL_PIN | FOLL_GET)))
407 		return ERR_PTR(-EINVAL);
408 
409 	/*
410 	 * Considering PTE level hugetlb, like continuous-PTE hugetlb on
411 	 * ARM64 architecture.
412 	 */
413 	if (is_vm_hugetlb_page(vma)) {
414 		page = follow_huge_pmd_pte(vma, address, flags);
415 		if (page)
416 			return page;
417 		return no_page_table(vma, flags);
418 	}
419 
420 retry:
421 	if (unlikely(pmd_bad(*pmd)))
422 		return no_page_table(vma, flags);
423 
424 	ptep = pte_offset_map_lock(mm, pmd, address, &ptl);
425 	pte = *ptep;
426 	if (!pte_present(pte)) {
427 		swp_entry_t entry;
428 		/*
429 		 * KSM's break_ksm() relies upon recognizing a ksm page
430 		 * even while it is being migrated, so for that case we
431 		 * need migration_entry_wait().
432 		 */
433 		if (likely(!(flags & FOLL_MIGRATION)))
434 			goto no_page;
435 		if (pte_none(pte))
436 			goto no_page;
437 		entry = pte_to_swp_entry(pte);
438 		if (!is_migration_entry(entry))
439 			goto no_page;
440 		pte_unmap_unlock(ptep, ptl);
441 		migration_entry_wait(mm, pmd, address);
442 		goto retry;
443 	}
444 	if ((flags & FOLL_NUMA) && pte_protnone(pte))
445 		goto no_page;
446 	if ((flags & FOLL_WRITE) && !can_follow_write_pte(pte, flags)) {
447 		pte_unmap_unlock(ptep, ptl);
448 		return NULL;
449 	}
450 
451 	page = vm_normal_page(vma, address, pte);
452 	if (!page && pte_devmap(pte) && (flags & (FOLL_GET | FOLL_PIN))) {
453 		/*
454 		 * Only return device mapping pages in the FOLL_GET or FOLL_PIN
455 		 * case since they are only valid while holding the pgmap
456 		 * reference.
457 		 */
458 		*pgmap = get_dev_pagemap(pte_pfn(pte), *pgmap);
459 		if (*pgmap)
460 			page = pte_page(pte);
461 		else
462 			goto no_page;
463 	} else if (unlikely(!page)) {
464 		if (flags & FOLL_DUMP) {
465 			/* Avoid special (like zero) pages in core dumps */
466 			page = ERR_PTR(-EFAULT);
467 			goto out;
468 		}
469 
470 		if (is_zero_pfn(pte_pfn(pte))) {
471 			page = pte_page(pte);
472 		} else {
473 			ret = follow_pfn_pte(vma, address, ptep, flags);
474 			page = ERR_PTR(ret);
475 			goto out;
476 		}
477 	}
478 
479 	if (flags & FOLL_SPLIT && PageTransCompound(page)) {
480 		get_page(page);
481 		pte_unmap_unlock(ptep, ptl);
482 		lock_page(page);
483 		ret = split_huge_page(page);
484 		unlock_page(page);
485 		put_page(page);
486 		if (ret)
487 			return ERR_PTR(ret);
488 		goto retry;
489 	}
490 
491 	/* try_grab_page() does nothing unless FOLL_GET or FOLL_PIN is set. */
492 	if (unlikely(!try_grab_page(page, flags))) {
493 		page = ERR_PTR(-ENOMEM);
494 		goto out;
495 	}
496 	/*
497 	 * We need to make the page accessible if and only if we are going
498 	 * to access its content (the FOLL_PIN case).  Please see
499 	 * Documentation/core-api/pin_user_pages.rst for details.
500 	 */
501 	if (flags & FOLL_PIN) {
502 		ret = arch_make_page_accessible(page);
503 		if (ret) {
504 			unpin_user_page(page);
505 			page = ERR_PTR(ret);
506 			goto out;
507 		}
508 	}
509 	if (flags & FOLL_TOUCH) {
510 		if ((flags & FOLL_WRITE) &&
511 		    !pte_dirty(pte) && !PageDirty(page))
512 			set_page_dirty(page);
513 		/*
514 		 * pte_mkyoung() would be more correct here, but atomic care
515 		 * is needed to avoid losing the dirty bit: it is easier to use
516 		 * mark_page_accessed().
517 		 */
518 		mark_page_accessed(page);
519 	}
520 	if ((flags & FOLL_MLOCK) && (vma->vm_flags & VM_LOCKED)) {
521 		/* Do not mlock pte-mapped THP */
522 		if (PageTransCompound(page))
523 			goto out;
524 
525 		/*
526 		 * The preliminary mapping check is mainly to avoid the
527 		 * pointless overhead of lock_page on the ZERO_PAGE
528 		 * which might bounce very badly if there is contention.
529 		 *
530 		 * If the page is already locked, we don't need to
531 		 * handle it now - vmscan will handle it later if and
532 		 * when it attempts to reclaim the page.
533 		 */
534 		if (page->mapping && trylock_page(page)) {
535 			lru_add_drain();  /* push cached pages to LRU */
536 			/*
537 			 * Because we lock page here, and migration is
538 			 * blocked by the pte's page reference, and we
539 			 * know the page is still mapped, we don't even
540 			 * need to check for file-cache page truncation.
541 			 */
542 			mlock_vma_page(page);
543 			unlock_page(page);
544 		}
545 	}
546 out:
547 	pte_unmap_unlock(ptep, ptl);
548 	return page;
549 no_page:
550 	pte_unmap_unlock(ptep, ptl);
551 	if (!pte_none(pte))
552 		return NULL;
553 	return no_page_table(vma, flags);
554 }
555 
follow_pmd_mask(struct vm_area_struct * vma,unsigned long address,pud_t * pudp,unsigned int flags,struct follow_page_context * ctx)556 static struct page *follow_pmd_mask(struct vm_area_struct *vma,
557 				    unsigned long address, pud_t *pudp,
558 				    unsigned int flags,
559 				    struct follow_page_context *ctx)
560 {
561 	pmd_t *pmd, pmdval;
562 	spinlock_t *ptl;
563 	struct page *page;
564 	struct mm_struct *mm = vma->vm_mm;
565 
566 	pmd = pmd_offset(pudp, address);
567 	/*
568 	 * The READ_ONCE() will stabilize the pmdval in a register or
569 	 * on the stack so that it will stop changing under the code.
570 	 */
571 	pmdval = READ_ONCE(*pmd);
572 	if (pmd_none(pmdval))
573 		return no_page_table(vma, flags);
574 	if (pmd_huge(pmdval) && is_vm_hugetlb_page(vma)) {
575 		page = follow_huge_pmd_pte(vma, address, flags);
576 		if (page)
577 			return page;
578 		return no_page_table(vma, flags);
579 	}
580 	if (is_hugepd(__hugepd(pmd_val(pmdval)))) {
581 		page = follow_huge_pd(vma, address,
582 				      __hugepd(pmd_val(pmdval)), flags,
583 				      PMD_SHIFT);
584 		if (page)
585 			return page;
586 		return no_page_table(vma, flags);
587 	}
588 retry:
589 	if (!pmd_present(pmdval)) {
590 		if (likely(!(flags & FOLL_MIGRATION)))
591 			return no_page_table(vma, flags);
592 		VM_BUG_ON(thp_migration_supported() &&
593 				  !is_pmd_migration_entry(pmdval));
594 		if (is_pmd_migration_entry(pmdval))
595 			pmd_migration_entry_wait(mm, pmd);
596 		pmdval = READ_ONCE(*pmd);
597 		/*
598 		 * MADV_DONTNEED may convert the pmd to null because
599 		 * mmap_lock is held in read mode
600 		 */
601 		if (pmd_none(pmdval))
602 			return no_page_table(vma, flags);
603 		goto retry;
604 	}
605 	if (pmd_devmap(pmdval)) {
606 		ptl = pmd_lock(mm, pmd);
607 		page = follow_devmap_pmd(vma, address, pmd, flags, &ctx->pgmap);
608 		spin_unlock(ptl);
609 		if (page)
610 			return page;
611 	}
612 	if (likely(!pmd_trans_huge(pmdval)))
613 		return follow_page_pte(vma, address, pmd, flags, &ctx->pgmap);
614 
615 	if ((flags & FOLL_NUMA) && pmd_protnone(pmdval))
616 		return no_page_table(vma, flags);
617 
618 retry_locked:
619 	ptl = pmd_lock(mm, pmd);
620 	if (unlikely(pmd_none(*pmd))) {
621 		spin_unlock(ptl);
622 		return no_page_table(vma, flags);
623 	}
624 	if (unlikely(!pmd_present(*pmd))) {
625 		spin_unlock(ptl);
626 		if (likely(!(flags & FOLL_MIGRATION)))
627 			return no_page_table(vma, flags);
628 		pmd_migration_entry_wait(mm, pmd);
629 		goto retry_locked;
630 	}
631 	if (unlikely(!pmd_trans_huge(*pmd))) {
632 		spin_unlock(ptl);
633 		return follow_page_pte(vma, address, pmd, flags, &ctx->pgmap);
634 	}
635 	if (flags & (FOLL_SPLIT | FOLL_SPLIT_PMD)) {
636 		int ret;
637 		page = pmd_page(*pmd);
638 		if (is_huge_zero_page(page)) {
639 			spin_unlock(ptl);
640 			ret = 0;
641 			split_huge_pmd(vma, pmd, address);
642 			if (pmd_trans_unstable(pmd))
643 				ret = -EBUSY;
644 		} else if (flags & FOLL_SPLIT) {
645 			if (unlikely(!try_get_page(page))) {
646 				spin_unlock(ptl);
647 				return ERR_PTR(-ENOMEM);
648 			}
649 			spin_unlock(ptl);
650 			lock_page(page);
651 			ret = split_huge_page(page);
652 			unlock_page(page);
653 			put_page(page);
654 			if (pmd_none(*pmd))
655 				return no_page_table(vma, flags);
656 		} else {  /* flags & FOLL_SPLIT_PMD */
657 			spin_unlock(ptl);
658 			split_huge_pmd(vma, pmd, address);
659 			ret = pte_alloc(mm, pmd) ? -ENOMEM : 0;
660 		}
661 
662 		return ret ? ERR_PTR(ret) :
663 			follow_page_pte(vma, address, pmd, flags, &ctx->pgmap);
664 	}
665 	page = follow_trans_huge_pmd(vma, address, pmd, flags);
666 	spin_unlock(ptl);
667 	ctx->page_mask = HPAGE_PMD_NR - 1;
668 	return page;
669 }
670 
follow_pud_mask(struct vm_area_struct * vma,unsigned long address,p4d_t * p4dp,unsigned int flags,struct follow_page_context * ctx)671 static struct page *follow_pud_mask(struct vm_area_struct *vma,
672 				    unsigned long address, p4d_t *p4dp,
673 				    unsigned int flags,
674 				    struct follow_page_context *ctx)
675 {
676 	pud_t *pud;
677 	spinlock_t *ptl;
678 	struct page *page;
679 	struct mm_struct *mm = vma->vm_mm;
680 
681 	pud = pud_offset(p4dp, address);
682 	if (pud_none(*pud))
683 		return no_page_table(vma, flags);
684 	if (pud_huge(*pud) && is_vm_hugetlb_page(vma)) {
685 		page = follow_huge_pud(mm, address, pud, flags);
686 		if (page)
687 			return page;
688 		return no_page_table(vma, flags);
689 	}
690 	if (is_hugepd(__hugepd(pud_val(*pud)))) {
691 		page = follow_huge_pd(vma, address,
692 				      __hugepd(pud_val(*pud)), flags,
693 				      PUD_SHIFT);
694 		if (page)
695 			return page;
696 		return no_page_table(vma, flags);
697 	}
698 	if (pud_devmap(*pud)) {
699 		ptl = pud_lock(mm, pud);
700 		page = follow_devmap_pud(vma, address, pud, flags, &ctx->pgmap);
701 		spin_unlock(ptl);
702 		if (page)
703 			return page;
704 	}
705 	if (unlikely(pud_bad(*pud)))
706 		return no_page_table(vma, flags);
707 
708 	return follow_pmd_mask(vma, address, pud, flags, ctx);
709 }
710 
follow_p4d_mask(struct vm_area_struct * vma,unsigned long address,pgd_t * pgdp,unsigned int flags,struct follow_page_context * ctx)711 static struct page *follow_p4d_mask(struct vm_area_struct *vma,
712 				    unsigned long address, pgd_t *pgdp,
713 				    unsigned int flags,
714 				    struct follow_page_context *ctx)
715 {
716 	p4d_t *p4d;
717 	struct page *page;
718 
719 	p4d = p4d_offset(pgdp, address);
720 	if (p4d_none(*p4d))
721 		return no_page_table(vma, flags);
722 	BUILD_BUG_ON(p4d_huge(*p4d));
723 	if (unlikely(p4d_bad(*p4d)))
724 		return no_page_table(vma, flags);
725 
726 	if (is_hugepd(__hugepd(p4d_val(*p4d)))) {
727 		page = follow_huge_pd(vma, address,
728 				      __hugepd(p4d_val(*p4d)), flags,
729 				      P4D_SHIFT);
730 		if (page)
731 			return page;
732 		return no_page_table(vma, flags);
733 	}
734 	return follow_pud_mask(vma, address, p4d, flags, ctx);
735 }
736 
737 /**
738  * follow_page_mask - look up a page descriptor from a user-virtual address
739  * @vma: vm_area_struct mapping @address
740  * @address: virtual address to look up
741  * @flags: flags modifying lookup behaviour
742  * @ctx: contains dev_pagemap for %ZONE_DEVICE memory pinning and a
743  *       pointer to output page_mask
744  *
745  * @flags can have FOLL_ flags set, defined in <linux/mm.h>
746  *
747  * When getting pages from ZONE_DEVICE memory, the @ctx->pgmap caches
748  * the device's dev_pagemap metadata to avoid repeating expensive lookups.
749  *
750  * On output, the @ctx->page_mask is set according to the size of the page.
751  *
752  * Return: the mapped (struct page *), %NULL if no mapping exists, or
753  * an error pointer if there is a mapping to something not represented
754  * by a page descriptor (see also vm_normal_page()).
755  */
follow_page_mask(struct vm_area_struct * vma,unsigned long address,unsigned int flags,struct follow_page_context * ctx)756 static struct page *follow_page_mask(struct vm_area_struct *vma,
757 			      unsigned long address, unsigned int flags,
758 			      struct follow_page_context *ctx)
759 {
760 	pgd_t *pgd;
761 	struct page *page;
762 	struct mm_struct *mm = vma->vm_mm;
763 
764 	ctx->page_mask = 0;
765 
766 	/* make this handle hugepd */
767 	page = follow_huge_addr(mm, address, flags & FOLL_WRITE);
768 	if (!IS_ERR(page)) {
769 		WARN_ON_ONCE(flags & (FOLL_GET | FOLL_PIN));
770 		return page;
771 	}
772 
773 	pgd = pgd_offset(mm, address);
774 
775 	if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd)))
776 		return no_page_table(vma, flags);
777 
778 	if (pgd_huge(*pgd)) {
779 		page = follow_huge_pgd(mm, address, pgd, flags);
780 		if (page)
781 			return page;
782 		return no_page_table(vma, flags);
783 	}
784 	if (is_hugepd(__hugepd(pgd_val(*pgd)))) {
785 		page = follow_huge_pd(vma, address,
786 				      __hugepd(pgd_val(*pgd)), flags,
787 				      PGDIR_SHIFT);
788 		if (page)
789 			return page;
790 		return no_page_table(vma, flags);
791 	}
792 
793 	return follow_p4d_mask(vma, address, pgd, flags, ctx);
794 }
795 
follow_page(struct vm_area_struct * vma,unsigned long address,unsigned int foll_flags)796 struct page *follow_page(struct vm_area_struct *vma, unsigned long address,
797 			 unsigned int foll_flags)
798 {
799 	struct follow_page_context ctx = { NULL };
800 	struct page *page;
801 
802 	page = follow_page_mask(vma, address, foll_flags, &ctx);
803 	if (ctx.pgmap)
804 		put_dev_pagemap(ctx.pgmap);
805 	return page;
806 }
807 
get_gate_page(struct mm_struct * mm,unsigned long address,unsigned int gup_flags,struct vm_area_struct ** vma,struct page ** page)808 static int get_gate_page(struct mm_struct *mm, unsigned long address,
809 		unsigned int gup_flags, struct vm_area_struct **vma,
810 		struct page **page)
811 {
812 	pgd_t *pgd;
813 	p4d_t *p4d;
814 	pud_t *pud;
815 	pmd_t *pmd;
816 	pte_t *pte;
817 	int ret = -EFAULT;
818 
819 	/* user gate pages are read-only */
820 	if (gup_flags & FOLL_WRITE)
821 		return -EFAULT;
822 	if (address > TASK_SIZE)
823 		pgd = pgd_offset_k(address);
824 	else
825 		pgd = pgd_offset_gate(mm, address);
826 	if (pgd_none(*pgd))
827 		return -EFAULT;
828 	p4d = p4d_offset(pgd, address);
829 	if (p4d_none(*p4d))
830 		return -EFAULT;
831 	pud = pud_offset(p4d, address);
832 	if (pud_none(*pud))
833 		return -EFAULT;
834 	pmd = pmd_offset(pud, address);
835 	if (!pmd_present(*pmd))
836 		return -EFAULT;
837 	VM_BUG_ON(pmd_trans_huge(*pmd));
838 	pte = pte_offset_map(pmd, address);
839 	if (pte_none(*pte))
840 		goto unmap;
841 	*vma = get_gate_vma(mm);
842 	if (!page)
843 		goto out;
844 	*page = vm_normal_page(*vma, address, *pte);
845 	if (!*page) {
846 		if ((gup_flags & FOLL_DUMP) || !is_zero_pfn(pte_pfn(*pte)))
847 			goto unmap;
848 		*page = pte_page(*pte);
849 	}
850 	if (unlikely(!try_grab_page(*page, gup_flags))) {
851 		ret = -ENOMEM;
852 		goto unmap;
853 	}
854 out:
855 	ret = 0;
856 unmap:
857 	pte_unmap(pte);
858 	return ret;
859 }
860 
861 /*
862  * mmap_lock must be held on entry.  If @locked != NULL and *@flags
863  * does not include FOLL_NOWAIT, the mmap_lock may be released.  If it
864  * is, *@locked will be set to 0 and -EBUSY returned.
865  */
faultin_page(struct vm_area_struct * vma,unsigned long address,unsigned int * flags,int * locked)866 static int faultin_page(struct vm_area_struct *vma,
867 		unsigned long address, unsigned int *flags, int *locked)
868 {
869 	unsigned int fault_flags = 0;
870 	vm_fault_t ret;
871 
872 	/* mlock all present pages, but do not fault in new pages */
873 	if ((*flags & (FOLL_POPULATE | FOLL_MLOCK)) == FOLL_MLOCK)
874 		return -ENOENT;
875 	if (*flags & FOLL_WRITE)
876 		fault_flags |= FAULT_FLAG_WRITE;
877 	if (*flags & FOLL_REMOTE)
878 		fault_flags |= FAULT_FLAG_REMOTE;
879 	if (locked)
880 		fault_flags |= FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
881 	if (*flags & FOLL_NOWAIT)
882 		fault_flags |= FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_RETRY_NOWAIT;
883 	if (*flags & FOLL_TRIED) {
884 		/*
885 		 * Note: FAULT_FLAG_ALLOW_RETRY and FAULT_FLAG_TRIED
886 		 * can co-exist
887 		 */
888 		fault_flags |= FAULT_FLAG_TRIED;
889 	}
890 
891 	ret = handle_mm_fault(vma, address, fault_flags, NULL);
892 	if (ret & VM_FAULT_ERROR) {
893 		int err = vm_fault_to_errno(ret, *flags);
894 
895 		if (err)
896 			return err;
897 		BUG();
898 	}
899 
900 	if (ret & VM_FAULT_RETRY) {
901 		if (locked && !(fault_flags & FAULT_FLAG_RETRY_NOWAIT))
902 			*locked = 0;
903 		return -EBUSY;
904 	}
905 
906 	/*
907 	 * The VM_FAULT_WRITE bit tells us that do_wp_page has broken COW when
908 	 * necessary, even if maybe_mkwrite decided not to set pte_write. We
909 	 * can thus safely do subsequent page lookups as if they were reads.
910 	 * But only do so when looping for pte_write is futile: in some cases
911 	 * userspace may also be wanting to write to the gotten user page,
912 	 * which a read fault here might prevent (a readonly page might get
913 	 * reCOWed by userspace write).
914 	 */
915 	if ((ret & VM_FAULT_WRITE) && !(vma->vm_flags & VM_WRITE))
916 		*flags |= FOLL_COW;
917 	return 0;
918 }
919 
check_vma_flags(struct vm_area_struct * vma,unsigned long gup_flags)920 static int check_vma_flags(struct vm_area_struct *vma, unsigned long gup_flags)
921 {
922 	vm_flags_t vm_flags = vma->vm_flags;
923 	int write = (gup_flags & FOLL_WRITE);
924 	int foreign = (gup_flags & FOLL_REMOTE);
925 
926 	if (vm_flags & (VM_IO | VM_PFNMAP))
927 		return -EFAULT;
928 
929 	if (gup_flags & FOLL_ANON && !vma_is_anonymous(vma))
930 		return -EFAULT;
931 
932 	if (write) {
933 		if (!(vm_flags & VM_WRITE)) {
934 			if (!(gup_flags & FOLL_FORCE))
935 				return -EFAULT;
936 			/*
937 			 * We used to let the write,force case do COW in a
938 			 * VM_MAYWRITE VM_SHARED !VM_WRITE vma, so ptrace could
939 			 * set a breakpoint in a read-only mapping of an
940 			 * executable, without corrupting the file (yet only
941 			 * when that file had been opened for writing!).
942 			 * Anon pages in shared mappings are surprising: now
943 			 * just reject it.
944 			 */
945 			if (!is_cow_mapping(vm_flags))
946 				return -EFAULT;
947 		}
948 	} else if (!(vm_flags & VM_READ)) {
949 		if (!(gup_flags & FOLL_FORCE))
950 			return -EFAULT;
951 		/*
952 		 * Is there actually any vma we can reach here which does not
953 		 * have VM_MAYREAD set?
954 		 */
955 		if (!(vm_flags & VM_MAYREAD))
956 			return -EFAULT;
957 	}
958 	/*
959 	 * gups are always data accesses, not instruction
960 	 * fetches, so execute=false here
961 	 */
962 	if (!arch_vma_access_permitted(vma, write, false, foreign))
963 		return -EFAULT;
964 	return 0;
965 }
966 
967 /**
968  * __get_user_pages() - pin user pages in memory
969  * @mm:		mm_struct of target mm
970  * @start:	starting user address
971  * @nr_pages:	number of pages from start to pin
972  * @gup_flags:	flags modifying pin behaviour
973  * @pages:	array that receives pointers to the pages pinned.
974  *		Should be at least nr_pages long. Or NULL, if caller
975  *		only intends to ensure the pages are faulted in.
976  * @vmas:	array of pointers to vmas corresponding to each page.
977  *		Or NULL if the caller does not require them.
978  * @locked:     whether we're still with the mmap_lock held
979  *
980  * Returns either number of pages pinned (which may be less than the
981  * number requested), or an error. Details about the return value:
982  *
983  * -- If nr_pages is 0, returns 0.
984  * -- If nr_pages is >0, but no pages were pinned, returns -errno.
985  * -- If nr_pages is >0, and some pages were pinned, returns the number of
986  *    pages pinned. Again, this may be less than nr_pages.
987  * -- 0 return value is possible when the fault would need to be retried.
988  *
989  * The caller is responsible for releasing returned @pages, via put_page().
990  *
991  * @vmas are valid only as long as mmap_lock is held.
992  *
993  * Must be called with mmap_lock held.  It may be released.  See below.
994  *
995  * __get_user_pages walks a process's page tables and takes a reference to
996  * each struct page that each user address corresponds to at a given
997  * instant. That is, it takes the page that would be accessed if a user
998  * thread accesses the given user virtual address at that instant.
999  *
1000  * This does not guarantee that the page exists in the user mappings when
1001  * __get_user_pages returns, and there may even be a completely different
1002  * page there in some cases (eg. if mmapped pagecache has been invalidated
1003  * and subsequently re faulted). However it does guarantee that the page
1004  * won't be freed completely. And mostly callers simply care that the page
1005  * contains data that was valid *at some point in time*. Typically, an IO
1006  * or similar operation cannot guarantee anything stronger anyway because
1007  * locks can't be held over the syscall boundary.
1008  *
1009  * If @gup_flags & FOLL_WRITE == 0, the page must not be written to. If
1010  * the page is written to, set_page_dirty (or set_page_dirty_lock, as
1011  * appropriate) must be called after the page is finished with, and
1012  * before put_page is called.
1013  *
1014  * If @locked != NULL, *@locked will be set to 0 when mmap_lock is
1015  * released by an up_read().  That can happen if @gup_flags does not
1016  * have FOLL_NOWAIT.
1017  *
1018  * A caller using such a combination of @locked and @gup_flags
1019  * must therefore hold the mmap_lock for reading only, and recognize
1020  * when it's been released.  Otherwise, it must be held for either
1021  * reading or writing and will not be released.
1022  *
1023  * In most cases, get_user_pages or get_user_pages_fast should be used
1024  * instead of __get_user_pages. __get_user_pages should be used only if
1025  * you need some special @gup_flags.
1026  */
__get_user_pages(struct mm_struct * mm,unsigned long start,unsigned long nr_pages,unsigned int gup_flags,struct page ** pages,struct vm_area_struct ** vmas,int * locked)1027 static long __get_user_pages(struct mm_struct *mm,
1028 		unsigned long start, unsigned long nr_pages,
1029 		unsigned int gup_flags, struct page **pages,
1030 		struct vm_area_struct **vmas, int *locked)
1031 {
1032 	long ret = 0, i = 0;
1033 	struct vm_area_struct *vma = NULL;
1034 	struct follow_page_context ctx = { NULL };
1035 
1036 	if (!nr_pages)
1037 		return 0;
1038 
1039 	start = untagged_addr(start);
1040 
1041 	VM_BUG_ON(!!pages != !!(gup_flags & (FOLL_GET | FOLL_PIN)));
1042 
1043 	/*
1044 	 * If FOLL_FORCE is set then do not force a full fault as the hinting
1045 	 * fault information is unrelated to the reference behaviour of a task
1046 	 * using the address space
1047 	 */
1048 	if (!(gup_flags & FOLL_FORCE))
1049 		gup_flags |= FOLL_NUMA;
1050 
1051 	do {
1052 		struct page *page;
1053 		unsigned int foll_flags = gup_flags;
1054 		unsigned int page_increm;
1055 
1056 		/* first iteration or cross vma bound */
1057 		if (!vma || start >= vma->vm_end) {
1058 			vma = find_extend_vma(mm, start);
1059 			if (!vma && in_gate_area(mm, start)) {
1060 				ret = get_gate_page(mm, start & PAGE_MASK,
1061 						gup_flags, &vma,
1062 						pages ? &pages[i] : NULL);
1063 				if (ret)
1064 					goto out;
1065 				ctx.page_mask = 0;
1066 				goto next_page;
1067 			}
1068 
1069 			if (!vma || check_vma_flags(vma, gup_flags)) {
1070 				ret = -EFAULT;
1071 				goto out;
1072 			}
1073 			if (is_vm_hugetlb_page(vma)) {
1074 				i = follow_hugetlb_page(mm, vma, pages, vmas,
1075 						&start, &nr_pages, i,
1076 						gup_flags, locked);
1077 				if (locked && *locked == 0) {
1078 					/*
1079 					 * We've got a VM_FAULT_RETRY
1080 					 * and we've lost mmap_lock.
1081 					 * We must stop here.
1082 					 */
1083 					BUG_ON(gup_flags & FOLL_NOWAIT);
1084 					BUG_ON(ret != 0);
1085 					goto out;
1086 				}
1087 				continue;
1088 			}
1089 		}
1090 retry:
1091 		/*
1092 		 * If we have a pending SIGKILL, don't keep faulting pages and
1093 		 * potentially allocating memory.
1094 		 */
1095 		if (fatal_signal_pending(current)) {
1096 			ret = -EINTR;
1097 			goto out;
1098 		}
1099 		cond_resched();
1100 
1101 		page = follow_page_mask(vma, start, foll_flags, &ctx);
1102 		if (!page) {
1103 			ret = faultin_page(vma, start, &foll_flags, locked);
1104 			switch (ret) {
1105 			case 0:
1106 				goto retry;
1107 			case -EBUSY:
1108 				ret = 0;
1109 				fallthrough;
1110 			case -EFAULT:
1111 			case -ENOMEM:
1112 			case -EHWPOISON:
1113 				goto out;
1114 			case -ENOENT:
1115 				goto next_page;
1116 			}
1117 			BUG();
1118 		} else if (PTR_ERR(page) == -EEXIST) {
1119 			/*
1120 			 * Proper page table entry exists, but no corresponding
1121 			 * struct page.
1122 			 */
1123 			goto next_page;
1124 		} else if (IS_ERR(page)) {
1125 			ret = PTR_ERR(page);
1126 			goto out;
1127 		}
1128 		if (pages) {
1129 			pages[i] = page;
1130 			flush_anon_page(vma, page, start);
1131 			flush_dcache_page(page);
1132 			ctx.page_mask = 0;
1133 		}
1134 next_page:
1135 		if (vmas) {
1136 			vmas[i] = vma;
1137 			ctx.page_mask = 0;
1138 		}
1139 		page_increm = 1 + (~(start >> PAGE_SHIFT) & ctx.page_mask);
1140 		if (page_increm > nr_pages)
1141 			page_increm = nr_pages;
1142 		i += page_increm;
1143 		start += page_increm * PAGE_SIZE;
1144 		nr_pages -= page_increm;
1145 	} while (nr_pages);
1146 out:
1147 	if (ctx.pgmap)
1148 		put_dev_pagemap(ctx.pgmap);
1149 	return i ? i : ret;
1150 }
1151 
vma_permits_fault(struct vm_area_struct * vma,unsigned int fault_flags)1152 static bool vma_permits_fault(struct vm_area_struct *vma,
1153 			      unsigned int fault_flags)
1154 {
1155 	bool write   = !!(fault_flags & FAULT_FLAG_WRITE);
1156 	bool foreign = !!(fault_flags & FAULT_FLAG_REMOTE);
1157 	vm_flags_t vm_flags = write ? VM_WRITE : VM_READ;
1158 
1159 	if (!(vm_flags & vma->vm_flags))
1160 		return false;
1161 
1162 	/*
1163 	 * The architecture might have a hardware protection
1164 	 * mechanism other than read/write that can deny access.
1165 	 *
1166 	 * gup always represents data access, not instruction
1167 	 * fetches, so execute=false here:
1168 	 */
1169 	if (!arch_vma_access_permitted(vma, write, false, foreign))
1170 		return false;
1171 
1172 	return true;
1173 }
1174 
1175 /**
1176  * fixup_user_fault() - manually resolve a user page fault
1177  * @mm:		mm_struct of target mm
1178  * @address:	user address
1179  * @fault_flags:flags to pass down to handle_mm_fault()
1180  * @unlocked:	did we unlock the mmap_lock while retrying, maybe NULL if caller
1181  *		does not allow retry. If NULL, the caller must guarantee
1182  *		that fault_flags does not contain FAULT_FLAG_ALLOW_RETRY.
1183  *
1184  * This is meant to be called in the specific scenario where for locking reasons
1185  * we try to access user memory in atomic context (within a pagefault_disable()
1186  * section), this returns -EFAULT, and we want to resolve the user fault before
1187  * trying again.
1188  *
1189  * Typically this is meant to be used by the futex code.
1190  *
1191  * The main difference with get_user_pages() is that this function will
1192  * unconditionally call handle_mm_fault() which will in turn perform all the
1193  * necessary SW fixup of the dirty and young bits in the PTE, while
1194  * get_user_pages() only guarantees to update these in the struct page.
1195  *
1196  * This is important for some architectures where those bits also gate the
1197  * access permission to the page because they are maintained in software.  On
1198  * such architectures, gup() will not be enough to make a subsequent access
1199  * succeed.
1200  *
1201  * This function will not return with an unlocked mmap_lock. So it has not the
1202  * same semantics wrt the @mm->mmap_lock as does filemap_fault().
1203  */
fixup_user_fault(struct mm_struct * mm,unsigned long address,unsigned int fault_flags,bool * unlocked)1204 int fixup_user_fault(struct mm_struct *mm,
1205 		     unsigned long address, unsigned int fault_flags,
1206 		     bool *unlocked)
1207 {
1208 	struct vm_area_struct *vma;
1209 	vm_fault_t ret, major = 0;
1210 
1211 	address = untagged_addr(address);
1212 
1213 	if (unlocked)
1214 		fault_flags |= FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
1215 
1216 retry:
1217 	vma = find_extend_vma(mm, address);
1218 	if (!vma || address < vma->vm_start)
1219 		return -EFAULT;
1220 
1221 	if (!vma_permits_fault(vma, fault_flags))
1222 		return -EFAULT;
1223 
1224 	if ((fault_flags & FAULT_FLAG_KILLABLE) &&
1225 	    fatal_signal_pending(current))
1226 		return -EINTR;
1227 
1228 	ret = handle_mm_fault(vma, address, fault_flags, NULL);
1229 	major |= ret & VM_FAULT_MAJOR;
1230 	if (ret & VM_FAULT_ERROR) {
1231 		int err = vm_fault_to_errno(ret, 0);
1232 
1233 		if (err)
1234 			return err;
1235 		BUG();
1236 	}
1237 
1238 	if (ret & VM_FAULT_RETRY) {
1239 		mmap_read_lock(mm);
1240 		*unlocked = true;
1241 		fault_flags |= FAULT_FLAG_TRIED;
1242 		goto retry;
1243 	}
1244 
1245 	return 0;
1246 }
1247 EXPORT_SYMBOL_GPL(fixup_user_fault);
1248 
1249 /*
1250  * Please note that this function, unlike __get_user_pages will not
1251  * return 0 for nr_pages > 0 without FOLL_NOWAIT
1252  */
__get_user_pages_locked(struct mm_struct * mm,unsigned long start,unsigned long nr_pages,struct page ** pages,struct vm_area_struct ** vmas,int * locked,unsigned int flags)1253 static __always_inline long __get_user_pages_locked(struct mm_struct *mm,
1254 						unsigned long start,
1255 						unsigned long nr_pages,
1256 						struct page **pages,
1257 						struct vm_area_struct **vmas,
1258 						int *locked,
1259 						unsigned int flags)
1260 {
1261 	long ret, pages_done;
1262 	bool lock_dropped;
1263 
1264 	if (locked) {
1265 		/* if VM_FAULT_RETRY can be returned, vmas become invalid */
1266 		BUG_ON(vmas);
1267 		/* check caller initialized locked */
1268 		BUG_ON(*locked != 1);
1269 	}
1270 
1271 	if (flags & FOLL_PIN)
1272 		atomic_set(&mm->has_pinned, 1);
1273 
1274 	/*
1275 	 * FOLL_PIN and FOLL_GET are mutually exclusive. Traditional behavior
1276 	 * is to set FOLL_GET if the caller wants pages[] filled in (but has
1277 	 * carelessly failed to specify FOLL_GET), so keep doing that, but only
1278 	 * for FOLL_GET, not for the newer FOLL_PIN.
1279 	 *
1280 	 * FOLL_PIN always expects pages to be non-null, but no need to assert
1281 	 * that here, as any failures will be obvious enough.
1282 	 */
1283 	if (pages && !(flags & FOLL_PIN))
1284 		flags |= FOLL_GET;
1285 
1286 	pages_done = 0;
1287 	lock_dropped = false;
1288 	for (;;) {
1289 		ret = __get_user_pages(mm, start, nr_pages, flags, pages,
1290 				       vmas, locked);
1291 		if (!locked)
1292 			/* VM_FAULT_RETRY couldn't trigger, bypass */
1293 			return ret;
1294 
1295 		/* VM_FAULT_RETRY cannot return errors */
1296 		if (!*locked) {
1297 			BUG_ON(ret < 0);
1298 			BUG_ON(ret >= nr_pages);
1299 		}
1300 
1301 		if (ret > 0) {
1302 			nr_pages -= ret;
1303 			pages_done += ret;
1304 			if (!nr_pages)
1305 				break;
1306 		}
1307 		if (*locked) {
1308 			/*
1309 			 * VM_FAULT_RETRY didn't trigger or it was a
1310 			 * FOLL_NOWAIT.
1311 			 */
1312 			if (!pages_done)
1313 				pages_done = ret;
1314 			break;
1315 		}
1316 		/*
1317 		 * VM_FAULT_RETRY triggered, so seek to the faulting offset.
1318 		 * For the prefault case (!pages) we only update counts.
1319 		 */
1320 		if (likely(pages))
1321 			pages += ret;
1322 		start += ret << PAGE_SHIFT;
1323 		lock_dropped = true;
1324 
1325 retry:
1326 		/*
1327 		 * Repeat on the address that fired VM_FAULT_RETRY
1328 		 * with both FAULT_FLAG_ALLOW_RETRY and
1329 		 * FAULT_FLAG_TRIED.  Note that GUP can be interrupted
1330 		 * by fatal signals, so we need to check it before we
1331 		 * start trying again otherwise it can loop forever.
1332 		 */
1333 
1334 		if (fatal_signal_pending(current)) {
1335 			if (!pages_done)
1336 				pages_done = -EINTR;
1337 			break;
1338 		}
1339 
1340 		ret = mmap_read_lock_killable(mm);
1341 		if (ret) {
1342 			BUG_ON(ret > 0);
1343 			if (!pages_done)
1344 				pages_done = ret;
1345 			break;
1346 		}
1347 
1348 		*locked = 1;
1349 		ret = __get_user_pages(mm, start, 1, flags | FOLL_TRIED,
1350 				       pages, NULL, locked);
1351 		if (!*locked) {
1352 			/* Continue to retry until we succeeded */
1353 			BUG_ON(ret != 0);
1354 			goto retry;
1355 		}
1356 		if (ret != 1) {
1357 			BUG_ON(ret > 1);
1358 			if (!pages_done)
1359 				pages_done = ret;
1360 			break;
1361 		}
1362 		nr_pages--;
1363 		pages_done++;
1364 		if (!nr_pages)
1365 			break;
1366 		if (likely(pages))
1367 			pages++;
1368 		start += PAGE_SIZE;
1369 	}
1370 	if (lock_dropped && *locked) {
1371 		/*
1372 		 * We must let the caller know we temporarily dropped the lock
1373 		 * and so the critical section protected by it was lost.
1374 		 */
1375 		mmap_read_unlock(mm);
1376 		*locked = 0;
1377 	}
1378 	return pages_done;
1379 }
1380 
1381 /**
1382  * populate_vma_page_range() -  populate a range of pages in the vma.
1383  * @vma:   target vma
1384  * @start: start address
1385  * @end:   end address
1386  * @locked: whether the mmap_lock is still held
1387  *
1388  * This takes care of mlocking the pages too if VM_LOCKED is set.
1389  *
1390  * Return either number of pages pinned in the vma, or a negative error
1391  * code on error.
1392  *
1393  * vma->vm_mm->mmap_lock must be held.
1394  *
1395  * If @locked is NULL, it may be held for read or write and will
1396  * be unperturbed.
1397  *
1398  * If @locked is non-NULL, it must held for read only and may be
1399  * released.  If it's released, *@locked will be set to 0.
1400  */
populate_vma_page_range(struct vm_area_struct * vma,unsigned long start,unsigned long end,int * locked)1401 long populate_vma_page_range(struct vm_area_struct *vma,
1402 		unsigned long start, unsigned long end, int *locked)
1403 {
1404 	struct mm_struct *mm = vma->vm_mm;
1405 	unsigned long nr_pages = (end - start) / PAGE_SIZE;
1406 	int gup_flags;
1407 
1408 	VM_BUG_ON(start & ~PAGE_MASK);
1409 	VM_BUG_ON(end   & ~PAGE_MASK);
1410 	VM_BUG_ON_VMA(start < vma->vm_start, vma);
1411 	VM_BUG_ON_VMA(end   > vma->vm_end, vma);
1412 	mmap_assert_locked(mm);
1413 
1414 	gup_flags = FOLL_TOUCH | FOLL_POPULATE | FOLL_MLOCK;
1415 	if (vma->vm_flags & VM_LOCKONFAULT)
1416 		gup_flags &= ~FOLL_POPULATE;
1417 	/*
1418 	 * We want to touch writable mappings with a write fault in order
1419 	 * to break COW, except for shared mappings because these don't COW
1420 	 * and we would not want to dirty them for nothing.
1421 	 */
1422 	if ((vma->vm_flags & (VM_WRITE | VM_SHARED)) == VM_WRITE)
1423 		gup_flags |= FOLL_WRITE;
1424 
1425 	/*
1426 	 * We want mlock to succeed for regions that have any permissions
1427 	 * other than PROT_NONE.
1428 	 */
1429 	if (vma_is_accessible(vma))
1430 		gup_flags |= FOLL_FORCE;
1431 
1432 	/*
1433 	 * We made sure addr is within a VMA, so the following will
1434 	 * not result in a stack expansion that recurses back here.
1435 	 */
1436 	return __get_user_pages(mm, start, nr_pages, gup_flags,
1437 				NULL, NULL, locked);
1438 }
1439 
1440 /*
1441  * __mm_populate - populate and/or mlock pages within a range of address space.
1442  *
1443  * This is used to implement mlock() and the MAP_POPULATE / MAP_LOCKED mmap
1444  * flags. VMAs must be already marked with the desired vm_flags, and
1445  * mmap_lock must not be held.
1446  */
__mm_populate(unsigned long start,unsigned long len,int ignore_errors)1447 int __mm_populate(unsigned long start, unsigned long len, int ignore_errors)
1448 {
1449 	struct mm_struct *mm = current->mm;
1450 	unsigned long end, nstart, nend;
1451 	struct vm_area_struct *vma = NULL;
1452 	int locked = 0;
1453 	long ret = 0;
1454 
1455 	end = start + len;
1456 
1457 	for (nstart = start; nstart < end; nstart = nend) {
1458 		/*
1459 		 * We want to fault in pages for [nstart; end) address range.
1460 		 * Find first corresponding VMA.
1461 		 */
1462 		if (!locked) {
1463 			locked = 1;
1464 			mmap_read_lock(mm);
1465 			vma = find_vma(mm, nstart);
1466 		} else if (nstart >= vma->vm_end)
1467 			vma = vma->vm_next;
1468 		if (!vma || vma->vm_start >= end)
1469 			break;
1470 		/*
1471 		 * Set [nstart; nend) to intersection of desired address
1472 		 * range with the first VMA. Also, skip undesirable VMA types.
1473 		 */
1474 		nend = min(end, vma->vm_end);
1475 		if (vma->vm_flags & (VM_IO | VM_PFNMAP))
1476 			continue;
1477 		if (nstart < vma->vm_start)
1478 			nstart = vma->vm_start;
1479 		/*
1480 		 * Now fault in a range of pages. populate_vma_page_range()
1481 		 * double checks the vma flags, so that it won't mlock pages
1482 		 * if the vma was already munlocked.
1483 		 */
1484 		ret = populate_vma_page_range(vma, nstart, nend, &locked);
1485 		if (ret < 0) {
1486 			if (ignore_errors) {
1487 				ret = 0;
1488 				continue;	/* continue at next VMA */
1489 			}
1490 			break;
1491 		}
1492 		nend = nstart + ret * PAGE_SIZE;
1493 		ret = 0;
1494 	}
1495 	if (locked)
1496 		mmap_read_unlock(mm);
1497 	return ret;	/* 0 or negative error code */
1498 }
1499 #else /* CONFIG_MMU */
__get_user_pages_locked(struct mm_struct * mm,unsigned long start,unsigned long nr_pages,struct page ** pages,struct vm_area_struct ** vmas,int * locked,unsigned int foll_flags)1500 static long __get_user_pages_locked(struct mm_struct *mm, unsigned long start,
1501 		unsigned long nr_pages, struct page **pages,
1502 		struct vm_area_struct **vmas, int *locked,
1503 		unsigned int foll_flags)
1504 {
1505 	struct vm_area_struct *vma;
1506 	unsigned long vm_flags;
1507 	int i;
1508 
1509 	/* calculate required read or write permissions.
1510 	 * If FOLL_FORCE is set, we only require the "MAY" flags.
1511 	 */
1512 	vm_flags  = (foll_flags & FOLL_WRITE) ?
1513 			(VM_WRITE | VM_MAYWRITE) : (VM_READ | VM_MAYREAD);
1514 	vm_flags &= (foll_flags & FOLL_FORCE) ?
1515 			(VM_MAYREAD | VM_MAYWRITE) : (VM_READ | VM_WRITE);
1516 
1517 	for (i = 0; i < nr_pages; i++) {
1518 		vma = find_vma(mm, start);
1519 		if (!vma)
1520 			goto finish_or_fault;
1521 
1522 		/* protect what we can, including chardevs */
1523 		if ((vma->vm_flags & (VM_IO | VM_PFNMAP)) ||
1524 		    !(vm_flags & vma->vm_flags))
1525 			goto finish_or_fault;
1526 
1527 		if (pages) {
1528 			pages[i] = virt_to_page(start);
1529 			if (pages[i])
1530 				get_page(pages[i]);
1531 		}
1532 		if (vmas)
1533 			vmas[i] = vma;
1534 		start = (start + PAGE_SIZE) & PAGE_MASK;
1535 	}
1536 
1537 	return i;
1538 
1539 finish_or_fault:
1540 	return i ? : -EFAULT;
1541 }
1542 #endif /* !CONFIG_MMU */
1543 
1544 /**
1545  * get_dump_page() - pin user page in memory while writing it to core dump
1546  * @addr: user address
1547  *
1548  * Returns struct page pointer of user page pinned for dump,
1549  * to be freed afterwards by put_page().
1550  *
1551  * Returns NULL on any kind of failure - a hole must then be inserted into
1552  * the corefile, to preserve alignment with its headers; and also returns
1553  * NULL wherever the ZERO_PAGE, or an anonymous pte_none, has been found -
1554  * allowing a hole to be left in the corefile to save diskspace.
1555  *
1556  * Called without mmap_lock (takes and releases the mmap_lock by itself).
1557  */
1558 #ifdef CONFIG_ELF_CORE
get_dump_page(unsigned long addr)1559 struct page *get_dump_page(unsigned long addr)
1560 {
1561 	struct mm_struct *mm = current->mm;
1562 	struct page *page;
1563 	int locked = 1;
1564 	int ret;
1565 
1566 	if (mmap_read_lock_killable(mm))
1567 		return NULL;
1568 	ret = __get_user_pages_locked(mm, addr, 1, &page, NULL, &locked,
1569 				      FOLL_FORCE | FOLL_DUMP | FOLL_GET);
1570 	if (locked)
1571 		mmap_read_unlock(mm);
1572 	return (ret == 1) ? page : NULL;
1573 }
1574 #endif /* CONFIG_ELF_CORE */
1575 
1576 #if defined(CONFIG_FS_DAX) || defined (CONFIG_CMA)
check_dax_vmas(struct vm_area_struct ** vmas,long nr_pages)1577 static bool check_dax_vmas(struct vm_area_struct **vmas, long nr_pages)
1578 {
1579 	long i;
1580 	struct vm_area_struct *vma_prev = NULL;
1581 
1582 	for (i = 0; i < nr_pages; i++) {
1583 		struct vm_area_struct *vma = vmas[i];
1584 
1585 		if (vma == vma_prev)
1586 			continue;
1587 
1588 		vma_prev = vma;
1589 
1590 		if (vma_is_fsdax(vma))
1591 			return true;
1592 	}
1593 	return false;
1594 }
1595 
1596 #ifdef CONFIG_CMA
check_and_migrate_cma_pages(struct mm_struct * mm,unsigned long start,unsigned long nr_pages,struct page ** pages,struct vm_area_struct ** vmas,unsigned int gup_flags)1597 static long check_and_migrate_cma_pages(struct mm_struct *mm,
1598 					unsigned long start,
1599 					unsigned long nr_pages,
1600 					struct page **pages,
1601 					struct vm_area_struct **vmas,
1602 					unsigned int gup_flags)
1603 {
1604 	unsigned long i, isolation_error_count;
1605 	bool drain_allow;
1606 	LIST_HEAD(cma_page_list);
1607 	long ret = nr_pages;
1608 	struct page *prev_head, *head;
1609 	struct migration_target_control mtc = {
1610 		.nid = NUMA_NO_NODE,
1611 		.gfp_mask = GFP_USER | __GFP_MOVABLE | __GFP_NOWARN,
1612 	};
1613 
1614 check_again:
1615 	prev_head = NULL;
1616 	isolation_error_count = 0;
1617 	drain_allow = true;
1618 	for (i = 0; i < nr_pages; i++) {
1619 		head = compound_head(pages[i]);
1620 		if (head == prev_head)
1621 			continue;
1622 		prev_head = head;
1623 		/*
1624 		 * If we get a page from the CMA zone, since we are going to
1625 		 * be pinning these entries, we might as well move them out
1626 		 * of the CMA zone if possible.
1627 		 */
1628 		if (is_migrate_cma_page(head)) {
1629 			if (PageHuge(head)) {
1630 				if (!isolate_huge_page(head, &cma_page_list))
1631 					isolation_error_count++;
1632 			} else {
1633 				if (!PageLRU(head) && drain_allow) {
1634 					lru_add_drain_all();
1635 					drain_allow = false;
1636 				}
1637 
1638 				if (isolate_lru_page(head)) {
1639 					isolation_error_count++;
1640 					continue;
1641 				}
1642 				list_add_tail(&head->lru, &cma_page_list);
1643 				mod_node_page_state(page_pgdat(head),
1644 						    NR_ISOLATED_ANON +
1645 						    page_is_file_lru(head),
1646 						    thp_nr_pages(head));
1647 			}
1648 		}
1649 	}
1650 
1651 	/*
1652 	 * If list is empty, and no isolation errors, means that all pages are
1653 	 * in the correct zone.
1654 	 */
1655 	if (list_empty(&cma_page_list) && !isolation_error_count)
1656 		return ret;
1657 
1658 	if (!list_empty(&cma_page_list)) {
1659 		/*
1660 		 * drop the above get_user_pages reference.
1661 		 */
1662 		if (gup_flags & FOLL_PIN)
1663 			unpin_user_pages(pages, nr_pages);
1664 		else
1665 			for (i = 0; i < nr_pages; i++)
1666 				put_page(pages[i]);
1667 
1668 		ret = migrate_pages(&cma_page_list, alloc_migration_target,
1669 				    NULL, (unsigned long)&mtc, MIGRATE_SYNC,
1670 				    MR_CONTIG_RANGE);
1671 		if (ret) {
1672 			if (!list_empty(&cma_page_list))
1673 				putback_movable_pages(&cma_page_list);
1674 			return ret > 0 ? -ENOMEM : ret;
1675 		}
1676 
1677 		/* We unpinned pages before migration, pin them again */
1678 		ret = __get_user_pages_locked(mm, start, nr_pages, pages, vmas,
1679 					      NULL, gup_flags);
1680 		if (ret <= 0)
1681 			return ret;
1682 		nr_pages = ret;
1683 	}
1684 
1685 	/*
1686 	 * check again because pages were unpinned, and we also might have
1687 	 * had isolation errors and need more pages to migrate.
1688 	 */
1689 	goto check_again;
1690 }
1691 #else
check_and_migrate_cma_pages(struct mm_struct * mm,unsigned long start,unsigned long nr_pages,struct page ** pages,struct vm_area_struct ** vmas,unsigned int gup_flags)1692 static long check_and_migrate_cma_pages(struct mm_struct *mm,
1693 					unsigned long start,
1694 					unsigned long nr_pages,
1695 					struct page **pages,
1696 					struct vm_area_struct **vmas,
1697 					unsigned int gup_flags)
1698 {
1699 	return nr_pages;
1700 }
1701 #endif /* CONFIG_CMA */
1702 
1703 /*
1704  * __gup_longterm_locked() is a wrapper for __get_user_pages_locked which
1705  * allows us to process the FOLL_LONGTERM flag.
1706  */
__gup_longterm_locked(struct mm_struct * mm,unsigned long start,unsigned long nr_pages,struct page ** pages,struct vm_area_struct ** vmas,unsigned int gup_flags)1707 static long __gup_longterm_locked(struct mm_struct *mm,
1708 				  unsigned long start,
1709 				  unsigned long nr_pages,
1710 				  struct page **pages,
1711 				  struct vm_area_struct **vmas,
1712 				  unsigned int gup_flags)
1713 {
1714 	struct vm_area_struct **vmas_tmp = vmas;
1715 	unsigned long flags = 0;
1716 	long rc, i;
1717 
1718 	if (gup_flags & FOLL_LONGTERM) {
1719 		if (!pages)
1720 			return -EINVAL;
1721 
1722 		if (!vmas_tmp) {
1723 			vmas_tmp = kcalloc(nr_pages,
1724 					   sizeof(struct vm_area_struct *),
1725 					   GFP_KERNEL);
1726 			if (!vmas_tmp)
1727 				return -ENOMEM;
1728 		}
1729 		flags = memalloc_nocma_save();
1730 	}
1731 
1732 	rc = __get_user_pages_locked(mm, start, nr_pages, pages,
1733 				     vmas_tmp, NULL, gup_flags);
1734 
1735 	if (gup_flags & FOLL_LONGTERM) {
1736 		if (rc < 0)
1737 			goto out;
1738 
1739 		if (check_dax_vmas(vmas_tmp, rc)) {
1740 			if (gup_flags & FOLL_PIN)
1741 				unpin_user_pages(pages, rc);
1742 			else
1743 				for (i = 0; i < rc; i++)
1744 					put_page(pages[i]);
1745 			rc = -EOPNOTSUPP;
1746 			goto out;
1747 		}
1748 
1749 		rc = check_and_migrate_cma_pages(mm, start, rc, pages,
1750 						 vmas_tmp, gup_flags);
1751 out:
1752 		memalloc_nocma_restore(flags);
1753 	}
1754 
1755 	if (vmas_tmp != vmas)
1756 		kfree(vmas_tmp);
1757 	return rc;
1758 }
1759 #else /* !CONFIG_FS_DAX && !CONFIG_CMA */
__gup_longterm_locked(struct mm_struct * mm,unsigned long start,unsigned long nr_pages,struct page ** pages,struct vm_area_struct ** vmas,unsigned int flags)1760 static __always_inline long __gup_longterm_locked(struct mm_struct *mm,
1761 						  unsigned long start,
1762 						  unsigned long nr_pages,
1763 						  struct page **pages,
1764 						  struct vm_area_struct **vmas,
1765 						  unsigned int flags)
1766 {
1767 	return __get_user_pages_locked(mm, start, nr_pages, pages, vmas,
1768 				       NULL, flags);
1769 }
1770 #endif /* CONFIG_FS_DAX || CONFIG_CMA */
1771 
is_valid_gup_flags(unsigned int gup_flags)1772 static bool is_valid_gup_flags(unsigned int gup_flags)
1773 {
1774 	/*
1775 	 * FOLL_PIN must only be set internally by the pin_user_pages*() APIs,
1776 	 * never directly by the caller, so enforce that with an assertion:
1777 	 */
1778 	if (WARN_ON_ONCE(gup_flags & FOLL_PIN))
1779 		return false;
1780 	/*
1781 	 * FOLL_PIN is a prerequisite to FOLL_LONGTERM. Another way of saying
1782 	 * that is, FOLL_LONGTERM is a specific case, more restrictive case of
1783 	 * FOLL_PIN.
1784 	 */
1785 	if (WARN_ON_ONCE(gup_flags & FOLL_LONGTERM))
1786 		return false;
1787 
1788 	return true;
1789 }
1790 
1791 #ifdef CONFIG_MMU
__get_user_pages_remote(struct mm_struct * mm,unsigned long start,unsigned long nr_pages,unsigned int gup_flags,struct page ** pages,struct vm_area_struct ** vmas,int * locked)1792 static long __get_user_pages_remote(struct mm_struct *mm,
1793 				    unsigned long start, unsigned long nr_pages,
1794 				    unsigned int gup_flags, struct page **pages,
1795 				    struct vm_area_struct **vmas, int *locked)
1796 {
1797 	/*
1798 	 * Parts of FOLL_LONGTERM behavior are incompatible with
1799 	 * FAULT_FLAG_ALLOW_RETRY because of the FS DAX check requirement on
1800 	 * vmas. However, this only comes up if locked is set, and there are
1801 	 * callers that do request FOLL_LONGTERM, but do not set locked. So,
1802 	 * allow what we can.
1803 	 */
1804 	if (gup_flags & FOLL_LONGTERM) {
1805 		if (WARN_ON_ONCE(locked))
1806 			return -EINVAL;
1807 		/*
1808 		 * This will check the vmas (even if our vmas arg is NULL)
1809 		 * and return -ENOTSUPP if DAX isn't allowed in this case:
1810 		 */
1811 		return __gup_longterm_locked(mm, start, nr_pages, pages,
1812 					     vmas, gup_flags | FOLL_TOUCH |
1813 					     FOLL_REMOTE);
1814 	}
1815 
1816 	return __get_user_pages_locked(mm, start, nr_pages, pages, vmas,
1817 				       locked,
1818 				       gup_flags | FOLL_TOUCH | FOLL_REMOTE);
1819 }
1820 
1821 /**
1822  * get_user_pages_remote() - pin user pages in memory
1823  * @mm:		mm_struct of target mm
1824  * @start:	starting user address
1825  * @nr_pages:	number of pages from start to pin
1826  * @gup_flags:	flags modifying lookup behaviour
1827  * @pages:	array that receives pointers to the pages pinned.
1828  *		Should be at least nr_pages long. Or NULL, if caller
1829  *		only intends to ensure the pages are faulted in.
1830  * @vmas:	array of pointers to vmas corresponding to each page.
1831  *		Or NULL if the caller does not require them.
1832  * @locked:	pointer to lock flag indicating whether lock is held and
1833  *		subsequently whether VM_FAULT_RETRY functionality can be
1834  *		utilised. Lock must initially be held.
1835  *
1836  * Returns either number of pages pinned (which may be less than the
1837  * number requested), or an error. Details about the return value:
1838  *
1839  * -- If nr_pages is 0, returns 0.
1840  * -- If nr_pages is >0, but no pages were pinned, returns -errno.
1841  * -- If nr_pages is >0, and some pages were pinned, returns the number of
1842  *    pages pinned. Again, this may be less than nr_pages.
1843  *
1844  * The caller is responsible for releasing returned @pages, via put_page().
1845  *
1846  * @vmas are valid only as long as mmap_lock is held.
1847  *
1848  * Must be called with mmap_lock held for read or write.
1849  *
1850  * get_user_pages_remote walks a process's page tables and takes a reference
1851  * to each struct page that each user address corresponds to at a given
1852  * instant. That is, it takes the page that would be accessed if a user
1853  * thread accesses the given user virtual address at that instant.
1854  *
1855  * This does not guarantee that the page exists in the user mappings when
1856  * get_user_pages_remote returns, and there may even be a completely different
1857  * page there in some cases (eg. if mmapped pagecache has been invalidated
1858  * and subsequently re faulted). However it does guarantee that the page
1859  * won't be freed completely. And mostly callers simply care that the page
1860  * contains data that was valid *at some point in time*. Typically, an IO
1861  * or similar operation cannot guarantee anything stronger anyway because
1862  * locks can't be held over the syscall boundary.
1863  *
1864  * If gup_flags & FOLL_WRITE == 0, the page must not be written to. If the page
1865  * is written to, set_page_dirty (or set_page_dirty_lock, as appropriate) must
1866  * be called after the page is finished with, and before put_page is called.
1867  *
1868  * get_user_pages_remote is typically used for fewer-copy IO operations,
1869  * to get a handle on the memory by some means other than accesses
1870  * via the user virtual addresses. The pages may be submitted for
1871  * DMA to devices or accessed via their kernel linear mapping (via the
1872  * kmap APIs). Care should be taken to use the correct cache flushing APIs.
1873  *
1874  * See also get_user_pages_fast, for performance critical applications.
1875  *
1876  * get_user_pages_remote should be phased out in favor of
1877  * get_user_pages_locked|unlocked or get_user_pages_fast. Nothing
1878  * should use get_user_pages_remote because it cannot pass
1879  * FAULT_FLAG_ALLOW_RETRY to handle_mm_fault.
1880  */
get_user_pages_remote(struct mm_struct * mm,unsigned long start,unsigned long nr_pages,unsigned int gup_flags,struct page ** pages,struct vm_area_struct ** vmas,int * locked)1881 long get_user_pages_remote(struct mm_struct *mm,
1882 		unsigned long start, unsigned long nr_pages,
1883 		unsigned int gup_flags, struct page **pages,
1884 		struct vm_area_struct **vmas, int *locked)
1885 {
1886 	if (!is_valid_gup_flags(gup_flags))
1887 		return -EINVAL;
1888 
1889 	return __get_user_pages_remote(mm, start, nr_pages, gup_flags,
1890 				       pages, vmas, locked);
1891 }
1892 EXPORT_SYMBOL(get_user_pages_remote);
1893 
1894 #else /* CONFIG_MMU */
get_user_pages_remote(struct mm_struct * mm,unsigned long start,unsigned long nr_pages,unsigned int gup_flags,struct page ** pages,struct vm_area_struct ** vmas,int * locked)1895 long get_user_pages_remote(struct mm_struct *mm,
1896 			   unsigned long start, unsigned long nr_pages,
1897 			   unsigned int gup_flags, struct page **pages,
1898 			   struct vm_area_struct **vmas, int *locked)
1899 {
1900 	return 0;
1901 }
1902 
__get_user_pages_remote(struct mm_struct * mm,unsigned long start,unsigned long nr_pages,unsigned int gup_flags,struct page ** pages,struct vm_area_struct ** vmas,int * locked)1903 static long __get_user_pages_remote(struct mm_struct *mm,
1904 				    unsigned long start, unsigned long nr_pages,
1905 				    unsigned int gup_flags, struct page **pages,
1906 				    struct vm_area_struct **vmas, int *locked)
1907 {
1908 	return 0;
1909 }
1910 #endif /* !CONFIG_MMU */
1911 
1912 /**
1913  * get_user_pages() - pin user pages in memory
1914  * @start:      starting user address
1915  * @nr_pages:   number of pages from start to pin
1916  * @gup_flags:  flags modifying lookup behaviour
1917  * @pages:      array that receives pointers to the pages pinned.
1918  *              Should be at least nr_pages long. Or NULL, if caller
1919  *              only intends to ensure the pages are faulted in.
1920  * @vmas:       array of pointers to vmas corresponding to each page.
1921  *              Or NULL if the caller does not require them.
1922  *
1923  * This is the same as get_user_pages_remote(), just with a less-flexible
1924  * calling convention where we assume that the mm being operated on belongs to
1925  * the current task, and doesn't allow passing of a locked parameter.  We also
1926  * obviously don't pass FOLL_REMOTE in here.
1927  */
get_user_pages(unsigned long start,unsigned long nr_pages,unsigned int gup_flags,struct page ** pages,struct vm_area_struct ** vmas)1928 long get_user_pages(unsigned long start, unsigned long nr_pages,
1929 		unsigned int gup_flags, struct page **pages,
1930 		struct vm_area_struct **vmas)
1931 {
1932 	if (!is_valid_gup_flags(gup_flags))
1933 		return -EINVAL;
1934 
1935 	return __gup_longterm_locked(current->mm, start, nr_pages,
1936 				     pages, vmas, gup_flags | FOLL_TOUCH);
1937 }
1938 EXPORT_SYMBOL(get_user_pages);
1939 
1940 /**
1941  * get_user_pages_locked() is suitable to replace the form:
1942  *
1943  *      mmap_read_lock(mm);
1944  *      do_something()
1945  *      get_user_pages(mm, ..., pages, NULL);
1946  *      mmap_read_unlock(mm);
1947  *
1948  *  to:
1949  *
1950  *      int locked = 1;
1951  *      mmap_read_lock(mm);
1952  *      do_something()
1953  *      get_user_pages_locked(mm, ..., pages, &locked);
1954  *      if (locked)
1955  *          mmap_read_unlock(mm);
1956  *
1957  * @start:      starting user address
1958  * @nr_pages:   number of pages from start to pin
1959  * @gup_flags:  flags modifying lookup behaviour
1960  * @pages:      array that receives pointers to the pages pinned.
1961  *              Should be at least nr_pages long. Or NULL, if caller
1962  *              only intends to ensure the pages are faulted in.
1963  * @locked:     pointer to lock flag indicating whether lock is held and
1964  *              subsequently whether VM_FAULT_RETRY functionality can be
1965  *              utilised. Lock must initially be held.
1966  *
1967  * We can leverage the VM_FAULT_RETRY functionality in the page fault
1968  * paths better by using either get_user_pages_locked() or
1969  * get_user_pages_unlocked().
1970  *
1971  */
get_user_pages_locked(unsigned long start,unsigned long nr_pages,unsigned int gup_flags,struct page ** pages,int * locked)1972 long get_user_pages_locked(unsigned long start, unsigned long nr_pages,
1973 			   unsigned int gup_flags, struct page **pages,
1974 			   int *locked)
1975 {
1976 	/*
1977 	 * FIXME: Current FOLL_LONGTERM behavior is incompatible with
1978 	 * FAULT_FLAG_ALLOW_RETRY because of the FS DAX check requirement on
1979 	 * vmas.  As there are no users of this flag in this call we simply
1980 	 * disallow this option for now.
1981 	 */
1982 	if (WARN_ON_ONCE(gup_flags & FOLL_LONGTERM))
1983 		return -EINVAL;
1984 	/*
1985 	 * FOLL_PIN must only be set internally by the pin_user_pages*() APIs,
1986 	 * never directly by the caller, so enforce that:
1987 	 */
1988 	if (WARN_ON_ONCE(gup_flags & FOLL_PIN))
1989 		return -EINVAL;
1990 
1991 	return __get_user_pages_locked(current->mm, start, nr_pages,
1992 				       pages, NULL, locked,
1993 				       gup_flags | FOLL_TOUCH);
1994 }
1995 EXPORT_SYMBOL(get_user_pages_locked);
1996 
1997 /*
1998  * get_user_pages_unlocked() is suitable to replace the form:
1999  *
2000  *      mmap_read_lock(mm);
2001  *      get_user_pages(mm, ..., pages, NULL);
2002  *      mmap_read_unlock(mm);
2003  *
2004  *  with:
2005  *
2006  *      get_user_pages_unlocked(mm, ..., pages);
2007  *
2008  * It is functionally equivalent to get_user_pages_fast so
2009  * get_user_pages_fast should be used instead if specific gup_flags
2010  * (e.g. FOLL_FORCE) are not required.
2011  */
get_user_pages_unlocked(unsigned long start,unsigned long nr_pages,struct page ** pages,unsigned int gup_flags)2012 long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages,
2013 			     struct page **pages, unsigned int gup_flags)
2014 {
2015 	struct mm_struct *mm = current->mm;
2016 	int locked = 1;
2017 	long ret;
2018 
2019 	/*
2020 	 * FIXME: Current FOLL_LONGTERM behavior is incompatible with
2021 	 * FAULT_FLAG_ALLOW_RETRY because of the FS DAX check requirement on
2022 	 * vmas.  As there are no users of this flag in this call we simply
2023 	 * disallow this option for now.
2024 	 */
2025 	if (WARN_ON_ONCE(gup_flags & FOLL_LONGTERM))
2026 		return -EINVAL;
2027 
2028 	mmap_read_lock(mm);
2029 	ret = __get_user_pages_locked(mm, start, nr_pages, pages, NULL,
2030 				      &locked, gup_flags | FOLL_TOUCH);
2031 	if (locked)
2032 		mmap_read_unlock(mm);
2033 	return ret;
2034 }
2035 EXPORT_SYMBOL(get_user_pages_unlocked);
2036 
2037 /*
2038  * Fast GUP
2039  *
2040  * get_user_pages_fast attempts to pin user pages by walking the page
2041  * tables directly and avoids taking locks. Thus the walker needs to be
2042  * protected from page table pages being freed from under it, and should
2043  * block any THP splits.
2044  *
2045  * One way to achieve this is to have the walker disable interrupts, and
2046  * rely on IPIs from the TLB flushing code blocking before the page table
2047  * pages are freed. This is unsuitable for architectures that do not need
2048  * to broadcast an IPI when invalidating TLBs.
2049  *
2050  * Another way to achieve this is to batch up page table containing pages
2051  * belonging to more than one mm_user, then rcu_sched a callback to free those
2052  * pages. Disabling interrupts will allow the fast_gup walker to both block
2053  * the rcu_sched callback, and an IPI that we broadcast for splitting THPs
2054  * (which is a relatively rare event). The code below adopts this strategy.
2055  *
2056  * Before activating this code, please be aware that the following assumptions
2057  * are currently made:
2058  *
2059  *  *) Either MMU_GATHER_RCU_TABLE_FREE is enabled, and tlb_remove_table() is used to
2060  *  free pages containing page tables or TLB flushing requires IPI broadcast.
2061  *
2062  *  *) ptes can be read atomically by the architecture.
2063  *
2064  *  *) access_ok is sufficient to validate userspace address ranges.
2065  *
2066  * The last two assumptions can be relaxed by the addition of helper functions.
2067  *
2068  * This code is based heavily on the PowerPC implementation by Nick Piggin.
2069  */
2070 #ifdef CONFIG_HAVE_FAST_GUP
2071 #ifdef CONFIG_GUP_GET_PTE_LOW_HIGH
2072 
2073 /*
2074  * WARNING: only to be used in the get_user_pages_fast() implementation.
2075  *
2076  * With get_user_pages_fast(), we walk down the pagetables without taking any
2077  * locks.  For this we would like to load the pointers atomically, but sometimes
2078  * that is not possible (e.g. without expensive cmpxchg8b on x86_32 PAE).  What
2079  * we do have is the guarantee that a PTE will only either go from not present
2080  * to present, or present to not present or both -- it will not switch to a
2081  * completely different present page without a TLB flush in between; something
2082  * that we are blocking by holding interrupts off.
2083  *
2084  * Setting ptes from not present to present goes:
2085  *
2086  *   ptep->pte_high = h;
2087  *   smp_wmb();
2088  *   ptep->pte_low = l;
2089  *
2090  * And present to not present goes:
2091  *
2092  *   ptep->pte_low = 0;
2093  *   smp_wmb();
2094  *   ptep->pte_high = 0;
2095  *
2096  * We must ensure here that the load of pte_low sees 'l' IFF pte_high sees 'h'.
2097  * We load pte_high *after* loading pte_low, which ensures we don't see an older
2098  * value of pte_high.  *Then* we recheck pte_low, which ensures that we haven't
2099  * picked up a changed pte high. We might have gotten rubbish values from
2100  * pte_low and pte_high, but we are guaranteed that pte_low will not have the
2101  * present bit set *unless* it is 'l'. Because get_user_pages_fast() only
2102  * operates on present ptes we're safe.
2103  */
gup_get_pte(pte_t * ptep)2104 static inline pte_t gup_get_pte(pte_t *ptep)
2105 {
2106 	pte_t pte;
2107 
2108 	do {
2109 		pte.pte_low = ptep->pte_low;
2110 		smp_rmb();
2111 		pte.pte_high = ptep->pte_high;
2112 		smp_rmb();
2113 	} while (unlikely(pte.pte_low != ptep->pte_low));
2114 
2115 	return pte;
2116 }
2117 #else /* CONFIG_GUP_GET_PTE_LOW_HIGH */
2118 /*
2119  * We require that the PTE can be read atomically.
2120  */
gup_get_pte(pte_t * ptep)2121 static inline pte_t gup_get_pte(pte_t *ptep)
2122 {
2123 	return ptep_get(ptep);
2124 }
2125 #endif /* CONFIG_GUP_GET_PTE_LOW_HIGH */
2126 
undo_dev_pagemap(int * nr,int nr_start,unsigned int flags,struct page ** pages)2127 static void __maybe_unused undo_dev_pagemap(int *nr, int nr_start,
2128 					    unsigned int flags,
2129 					    struct page **pages)
2130 {
2131 	while ((*nr) - nr_start) {
2132 		struct page *page = pages[--(*nr)];
2133 
2134 		ClearPageReferenced(page);
2135 		if (flags & FOLL_PIN)
2136 			unpin_user_page(page);
2137 		else
2138 			put_page(page);
2139 	}
2140 }
2141 
2142 #ifdef CONFIG_ARCH_HAS_PTE_SPECIAL
gup_pte_range(pmd_t pmd,unsigned long addr,unsigned long end,unsigned int flags,struct page ** pages,int * nr)2143 static int gup_pte_range(pmd_t pmd, unsigned long addr, unsigned long end,
2144 			 unsigned int flags, struct page **pages, int *nr)
2145 {
2146 	struct dev_pagemap *pgmap = NULL;
2147 	int nr_start = *nr, ret = 0;
2148 	pte_t *ptep, *ptem;
2149 
2150 	ptem = ptep = pte_offset_map(&pmd, addr);
2151 	do {
2152 		pte_t pte = gup_get_pte(ptep);
2153 		struct page *head, *page;
2154 
2155 		/*
2156 		 * Similar to the PMD case below, NUMA hinting must take slow
2157 		 * path using the pte_protnone check.
2158 		 */
2159 		if (pte_protnone(pte))
2160 			goto pte_unmap;
2161 
2162 		if (!pte_access_permitted(pte, flags & FOLL_WRITE))
2163 			goto pte_unmap;
2164 
2165 		if (pte_devmap(pte)) {
2166 			if (unlikely(flags & FOLL_LONGTERM))
2167 				goto pte_unmap;
2168 
2169 			pgmap = get_dev_pagemap(pte_pfn(pte), pgmap);
2170 			if (unlikely(!pgmap)) {
2171 				undo_dev_pagemap(nr, nr_start, flags, pages);
2172 				goto pte_unmap;
2173 			}
2174 		} else if (pte_special(pte))
2175 			goto pte_unmap;
2176 
2177 		VM_BUG_ON(!pfn_valid(pte_pfn(pte)));
2178 		page = pte_page(pte);
2179 
2180 		head = try_grab_compound_head(page, 1, flags);
2181 		if (!head)
2182 			goto pte_unmap;
2183 
2184 		if (unlikely(pte_val(pte) != pte_val(*ptep))) {
2185 			put_compound_head(head, 1, flags);
2186 			goto pte_unmap;
2187 		}
2188 
2189 		VM_BUG_ON_PAGE(compound_head(page) != head, page);
2190 
2191 		/*
2192 		 * We need to make the page accessible if and only if we are
2193 		 * going to access its content (the FOLL_PIN case).  Please
2194 		 * see Documentation/core-api/pin_user_pages.rst for
2195 		 * details.
2196 		 */
2197 		if (flags & FOLL_PIN) {
2198 			ret = arch_make_page_accessible(page);
2199 			if (ret) {
2200 				unpin_user_page(page);
2201 				goto pte_unmap;
2202 			}
2203 		}
2204 		SetPageReferenced(page);
2205 		pages[*nr] = page;
2206 		(*nr)++;
2207 
2208 	} while (ptep++, addr += PAGE_SIZE, addr != end);
2209 
2210 	ret = 1;
2211 
2212 pte_unmap:
2213 	if (pgmap)
2214 		put_dev_pagemap(pgmap);
2215 	pte_unmap(ptem);
2216 	return ret;
2217 }
2218 #else
2219 
2220 /*
2221  * If we can't determine whether or not a pte is special, then fail immediately
2222  * for ptes. Note, we can still pin HugeTLB and THP as these are guaranteed not
2223  * to be special.
2224  *
2225  * For a futex to be placed on a THP tail page, get_futex_key requires a
2226  * get_user_pages_fast_only implementation that can pin pages. Thus it's still
2227  * useful to have gup_huge_pmd even if we can't operate on ptes.
2228  */
gup_pte_range(pmd_t pmd,unsigned long addr,unsigned long end,unsigned int flags,struct page ** pages,int * nr)2229 static int gup_pte_range(pmd_t pmd, unsigned long addr, unsigned long end,
2230 			 unsigned int flags, struct page **pages, int *nr)
2231 {
2232 	return 0;
2233 }
2234 #endif /* CONFIG_ARCH_HAS_PTE_SPECIAL */
2235 
2236 #if defined(CONFIG_ARCH_HAS_PTE_DEVMAP) && defined(CONFIG_TRANSPARENT_HUGEPAGE)
__gup_device_huge(unsigned long pfn,unsigned long addr,unsigned long end,unsigned int flags,struct page ** pages,int * nr)2237 static int __gup_device_huge(unsigned long pfn, unsigned long addr,
2238 			     unsigned long end, unsigned int flags,
2239 			     struct page **pages, int *nr)
2240 {
2241 	int nr_start = *nr;
2242 	struct dev_pagemap *pgmap = NULL;
2243 
2244 	do {
2245 		struct page *page = pfn_to_page(pfn);
2246 
2247 		pgmap = get_dev_pagemap(pfn, pgmap);
2248 		if (unlikely(!pgmap)) {
2249 			undo_dev_pagemap(nr, nr_start, flags, pages);
2250 			return 0;
2251 		}
2252 		SetPageReferenced(page);
2253 		pages[*nr] = page;
2254 		if (unlikely(!try_grab_page(page, flags))) {
2255 			undo_dev_pagemap(nr, nr_start, flags, pages);
2256 			return 0;
2257 		}
2258 		(*nr)++;
2259 		pfn++;
2260 	} while (addr += PAGE_SIZE, addr != end);
2261 
2262 	if (pgmap)
2263 		put_dev_pagemap(pgmap);
2264 	return 1;
2265 }
2266 
__gup_device_huge_pmd(pmd_t orig,pmd_t * pmdp,unsigned long addr,unsigned long end,unsigned int flags,struct page ** pages,int * nr)2267 static int __gup_device_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr,
2268 				 unsigned long end, unsigned int flags,
2269 				 struct page **pages, int *nr)
2270 {
2271 	unsigned long fault_pfn;
2272 	int nr_start = *nr;
2273 
2274 	fault_pfn = pmd_pfn(orig) + ((addr & ~PMD_MASK) >> PAGE_SHIFT);
2275 	if (!__gup_device_huge(fault_pfn, addr, end, flags, pages, nr))
2276 		return 0;
2277 
2278 	if (unlikely(pmd_val(orig) != pmd_val(*pmdp))) {
2279 		undo_dev_pagemap(nr, nr_start, flags, pages);
2280 		return 0;
2281 	}
2282 	return 1;
2283 }
2284 
__gup_device_huge_pud(pud_t orig,pud_t * pudp,unsigned long addr,unsigned long end,unsigned int flags,struct page ** pages,int * nr)2285 static int __gup_device_huge_pud(pud_t orig, pud_t *pudp, unsigned long addr,
2286 				 unsigned long end, unsigned int flags,
2287 				 struct page **pages, int *nr)
2288 {
2289 	unsigned long fault_pfn;
2290 	int nr_start = *nr;
2291 
2292 	fault_pfn = pud_pfn(orig) + ((addr & ~PUD_MASK) >> PAGE_SHIFT);
2293 	if (!__gup_device_huge(fault_pfn, addr, end, flags, pages, nr))
2294 		return 0;
2295 
2296 	if (unlikely(pud_val(orig) != pud_val(*pudp))) {
2297 		undo_dev_pagemap(nr, nr_start, flags, pages);
2298 		return 0;
2299 	}
2300 	return 1;
2301 }
2302 #else
__gup_device_huge_pmd(pmd_t orig,pmd_t * pmdp,unsigned long addr,unsigned long end,unsigned int flags,struct page ** pages,int * nr)2303 static int __gup_device_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr,
2304 				 unsigned long end, unsigned int flags,
2305 				 struct page **pages, int *nr)
2306 {
2307 	BUILD_BUG();
2308 	return 0;
2309 }
2310 
__gup_device_huge_pud(pud_t pud,pud_t * pudp,unsigned long addr,unsigned long end,unsigned int flags,struct page ** pages,int * nr)2311 static int __gup_device_huge_pud(pud_t pud, pud_t *pudp, unsigned long addr,
2312 				 unsigned long end, unsigned int flags,
2313 				 struct page **pages, int *nr)
2314 {
2315 	BUILD_BUG();
2316 	return 0;
2317 }
2318 #endif
2319 
record_subpages(struct page * page,unsigned long addr,unsigned long end,struct page ** pages)2320 static int record_subpages(struct page *page, unsigned long addr,
2321 			   unsigned long end, struct page **pages)
2322 {
2323 	int nr;
2324 
2325 	for (nr = 0; addr != end; addr += PAGE_SIZE)
2326 		pages[nr++] = page++;
2327 
2328 	return nr;
2329 }
2330 
2331 #ifdef CONFIG_ARCH_HAS_HUGEPD
hugepte_addr_end(unsigned long addr,unsigned long end,unsigned long sz)2332 static unsigned long hugepte_addr_end(unsigned long addr, unsigned long end,
2333 				      unsigned long sz)
2334 {
2335 	unsigned long __boundary = (addr + sz) & ~(sz-1);
2336 	return (__boundary - 1 < end - 1) ? __boundary : end;
2337 }
2338 
gup_hugepte(pte_t * ptep,unsigned long sz,unsigned long addr,unsigned long end,unsigned int flags,struct page ** pages,int * nr)2339 static int gup_hugepte(pte_t *ptep, unsigned long sz, unsigned long addr,
2340 		       unsigned long end, unsigned int flags,
2341 		       struct page **pages, int *nr)
2342 {
2343 	unsigned long pte_end;
2344 	struct page *head, *page;
2345 	pte_t pte;
2346 	int refs;
2347 
2348 	pte_end = (addr + sz) & ~(sz-1);
2349 	if (pte_end < end)
2350 		end = pte_end;
2351 
2352 	pte = huge_ptep_get(ptep);
2353 
2354 	if (!pte_access_permitted(pte, flags & FOLL_WRITE))
2355 		return 0;
2356 
2357 	/* hugepages are never "special" */
2358 	VM_BUG_ON(!pfn_valid(pte_pfn(pte)));
2359 
2360 	head = pte_page(pte);
2361 	page = head + ((addr & (sz-1)) >> PAGE_SHIFT);
2362 	refs = record_subpages(page, addr, end, pages + *nr);
2363 
2364 	head = try_grab_compound_head(head, refs, flags);
2365 	if (!head)
2366 		return 0;
2367 
2368 	if (unlikely(pte_val(pte) != pte_val(*ptep))) {
2369 		put_compound_head(head, refs, flags);
2370 		return 0;
2371 	}
2372 
2373 	*nr += refs;
2374 	SetPageReferenced(head);
2375 	return 1;
2376 }
2377 
gup_huge_pd(hugepd_t hugepd,unsigned long addr,unsigned int pdshift,unsigned long end,unsigned int flags,struct page ** pages,int * nr)2378 static int gup_huge_pd(hugepd_t hugepd, unsigned long addr,
2379 		unsigned int pdshift, unsigned long end, unsigned int flags,
2380 		struct page **pages, int *nr)
2381 {
2382 	pte_t *ptep;
2383 	unsigned long sz = 1UL << hugepd_shift(hugepd);
2384 	unsigned long next;
2385 
2386 	ptep = hugepte_offset(hugepd, addr, pdshift);
2387 	do {
2388 		next = hugepte_addr_end(addr, end, sz);
2389 		if (!gup_hugepte(ptep, sz, addr, end, flags, pages, nr))
2390 			return 0;
2391 	} while (ptep++, addr = next, addr != end);
2392 
2393 	return 1;
2394 }
2395 #else
gup_huge_pd(hugepd_t hugepd,unsigned long addr,unsigned int pdshift,unsigned long end,unsigned int flags,struct page ** pages,int * nr)2396 static inline int gup_huge_pd(hugepd_t hugepd, unsigned long addr,
2397 		unsigned int pdshift, unsigned long end, unsigned int flags,
2398 		struct page **pages, int *nr)
2399 {
2400 	return 0;
2401 }
2402 #endif /* CONFIG_ARCH_HAS_HUGEPD */
2403 
gup_huge_pmd(pmd_t orig,pmd_t * pmdp,unsigned long addr,unsigned long end,unsigned int flags,struct page ** pages,int * nr)2404 static int gup_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr,
2405 			unsigned long end, unsigned int flags,
2406 			struct page **pages, int *nr)
2407 {
2408 	struct page *head, *page;
2409 	int refs;
2410 
2411 	if (!pmd_access_permitted(orig, flags & FOLL_WRITE))
2412 		return 0;
2413 
2414 	if (pmd_devmap(orig)) {
2415 		if (unlikely(flags & FOLL_LONGTERM))
2416 			return 0;
2417 		return __gup_device_huge_pmd(orig, pmdp, addr, end, flags,
2418 					     pages, nr);
2419 	}
2420 
2421 	page = pmd_page(orig) + ((addr & ~PMD_MASK) >> PAGE_SHIFT);
2422 	refs = record_subpages(page, addr, end, pages + *nr);
2423 
2424 	head = try_grab_compound_head(pmd_page(orig), refs, flags);
2425 	if (!head)
2426 		return 0;
2427 
2428 	if (unlikely(pmd_val(orig) != pmd_val(*pmdp))) {
2429 		put_compound_head(head, refs, flags);
2430 		return 0;
2431 	}
2432 
2433 	*nr += refs;
2434 	SetPageReferenced(head);
2435 	return 1;
2436 }
2437 
gup_huge_pud(pud_t orig,pud_t * pudp,unsigned long addr,unsigned long end,unsigned int flags,struct page ** pages,int * nr)2438 static int gup_huge_pud(pud_t orig, pud_t *pudp, unsigned long addr,
2439 			unsigned long end, unsigned int flags,
2440 			struct page **pages, int *nr)
2441 {
2442 	struct page *head, *page;
2443 	int refs;
2444 
2445 	if (!pud_access_permitted(orig, flags & FOLL_WRITE))
2446 		return 0;
2447 
2448 	if (pud_devmap(orig)) {
2449 		if (unlikely(flags & FOLL_LONGTERM))
2450 			return 0;
2451 		return __gup_device_huge_pud(orig, pudp, addr, end, flags,
2452 					     pages, nr);
2453 	}
2454 
2455 	page = pud_page(orig) + ((addr & ~PUD_MASK) >> PAGE_SHIFT);
2456 	refs = record_subpages(page, addr, end, pages + *nr);
2457 
2458 	head = try_grab_compound_head(pud_page(orig), refs, flags);
2459 	if (!head)
2460 		return 0;
2461 
2462 	if (unlikely(pud_val(orig) != pud_val(*pudp))) {
2463 		put_compound_head(head, refs, flags);
2464 		return 0;
2465 	}
2466 
2467 	*nr += refs;
2468 	SetPageReferenced(head);
2469 	return 1;
2470 }
2471 
gup_huge_pgd(pgd_t orig,pgd_t * pgdp,unsigned long addr,unsigned long end,unsigned int flags,struct page ** pages,int * nr)2472 static int gup_huge_pgd(pgd_t orig, pgd_t *pgdp, unsigned long addr,
2473 			unsigned long end, unsigned int flags,
2474 			struct page **pages, int *nr)
2475 {
2476 	int refs;
2477 	struct page *head, *page;
2478 
2479 	if (!pgd_access_permitted(orig, flags & FOLL_WRITE))
2480 		return 0;
2481 
2482 	BUILD_BUG_ON(pgd_devmap(orig));
2483 
2484 	page = pgd_page(orig) + ((addr & ~PGDIR_MASK) >> PAGE_SHIFT);
2485 	refs = record_subpages(page, addr, end, pages + *nr);
2486 
2487 	head = try_grab_compound_head(pgd_page(orig), refs, flags);
2488 	if (!head)
2489 		return 0;
2490 
2491 	if (unlikely(pgd_val(orig) != pgd_val(*pgdp))) {
2492 		put_compound_head(head, refs, flags);
2493 		return 0;
2494 	}
2495 
2496 	*nr += refs;
2497 	SetPageReferenced(head);
2498 	return 1;
2499 }
2500 
gup_pmd_range(pud_t * pudp,pud_t pud,unsigned long addr,unsigned long end,unsigned int flags,struct page ** pages,int * nr)2501 static int gup_pmd_range(pud_t *pudp, pud_t pud, unsigned long addr, unsigned long end,
2502 		unsigned int flags, struct page **pages, int *nr)
2503 {
2504 	unsigned long next;
2505 	pmd_t *pmdp;
2506 
2507 	pmdp = pmd_offset_lockless(pudp, pud, addr);
2508 	do {
2509 		pmd_t pmd = READ_ONCE(*pmdp);
2510 
2511 		next = pmd_addr_end(addr, end);
2512 		if (!pmd_present(pmd))
2513 			return 0;
2514 
2515 		if (unlikely(pmd_trans_huge(pmd) || pmd_huge(pmd) ||
2516 			     pmd_devmap(pmd))) {
2517 			/*
2518 			 * NUMA hinting faults need to be handled in the GUP
2519 			 * slowpath for accounting purposes and so that they
2520 			 * can be serialised against THP migration.
2521 			 */
2522 			if (pmd_protnone(pmd))
2523 				return 0;
2524 
2525 			if (!gup_huge_pmd(pmd, pmdp, addr, next, flags,
2526 				pages, nr))
2527 				return 0;
2528 
2529 		} else if (unlikely(is_hugepd(__hugepd(pmd_val(pmd))))) {
2530 			/*
2531 			 * architecture have different format for hugetlbfs
2532 			 * pmd format and THP pmd format
2533 			 */
2534 			if (!gup_huge_pd(__hugepd(pmd_val(pmd)), addr,
2535 					 PMD_SHIFT, next, flags, pages, nr))
2536 				return 0;
2537 		} else if (!gup_pte_range(pmd, addr, next, flags, pages, nr))
2538 			return 0;
2539 	} while (pmdp++, addr = next, addr != end);
2540 
2541 	return 1;
2542 }
2543 
gup_pud_range(p4d_t * p4dp,p4d_t p4d,unsigned long addr,unsigned long end,unsigned int flags,struct page ** pages,int * nr)2544 static int gup_pud_range(p4d_t *p4dp, p4d_t p4d, unsigned long addr, unsigned long end,
2545 			 unsigned int flags, struct page **pages, int *nr)
2546 {
2547 	unsigned long next;
2548 	pud_t *pudp;
2549 
2550 	pudp = pud_offset_lockless(p4dp, p4d, addr);
2551 	do {
2552 		pud_t pud = READ_ONCE(*pudp);
2553 
2554 		next = pud_addr_end(addr, end);
2555 		if (unlikely(!pud_present(pud)))
2556 			return 0;
2557 		if (unlikely(pud_huge(pud))) {
2558 			if (!gup_huge_pud(pud, pudp, addr, next, flags,
2559 					  pages, nr))
2560 				return 0;
2561 		} else if (unlikely(is_hugepd(__hugepd(pud_val(pud))))) {
2562 			if (!gup_huge_pd(__hugepd(pud_val(pud)), addr,
2563 					 PUD_SHIFT, next, flags, pages, nr))
2564 				return 0;
2565 		} else if (!gup_pmd_range(pudp, pud, addr, next, flags, pages, nr))
2566 			return 0;
2567 	} while (pudp++, addr = next, addr != end);
2568 
2569 	return 1;
2570 }
2571 
gup_p4d_range(pgd_t * pgdp,pgd_t pgd,unsigned long addr,unsigned long end,unsigned int flags,struct page ** pages,int * nr)2572 static int gup_p4d_range(pgd_t *pgdp, pgd_t pgd, unsigned long addr, unsigned long end,
2573 			 unsigned int flags, struct page **pages, int *nr)
2574 {
2575 	unsigned long next;
2576 	p4d_t *p4dp;
2577 
2578 	p4dp = p4d_offset_lockless(pgdp, pgd, addr);
2579 	do {
2580 		p4d_t p4d = READ_ONCE(*p4dp);
2581 
2582 		next = p4d_addr_end(addr, end);
2583 		if (p4d_none(p4d))
2584 			return 0;
2585 		BUILD_BUG_ON(p4d_huge(p4d));
2586 		if (unlikely(is_hugepd(__hugepd(p4d_val(p4d))))) {
2587 			if (!gup_huge_pd(__hugepd(p4d_val(p4d)), addr,
2588 					 P4D_SHIFT, next, flags, pages, nr))
2589 				return 0;
2590 		} else if (!gup_pud_range(p4dp, p4d, addr, next, flags, pages, nr))
2591 			return 0;
2592 	} while (p4dp++, addr = next, addr != end);
2593 
2594 	return 1;
2595 }
2596 
gup_pgd_range(unsigned long addr,unsigned long end,unsigned int flags,struct page ** pages,int * nr)2597 static void gup_pgd_range(unsigned long addr, unsigned long end,
2598 		unsigned int flags, struct page **pages, int *nr)
2599 {
2600 	unsigned long next;
2601 	pgd_t *pgdp;
2602 
2603 	pgdp = pgd_offset(current->mm, addr);
2604 	do {
2605 		pgd_t pgd = READ_ONCE(*pgdp);
2606 
2607 		next = pgd_addr_end(addr, end);
2608 		if (pgd_none(pgd))
2609 			return;
2610 		if (unlikely(pgd_huge(pgd))) {
2611 			if (!gup_huge_pgd(pgd, pgdp, addr, next, flags,
2612 					  pages, nr))
2613 				return;
2614 		} else if (unlikely(is_hugepd(__hugepd(pgd_val(pgd))))) {
2615 			if (!gup_huge_pd(__hugepd(pgd_val(pgd)), addr,
2616 					 PGDIR_SHIFT, next, flags, pages, nr))
2617 				return;
2618 		} else if (!gup_p4d_range(pgdp, pgd, addr, next, flags, pages, nr))
2619 			return;
2620 	} while (pgdp++, addr = next, addr != end);
2621 }
2622 #else
gup_pgd_range(unsigned long addr,unsigned long end,unsigned int flags,struct page ** pages,int * nr)2623 static inline void gup_pgd_range(unsigned long addr, unsigned long end,
2624 		unsigned int flags, struct page **pages, int *nr)
2625 {
2626 }
2627 #endif /* CONFIG_HAVE_FAST_GUP */
2628 
2629 #ifndef gup_fast_permitted
2630 /*
2631  * Check if it's allowed to use get_user_pages_fast_only() for the range, or
2632  * we need to fall back to the slow version:
2633  */
gup_fast_permitted(unsigned long start,unsigned long end)2634 static bool gup_fast_permitted(unsigned long start, unsigned long end)
2635 {
2636 	return true;
2637 }
2638 #endif
2639 
__gup_longterm_unlocked(unsigned long start,int nr_pages,unsigned int gup_flags,struct page ** pages)2640 static int __gup_longterm_unlocked(unsigned long start, int nr_pages,
2641 				   unsigned int gup_flags, struct page **pages)
2642 {
2643 	int ret;
2644 
2645 	/*
2646 	 * FIXME: FOLL_LONGTERM does not work with
2647 	 * get_user_pages_unlocked() (see comments in that function)
2648 	 */
2649 	if (gup_flags & FOLL_LONGTERM) {
2650 		mmap_read_lock(current->mm);
2651 		ret = __gup_longterm_locked(current->mm,
2652 					    start, nr_pages,
2653 					    pages, NULL, gup_flags);
2654 		mmap_read_unlock(current->mm);
2655 	} else {
2656 		ret = get_user_pages_unlocked(start, nr_pages,
2657 					      pages, gup_flags);
2658 	}
2659 
2660 	return ret;
2661 }
2662 
lockless_pages_from_mm(unsigned long start,unsigned long end,unsigned int gup_flags,struct page ** pages)2663 static unsigned long lockless_pages_from_mm(unsigned long start,
2664 					    unsigned long end,
2665 					    unsigned int gup_flags,
2666 					    struct page **pages)
2667 {
2668 	unsigned long flags;
2669 	int nr_pinned = 0;
2670 	unsigned seq;
2671 
2672 	if (!IS_ENABLED(CONFIG_HAVE_FAST_GUP) ||
2673 	    !gup_fast_permitted(start, end))
2674 		return 0;
2675 
2676 	if (gup_flags & FOLL_PIN) {
2677 		seq = raw_read_seqcount(&current->mm->write_protect_seq);
2678 		if (seq & 1)
2679 			return 0;
2680 	}
2681 
2682 	/*
2683 	 * Disable interrupts. The nested form is used, in order to allow full,
2684 	 * general purpose use of this routine.
2685 	 *
2686 	 * With interrupts disabled, we block page table pages from being freed
2687 	 * from under us. See struct mmu_table_batch comments in
2688 	 * include/asm-generic/tlb.h for more details.
2689 	 *
2690 	 * We do not adopt an rcu_read_lock() here as we also want to block IPIs
2691 	 * that come from THPs splitting.
2692 	 */
2693 	local_irq_save(flags);
2694 	gup_pgd_range(start, end, gup_flags, pages, &nr_pinned);
2695 	local_irq_restore(flags);
2696 
2697 	/*
2698 	 * When pinning pages for DMA there could be a concurrent write protect
2699 	 * from fork() via copy_page_range(), in this case always fail fast GUP.
2700 	 */
2701 	if (gup_flags & FOLL_PIN) {
2702 		if (read_seqcount_retry(&current->mm->write_protect_seq, seq)) {
2703 			unpin_user_pages(pages, nr_pinned);
2704 			return 0;
2705 		}
2706 	}
2707 	return nr_pinned;
2708 }
2709 
internal_get_user_pages_fast(unsigned long start,unsigned long nr_pages,unsigned int gup_flags,struct page ** pages)2710 static int internal_get_user_pages_fast(unsigned long start,
2711 					unsigned long nr_pages,
2712 					unsigned int gup_flags,
2713 					struct page **pages)
2714 {
2715 	unsigned long len, end;
2716 	unsigned long nr_pinned;
2717 	int ret;
2718 
2719 	if (WARN_ON_ONCE(gup_flags & ~(FOLL_WRITE | FOLL_LONGTERM |
2720 				       FOLL_FORCE | FOLL_PIN | FOLL_GET |
2721 				       FOLL_FAST_ONLY)))
2722 		return -EINVAL;
2723 
2724 	if (gup_flags & FOLL_PIN)
2725 		atomic_set(&current->mm->has_pinned, 1);
2726 
2727 	if (!(gup_flags & FOLL_FAST_ONLY))
2728 		might_lock_read(&current->mm->mmap_lock);
2729 
2730 	start = untagged_addr(start) & PAGE_MASK;
2731 	len = nr_pages << PAGE_SHIFT;
2732 	if (check_add_overflow(start, len, &end))
2733 		return 0;
2734 	if (unlikely(!access_ok((void __user *)start, len)))
2735 		return -EFAULT;
2736 
2737 	nr_pinned = lockless_pages_from_mm(start, end, gup_flags, pages);
2738 	if (nr_pinned == nr_pages || gup_flags & FOLL_FAST_ONLY)
2739 		return nr_pinned;
2740 
2741 	/* Slow path: try to get the remaining pages with get_user_pages */
2742 	start += nr_pinned << PAGE_SHIFT;
2743 	pages += nr_pinned;
2744 	ret = __gup_longterm_unlocked(start, nr_pages - nr_pinned, gup_flags,
2745 				      pages);
2746 	if (ret < 0) {
2747 		/*
2748 		 * The caller has to unpin the pages we already pinned so
2749 		 * returning -errno is not an option
2750 		 */
2751 		if (nr_pinned)
2752 			return nr_pinned;
2753 		return ret;
2754 	}
2755 	return ret + nr_pinned;
2756 }
2757 
2758 /**
2759  * get_user_pages_fast_only() - pin user pages in memory
2760  * @start:      starting user address
2761  * @nr_pages:   number of pages from start to pin
2762  * @gup_flags:  flags modifying pin behaviour
2763  * @pages:      array that receives pointers to the pages pinned.
2764  *              Should be at least nr_pages long.
2765  *
2766  * Like get_user_pages_fast() except it's IRQ-safe in that it won't fall back to
2767  * the regular GUP.
2768  * Note a difference with get_user_pages_fast: this always returns the
2769  * number of pages pinned, 0 if no pages were pinned.
2770  *
2771  * If the architecture does not support this function, simply return with no
2772  * pages pinned.
2773  *
2774  * Careful, careful! COW breaking can go either way, so a non-write
2775  * access can get ambiguous page results. If you call this function without
2776  * 'write' set, you'd better be sure that you're ok with that ambiguity.
2777  */
get_user_pages_fast_only(unsigned long start,int nr_pages,unsigned int gup_flags,struct page ** pages)2778 int get_user_pages_fast_only(unsigned long start, int nr_pages,
2779 			     unsigned int gup_flags, struct page **pages)
2780 {
2781 	int nr_pinned;
2782 	/*
2783 	 * Internally (within mm/gup.c), gup fast variants must set FOLL_GET,
2784 	 * because gup fast is always a "pin with a +1 page refcount" request.
2785 	 *
2786 	 * FOLL_FAST_ONLY is required in order to match the API description of
2787 	 * this routine: no fall back to regular ("slow") GUP.
2788 	 */
2789 	gup_flags |= FOLL_GET | FOLL_FAST_ONLY;
2790 
2791 	nr_pinned = internal_get_user_pages_fast(start, nr_pages, gup_flags,
2792 						 pages);
2793 
2794 	/*
2795 	 * As specified in the API description above, this routine is not
2796 	 * allowed to return negative values. However, the common core
2797 	 * routine internal_get_user_pages_fast() *can* return -errno.
2798 	 * Therefore, correct for that here:
2799 	 */
2800 	if (nr_pinned < 0)
2801 		nr_pinned = 0;
2802 
2803 	return nr_pinned;
2804 }
2805 EXPORT_SYMBOL_GPL(get_user_pages_fast_only);
2806 
2807 /**
2808  * get_user_pages_fast() - pin user pages in memory
2809  * @start:      starting user address
2810  * @nr_pages:   number of pages from start to pin
2811  * @gup_flags:  flags modifying pin behaviour
2812  * @pages:      array that receives pointers to the pages pinned.
2813  *              Should be at least nr_pages long.
2814  *
2815  * Attempt to pin user pages in memory without taking mm->mmap_lock.
2816  * If not successful, it will fall back to taking the lock and
2817  * calling get_user_pages().
2818  *
2819  * Returns number of pages pinned. This may be fewer than the number requested.
2820  * If nr_pages is 0 or negative, returns 0. If no pages were pinned, returns
2821  * -errno.
2822  */
get_user_pages_fast(unsigned long start,int nr_pages,unsigned int gup_flags,struct page ** pages)2823 int get_user_pages_fast(unsigned long start, int nr_pages,
2824 			unsigned int gup_flags, struct page **pages)
2825 {
2826 	if (!is_valid_gup_flags(gup_flags))
2827 		return -EINVAL;
2828 
2829 	/*
2830 	 * The caller may or may not have explicitly set FOLL_GET; either way is
2831 	 * OK. However, internally (within mm/gup.c), gup fast variants must set
2832 	 * FOLL_GET, because gup fast is always a "pin with a +1 page refcount"
2833 	 * request.
2834 	 */
2835 	gup_flags |= FOLL_GET;
2836 	return internal_get_user_pages_fast(start, nr_pages, gup_flags, pages);
2837 }
2838 EXPORT_SYMBOL_GPL(get_user_pages_fast);
2839 
2840 /**
2841  * pin_user_pages_fast() - pin user pages in memory without taking locks
2842  *
2843  * @start:      starting user address
2844  * @nr_pages:   number of pages from start to pin
2845  * @gup_flags:  flags modifying pin behaviour
2846  * @pages:      array that receives pointers to the pages pinned.
2847  *              Should be at least nr_pages long.
2848  *
2849  * Nearly the same as get_user_pages_fast(), except that FOLL_PIN is set. See
2850  * get_user_pages_fast() for documentation on the function arguments, because
2851  * the arguments here are identical.
2852  *
2853  * FOLL_PIN means that the pages must be released via unpin_user_page(). Please
2854  * see Documentation/core-api/pin_user_pages.rst for further details.
2855  */
pin_user_pages_fast(unsigned long start,int nr_pages,unsigned int gup_flags,struct page ** pages)2856 int pin_user_pages_fast(unsigned long start, int nr_pages,
2857 			unsigned int gup_flags, struct page **pages)
2858 {
2859 	/* FOLL_GET and FOLL_PIN are mutually exclusive. */
2860 	if (WARN_ON_ONCE(gup_flags & FOLL_GET))
2861 		return -EINVAL;
2862 
2863 	gup_flags |= FOLL_PIN;
2864 	return internal_get_user_pages_fast(start, nr_pages, gup_flags, pages);
2865 }
2866 EXPORT_SYMBOL_GPL(pin_user_pages_fast);
2867 
2868 /*
2869  * This is the FOLL_PIN equivalent of get_user_pages_fast_only(). Behavior
2870  * is the same, except that this one sets FOLL_PIN instead of FOLL_GET.
2871  *
2872  * The API rules are the same, too: no negative values may be returned.
2873  */
pin_user_pages_fast_only(unsigned long start,int nr_pages,unsigned int gup_flags,struct page ** pages)2874 int pin_user_pages_fast_only(unsigned long start, int nr_pages,
2875 			     unsigned int gup_flags, struct page **pages)
2876 {
2877 	int nr_pinned;
2878 
2879 	/*
2880 	 * FOLL_GET and FOLL_PIN are mutually exclusive. Note that the API
2881 	 * rules require returning 0, rather than -errno:
2882 	 */
2883 	if (WARN_ON_ONCE(gup_flags & FOLL_GET))
2884 		return 0;
2885 	/*
2886 	 * FOLL_FAST_ONLY is required in order to match the API description of
2887 	 * this routine: no fall back to regular ("slow") GUP.
2888 	 */
2889 	gup_flags |= (FOLL_PIN | FOLL_FAST_ONLY);
2890 	nr_pinned = internal_get_user_pages_fast(start, nr_pages, gup_flags,
2891 						 pages);
2892 	/*
2893 	 * This routine is not allowed to return negative values. However,
2894 	 * internal_get_user_pages_fast() *can* return -errno. Therefore,
2895 	 * correct for that here:
2896 	 */
2897 	if (nr_pinned < 0)
2898 		nr_pinned = 0;
2899 
2900 	return nr_pinned;
2901 }
2902 EXPORT_SYMBOL_GPL(pin_user_pages_fast_only);
2903 
2904 /**
2905  * pin_user_pages_remote() - pin pages of a remote process
2906  *
2907  * @mm:		mm_struct of target mm
2908  * @start:	starting user address
2909  * @nr_pages:	number of pages from start to pin
2910  * @gup_flags:	flags modifying lookup behaviour
2911  * @pages:	array that receives pointers to the pages pinned.
2912  *		Should be at least nr_pages long. Or NULL, if caller
2913  *		only intends to ensure the pages are faulted in.
2914  * @vmas:	array of pointers to vmas corresponding to each page.
2915  *		Or NULL if the caller does not require them.
2916  * @locked:	pointer to lock flag indicating whether lock is held and
2917  *		subsequently whether VM_FAULT_RETRY functionality can be
2918  *		utilised. Lock must initially be held.
2919  *
2920  * Nearly the same as get_user_pages_remote(), except that FOLL_PIN is set. See
2921  * get_user_pages_remote() for documentation on the function arguments, because
2922  * the arguments here are identical.
2923  *
2924  * FOLL_PIN means that the pages must be released via unpin_user_page(). Please
2925  * see Documentation/core-api/pin_user_pages.rst for details.
2926  */
pin_user_pages_remote(struct mm_struct * mm,unsigned long start,unsigned long nr_pages,unsigned int gup_flags,struct page ** pages,struct vm_area_struct ** vmas,int * locked)2927 long pin_user_pages_remote(struct mm_struct *mm,
2928 			   unsigned long start, unsigned long nr_pages,
2929 			   unsigned int gup_flags, struct page **pages,
2930 			   struct vm_area_struct **vmas, int *locked)
2931 {
2932 	/* FOLL_GET and FOLL_PIN are mutually exclusive. */
2933 	if (WARN_ON_ONCE(gup_flags & FOLL_GET))
2934 		return -EINVAL;
2935 
2936 	gup_flags |= FOLL_PIN;
2937 	return __get_user_pages_remote(mm, start, nr_pages, gup_flags,
2938 				       pages, vmas, locked);
2939 }
2940 EXPORT_SYMBOL(pin_user_pages_remote);
2941 
2942 /**
2943  * pin_user_pages() - pin user pages in memory for use by other devices
2944  *
2945  * @start:	starting user address
2946  * @nr_pages:	number of pages from start to pin
2947  * @gup_flags:	flags modifying lookup behaviour
2948  * @pages:	array that receives pointers to the pages pinned.
2949  *		Should be at least nr_pages long. Or NULL, if caller
2950  *		only intends to ensure the pages are faulted in.
2951  * @vmas:	array of pointers to vmas corresponding to each page.
2952  *		Or NULL if the caller does not require them.
2953  *
2954  * Nearly the same as get_user_pages(), except that FOLL_TOUCH is not set, and
2955  * FOLL_PIN is set.
2956  *
2957  * FOLL_PIN means that the pages must be released via unpin_user_page(). Please
2958  * see Documentation/core-api/pin_user_pages.rst for details.
2959  */
pin_user_pages(unsigned long start,unsigned long nr_pages,unsigned int gup_flags,struct page ** pages,struct vm_area_struct ** vmas)2960 long pin_user_pages(unsigned long start, unsigned long nr_pages,
2961 		    unsigned int gup_flags, struct page **pages,
2962 		    struct vm_area_struct **vmas)
2963 {
2964 	/* FOLL_GET and FOLL_PIN are mutually exclusive. */
2965 	if (WARN_ON_ONCE(gup_flags & FOLL_GET))
2966 		return -EINVAL;
2967 
2968 	gup_flags |= FOLL_PIN;
2969 	return __gup_longterm_locked(current->mm, start, nr_pages,
2970 				     pages, vmas, gup_flags);
2971 }
2972 EXPORT_SYMBOL(pin_user_pages);
2973 
2974 /*
2975  * pin_user_pages_unlocked() is the FOLL_PIN variant of
2976  * get_user_pages_unlocked(). Behavior is the same, except that this one sets
2977  * FOLL_PIN and rejects FOLL_GET.
2978  */
pin_user_pages_unlocked(unsigned long start,unsigned long nr_pages,struct page ** pages,unsigned int gup_flags)2979 long pin_user_pages_unlocked(unsigned long start, unsigned long nr_pages,
2980 			     struct page **pages, unsigned int gup_flags)
2981 {
2982 	/* FOLL_GET and FOLL_PIN are mutually exclusive. */
2983 	if (WARN_ON_ONCE(gup_flags & FOLL_GET))
2984 		return -EINVAL;
2985 
2986 	gup_flags |= FOLL_PIN;
2987 	return get_user_pages_unlocked(start, nr_pages, pages, gup_flags);
2988 }
2989 EXPORT_SYMBOL(pin_user_pages_unlocked);
2990 
2991 /*
2992  * pin_user_pages_locked() is the FOLL_PIN variant of get_user_pages_locked().
2993  * Behavior is the same, except that this one sets FOLL_PIN and rejects
2994  * FOLL_GET.
2995  */
pin_user_pages_locked(unsigned long start,unsigned long nr_pages,unsigned int gup_flags,struct page ** pages,int * locked)2996 long pin_user_pages_locked(unsigned long start, unsigned long nr_pages,
2997 			   unsigned int gup_flags, struct page **pages,
2998 			   int *locked)
2999 {
3000 	/*
3001 	 * FIXME: Current FOLL_LONGTERM behavior is incompatible with
3002 	 * FAULT_FLAG_ALLOW_RETRY because of the FS DAX check requirement on
3003 	 * vmas.  As there are no users of this flag in this call we simply
3004 	 * disallow this option for now.
3005 	 */
3006 	if (WARN_ON_ONCE(gup_flags & FOLL_LONGTERM))
3007 		return -EINVAL;
3008 
3009 	/* FOLL_GET and FOLL_PIN are mutually exclusive. */
3010 	if (WARN_ON_ONCE(gup_flags & FOLL_GET))
3011 		return -EINVAL;
3012 
3013 	gup_flags |= FOLL_PIN;
3014 	return __get_user_pages_locked(current->mm, start, nr_pages,
3015 				       pages, NULL, locked,
3016 				       gup_flags | FOLL_TOUCH);
3017 }
3018 EXPORT_SYMBOL(pin_user_pages_locked);
3019