1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * linux/mm/madvise.c
4 *
5 * Copyright (C) 1999 Linus Torvalds
6 * Copyright (C) 2002 Christoph Hellwig
7 */
8
9 #include <linux/mman.h>
10 #include <linux/pagemap.h>
11 #include <linux/syscalls.h>
12 #include <linux/mempolicy.h>
13 #include <linux/page-isolation.h>
14 #include <linux/page_idle.h>
15 #include <linux/userfaultfd_k.h>
16 #include <linux/hugetlb.h>
17 #include <linux/falloc.h>
18 #include <linux/fadvise.h>
19 #include <linux/sched.h>
20 #include <linux/ksm.h>
21 #include <linux/fs.h>
22 #include <linux/file.h>
23 #include <linux/blkdev.h>
24 #include <linux/backing-dev.h>
25 #include <linux/pagewalk.h>
26 #include <linux/swap.h>
27 #include <linux/swapops.h>
28 #include <linux/shmem_fs.h>
29 #include <linux/mmu_notifier.h>
30
31 #include <asm/tlb.h>
32
33 #include "internal.h"
34
35 struct madvise_walk_private {
36 struct mmu_gather *tlb;
37 bool pageout;
38 };
39
40 /*
41 * Any behaviour which results in changes to the vma->vm_flags needs to
42 * take mmap_sem for writing. Others, which simply traverse vmas, need
43 * to only take it for reading.
44 */
madvise_need_mmap_write(int behavior)45 static int madvise_need_mmap_write(int behavior)
46 {
47 switch (behavior) {
48 case MADV_REMOVE:
49 case MADV_WILLNEED:
50 case MADV_DONTNEED:
51 case MADV_COLD:
52 case MADV_PAGEOUT:
53 case MADV_FREE:
54 return 0;
55 default:
56 /* be safe, default to 1. list exceptions explicitly */
57 return 1;
58 }
59 }
60
61 /*
62 * We can potentially split a vm area into separate
63 * areas, each area with its own behavior.
64 */
madvise_behavior(struct vm_area_struct * vma,struct vm_area_struct ** prev,unsigned long start,unsigned long end,int behavior)65 static long madvise_behavior(struct vm_area_struct *vma,
66 struct vm_area_struct **prev,
67 unsigned long start, unsigned long end, int behavior)
68 {
69 struct mm_struct *mm = vma->vm_mm;
70 int error = 0;
71 pgoff_t pgoff;
72 unsigned long new_flags = vma->vm_flags;
73
74 switch (behavior) {
75 case MADV_NORMAL:
76 new_flags = new_flags & ~VM_RAND_READ & ~VM_SEQ_READ;
77 break;
78 case MADV_SEQUENTIAL:
79 new_flags = (new_flags & ~VM_RAND_READ) | VM_SEQ_READ;
80 break;
81 case MADV_RANDOM:
82 new_flags = (new_flags & ~VM_SEQ_READ) | VM_RAND_READ;
83 break;
84 case MADV_DONTFORK:
85 new_flags |= VM_DONTCOPY;
86 break;
87 case MADV_DOFORK:
88 if (vma->vm_flags & VM_IO) {
89 error = -EINVAL;
90 goto out;
91 }
92 new_flags &= ~VM_DONTCOPY;
93 break;
94 case MADV_WIPEONFORK:
95 /* MADV_WIPEONFORK is only supported on anonymous memory. */
96 if (vma->vm_file || vma->vm_flags & VM_SHARED) {
97 error = -EINVAL;
98 goto out;
99 }
100 new_flags |= VM_WIPEONFORK;
101 break;
102 case MADV_KEEPONFORK:
103 new_flags &= ~VM_WIPEONFORK;
104 break;
105 case MADV_DONTDUMP:
106 new_flags |= VM_DONTDUMP;
107 break;
108 case MADV_DODUMP:
109 if (!is_vm_hugetlb_page(vma) && new_flags & VM_SPECIAL) {
110 error = -EINVAL;
111 goto out;
112 }
113 new_flags &= ~VM_DONTDUMP;
114 break;
115 case MADV_MERGEABLE:
116 case MADV_UNMERGEABLE:
117 error = ksm_madvise(vma, start, end, behavior, &new_flags);
118 if (error)
119 goto out_convert_errno;
120 break;
121 case MADV_HUGEPAGE:
122 case MADV_NOHUGEPAGE:
123 error = hugepage_madvise(vma, &new_flags, behavior);
124 if (error)
125 goto out_convert_errno;
126 break;
127 }
128
129 if (new_flags == vma->vm_flags) {
130 *prev = vma;
131 goto out;
132 }
133
134 pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT);
135 *prev = vma_merge(mm, *prev, start, end, new_flags, vma->anon_vma,
136 vma->vm_file, pgoff, vma_policy(vma),
137 vma->vm_userfaultfd_ctx, vma_get_anon_name(vma));
138 if (*prev) {
139 vma = *prev;
140 goto success;
141 }
142
143 *prev = vma;
144
145 if (start != vma->vm_start) {
146 if (unlikely(mm->map_count >= sysctl_max_map_count)) {
147 error = -ENOMEM;
148 goto out;
149 }
150 error = __split_vma(mm, vma, start, 1);
151 if (error)
152 goto out_convert_errno;
153 }
154
155 if (end != vma->vm_end) {
156 if (unlikely(mm->map_count >= sysctl_max_map_count)) {
157 error = -ENOMEM;
158 goto out;
159 }
160 error = __split_vma(mm, vma, end, 0);
161 if (error)
162 goto out_convert_errno;
163 }
164
165 success:
166 /*
167 * vm_flags is protected by the mmap_sem held in write mode.
168 */
169 vma->vm_flags = new_flags;
170
171 out_convert_errno:
172 /*
173 * madvise() returns EAGAIN if kernel resources, such as
174 * slab, are temporarily unavailable.
175 */
176 if (error == -ENOMEM)
177 error = -EAGAIN;
178 out:
179 return error;
180 }
181
182 #ifdef CONFIG_SWAP
swapin_walk_pmd_entry(pmd_t * pmd,unsigned long start,unsigned long end,struct mm_walk * walk)183 static int swapin_walk_pmd_entry(pmd_t *pmd, unsigned long start,
184 unsigned long end, struct mm_walk *walk)
185 {
186 pte_t *orig_pte;
187 struct vm_area_struct *vma = walk->private;
188 unsigned long index;
189
190 if (pmd_none_or_trans_huge_or_clear_bad(pmd))
191 return 0;
192
193 for (index = start; index != end; index += PAGE_SIZE) {
194 pte_t pte;
195 swp_entry_t entry;
196 struct page *page;
197 spinlock_t *ptl;
198
199 orig_pte = pte_offset_map_lock(vma->vm_mm, pmd, start, &ptl);
200 pte = *(orig_pte + ((index - start) / PAGE_SIZE));
201 pte_unmap_unlock(orig_pte, ptl);
202
203 if (pte_present(pte) || pte_none(pte))
204 continue;
205 entry = pte_to_swp_entry(pte);
206 if (unlikely(non_swap_entry(entry)))
207 continue;
208
209 page = read_swap_cache_async(entry, GFP_HIGHUSER_MOVABLE,
210 vma, index, false);
211 if (page)
212 put_page(page);
213 }
214
215 return 0;
216 }
217
218 static const struct mm_walk_ops swapin_walk_ops = {
219 .pmd_entry = swapin_walk_pmd_entry,
220 };
221
force_shm_swapin_readahead(struct vm_area_struct * vma,unsigned long start,unsigned long end,struct address_space * mapping)222 static void force_shm_swapin_readahead(struct vm_area_struct *vma,
223 unsigned long start, unsigned long end,
224 struct address_space *mapping)
225 {
226 pgoff_t index;
227 struct page *page;
228 swp_entry_t swap;
229
230 for (; start < end; start += PAGE_SIZE) {
231 index = ((start - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
232
233 page = find_get_entry(mapping, index);
234 if (!xa_is_value(page)) {
235 if (page)
236 put_page(page);
237 continue;
238 }
239 swap = radix_to_swp_entry(page);
240 page = read_swap_cache_async(swap, GFP_HIGHUSER_MOVABLE,
241 NULL, 0, false);
242 if (page)
243 put_page(page);
244 }
245
246 lru_add_drain(); /* Push any new pages onto the LRU now */
247 }
248 #endif /* CONFIG_SWAP */
249
250 /*
251 * Schedule all required I/O operations. Do not wait for completion.
252 */
madvise_willneed(struct vm_area_struct * vma,struct vm_area_struct ** prev,unsigned long start,unsigned long end)253 static long madvise_willneed(struct vm_area_struct *vma,
254 struct vm_area_struct **prev,
255 unsigned long start, unsigned long end)
256 {
257 struct file *file = vma->vm_file;
258 loff_t offset;
259
260 *prev = vma;
261 #ifdef CONFIG_SWAP
262 if (!file) {
263 walk_page_range(vma->vm_mm, start, end, &swapin_walk_ops, vma);
264 lru_add_drain(); /* Push any new pages onto the LRU now */
265 return 0;
266 }
267
268 if (shmem_mapping(file->f_mapping)) {
269 force_shm_swapin_readahead(vma, start, end,
270 file->f_mapping);
271 return 0;
272 }
273 #else
274 if (!file)
275 return -EBADF;
276 #endif
277
278 if (IS_DAX(file_inode(file))) {
279 /* no bad return value, but ignore advice */
280 return 0;
281 }
282
283 /*
284 * Filesystem's fadvise may need to take various locks. We need to
285 * explicitly grab a reference because the vma (and hence the
286 * vma's reference to the file) can go away as soon as we drop
287 * mmap_sem.
288 */
289 *prev = NULL; /* tell sys_madvise we drop mmap_sem */
290 get_file(file);
291 offset = (loff_t)(start - vma->vm_start)
292 + ((loff_t)vma->vm_pgoff << PAGE_SHIFT);
293 up_read(¤t->mm->mmap_sem);
294 vfs_fadvise(file, offset, end - start, POSIX_FADV_WILLNEED);
295 fput(file);
296 down_read(¤t->mm->mmap_sem);
297 return 0;
298 }
299
madvise_cold_or_pageout_pte_range(pmd_t * pmd,unsigned long addr,unsigned long end,struct mm_walk * walk)300 static int madvise_cold_or_pageout_pte_range(pmd_t *pmd,
301 unsigned long addr, unsigned long end,
302 struct mm_walk *walk)
303 {
304 struct madvise_walk_private *private = walk->private;
305 struct mmu_gather *tlb = private->tlb;
306 bool pageout = private->pageout;
307 struct mm_struct *mm = tlb->mm;
308 struct vm_area_struct *vma = walk->vma;
309 pte_t *orig_pte, *pte, ptent;
310 spinlock_t *ptl;
311 struct page *page = NULL;
312 LIST_HEAD(page_list);
313
314 if (fatal_signal_pending(current))
315 return -EINTR;
316
317 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
318 if (pmd_trans_huge(*pmd)) {
319 pmd_t orig_pmd;
320 unsigned long next = pmd_addr_end(addr, end);
321
322 tlb_change_page_size(tlb, HPAGE_PMD_SIZE);
323 ptl = pmd_trans_huge_lock(pmd, vma);
324 if (!ptl)
325 return 0;
326
327 orig_pmd = *pmd;
328 if (is_huge_zero_pmd(orig_pmd))
329 goto huge_unlock;
330
331 if (unlikely(!pmd_present(orig_pmd))) {
332 VM_BUG_ON(thp_migration_supported() &&
333 !is_pmd_migration_entry(orig_pmd));
334 goto huge_unlock;
335 }
336
337 page = pmd_page(orig_pmd);
338
339 /* Do not interfere with other mappings of this page */
340 if (page_mapcount(page) != 1)
341 goto huge_unlock;
342
343 if (next - addr != HPAGE_PMD_SIZE) {
344 int err;
345
346 get_page(page);
347 spin_unlock(ptl);
348 lock_page(page);
349 err = split_huge_page(page);
350 unlock_page(page);
351 put_page(page);
352 if (!err)
353 goto regular_page;
354 return 0;
355 }
356
357 if (pmd_young(orig_pmd)) {
358 pmdp_invalidate(vma, addr, pmd);
359 orig_pmd = pmd_mkold(orig_pmd);
360
361 set_pmd_at(mm, addr, pmd, orig_pmd);
362 tlb_remove_pmd_tlb_entry(tlb, pmd, addr);
363 }
364
365 ClearPageReferenced(page);
366 test_and_clear_page_young(page);
367 if (pageout) {
368 if (!isolate_lru_page(page)) {
369 if (PageUnevictable(page))
370 putback_lru_page(page);
371 else
372 list_add(&page->lru, &page_list);
373 }
374 } else
375 deactivate_page(page);
376 huge_unlock:
377 spin_unlock(ptl);
378 if (pageout)
379 reclaim_pages(&page_list);
380 return 0;
381 }
382
383 regular_page:
384 if (pmd_trans_unstable(pmd))
385 return 0;
386 #endif
387 tlb_change_page_size(tlb, PAGE_SIZE);
388 orig_pte = pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
389 flush_tlb_batched_pending(mm);
390 arch_enter_lazy_mmu_mode();
391 for (; addr < end; pte++, addr += PAGE_SIZE) {
392 ptent = *pte;
393
394 if (pte_none(ptent))
395 continue;
396
397 if (!pte_present(ptent))
398 continue;
399
400 page = vm_normal_page(vma, addr, ptent);
401 if (!page)
402 continue;
403
404 /*
405 * Creating a THP page is expensive so split it only if we
406 * are sure it's worth. Split it if we are only owner.
407 */
408 if (PageTransCompound(page)) {
409 if (page_mapcount(page) != 1)
410 break;
411 get_page(page);
412 if (!trylock_page(page)) {
413 put_page(page);
414 break;
415 }
416 pte_unmap_unlock(orig_pte, ptl);
417 if (split_huge_page(page)) {
418 unlock_page(page);
419 put_page(page);
420 pte_offset_map_lock(mm, pmd, addr, &ptl);
421 break;
422 }
423 unlock_page(page);
424 put_page(page);
425 pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
426 pte--;
427 addr -= PAGE_SIZE;
428 continue;
429 }
430
431 /*
432 * Do not interfere with other mappings of this page and
433 * non-LRU page.
434 */
435 if (!PageLRU(page) || page_mapcount(page) != 1)
436 continue;
437
438 VM_BUG_ON_PAGE(PageTransCompound(page), page);
439
440 if (pte_young(ptent)) {
441 ptent = ptep_get_and_clear_full(mm, addr, pte,
442 tlb->fullmm);
443 ptent = pte_mkold(ptent);
444 set_pte_at(mm, addr, pte, ptent);
445 tlb_remove_tlb_entry(tlb, pte, addr);
446 }
447
448 /*
449 * We are deactivating a page for accelerating reclaiming.
450 * VM couldn't reclaim the page unless we clear PG_young.
451 * As a side effect, it makes confuse idle-page tracking
452 * because they will miss recent referenced history.
453 */
454 ClearPageReferenced(page);
455 test_and_clear_page_young(page);
456 if (pageout) {
457 if (!isolate_lru_page(page)) {
458 if (PageUnevictable(page))
459 putback_lru_page(page);
460 else
461 list_add(&page->lru, &page_list);
462 }
463 } else
464 deactivate_page(page);
465 }
466
467 arch_leave_lazy_mmu_mode();
468 pte_unmap_unlock(orig_pte, ptl);
469 if (pageout)
470 reclaim_pages(&page_list);
471 cond_resched();
472
473 return 0;
474 }
475
476 static const struct mm_walk_ops cold_walk_ops = {
477 .pmd_entry = madvise_cold_or_pageout_pte_range,
478 };
479
madvise_cold_page_range(struct mmu_gather * tlb,struct vm_area_struct * vma,unsigned long addr,unsigned long end)480 static void madvise_cold_page_range(struct mmu_gather *tlb,
481 struct vm_area_struct *vma,
482 unsigned long addr, unsigned long end)
483 {
484 struct madvise_walk_private walk_private = {
485 .pageout = false,
486 .tlb = tlb,
487 };
488
489 tlb_start_vma(tlb, vma);
490 walk_page_range(vma->vm_mm, addr, end, &cold_walk_ops, &walk_private);
491 tlb_end_vma(tlb, vma);
492 }
493
madvise_cold(struct vm_area_struct * vma,struct vm_area_struct ** prev,unsigned long start_addr,unsigned long end_addr)494 static long madvise_cold(struct vm_area_struct *vma,
495 struct vm_area_struct **prev,
496 unsigned long start_addr, unsigned long end_addr)
497 {
498 struct mm_struct *mm = vma->vm_mm;
499 struct mmu_gather tlb;
500
501 *prev = vma;
502 if (!can_madv_lru_vma(vma))
503 return -EINVAL;
504
505 lru_add_drain();
506 tlb_gather_mmu(&tlb, mm, start_addr, end_addr);
507 madvise_cold_page_range(&tlb, vma, start_addr, end_addr);
508 tlb_finish_mmu(&tlb, start_addr, end_addr);
509
510 return 0;
511 }
512
madvise_pageout_page_range(struct mmu_gather * tlb,struct vm_area_struct * vma,unsigned long addr,unsigned long end)513 static void madvise_pageout_page_range(struct mmu_gather *tlb,
514 struct vm_area_struct *vma,
515 unsigned long addr, unsigned long end)
516 {
517 struct madvise_walk_private walk_private = {
518 .pageout = true,
519 .tlb = tlb,
520 };
521
522 tlb_start_vma(tlb, vma);
523 walk_page_range(vma->vm_mm, addr, end, &cold_walk_ops, &walk_private);
524 tlb_end_vma(tlb, vma);
525 }
526
can_do_pageout(struct vm_area_struct * vma)527 static inline bool can_do_pageout(struct vm_area_struct *vma)
528 {
529 if (vma_is_anonymous(vma))
530 return true;
531 if (!vma->vm_file)
532 return false;
533 /*
534 * paging out pagecache only for non-anonymous mappings that correspond
535 * to the files the calling process could (if tried) open for writing;
536 * otherwise we'd be including shared non-exclusive mappings, which
537 * opens a side channel.
538 */
539 return inode_owner_or_capable(file_inode(vma->vm_file)) ||
540 inode_permission(file_inode(vma->vm_file), MAY_WRITE) == 0;
541 }
542
madvise_pageout(struct vm_area_struct * vma,struct vm_area_struct ** prev,unsigned long start_addr,unsigned long end_addr)543 static long madvise_pageout(struct vm_area_struct *vma,
544 struct vm_area_struct **prev,
545 unsigned long start_addr, unsigned long end_addr)
546 {
547 struct mm_struct *mm = vma->vm_mm;
548 struct mmu_gather tlb;
549
550 *prev = vma;
551 if (!can_madv_lru_vma(vma))
552 return -EINVAL;
553
554 if (!can_do_pageout(vma))
555 return 0;
556
557 lru_add_drain();
558 tlb_gather_mmu(&tlb, mm, start_addr, end_addr);
559 madvise_pageout_page_range(&tlb, vma, start_addr, end_addr);
560 tlb_finish_mmu(&tlb, start_addr, end_addr);
561
562 return 0;
563 }
564
madvise_free_pte_range(pmd_t * pmd,unsigned long addr,unsigned long end,struct mm_walk * walk)565 static int madvise_free_pte_range(pmd_t *pmd, unsigned long addr,
566 unsigned long end, struct mm_walk *walk)
567
568 {
569 struct mmu_gather *tlb = walk->private;
570 struct mm_struct *mm = tlb->mm;
571 struct vm_area_struct *vma = walk->vma;
572 spinlock_t *ptl;
573 pte_t *orig_pte, *pte, ptent;
574 struct page *page;
575 int nr_swap = 0;
576 unsigned long next;
577
578 next = pmd_addr_end(addr, end);
579 if (pmd_trans_huge(*pmd))
580 if (madvise_free_huge_pmd(tlb, vma, pmd, addr, next))
581 goto next;
582
583 if (pmd_trans_unstable(pmd))
584 return 0;
585
586 tlb_change_page_size(tlb, PAGE_SIZE);
587 orig_pte = pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
588 flush_tlb_batched_pending(mm);
589 arch_enter_lazy_mmu_mode();
590 for (; addr != end; pte++, addr += PAGE_SIZE) {
591 ptent = *pte;
592
593 if (pte_none(ptent))
594 continue;
595 /*
596 * If the pte has swp_entry, just clear page table to
597 * prevent swap-in which is more expensive rather than
598 * (page allocation + zeroing).
599 */
600 if (!pte_present(ptent)) {
601 swp_entry_t entry;
602
603 entry = pte_to_swp_entry(ptent);
604 if (non_swap_entry(entry))
605 continue;
606 nr_swap--;
607 free_swap_and_cache(entry);
608 pte_clear_not_present_full(mm, addr, pte, tlb->fullmm);
609 continue;
610 }
611
612 page = vm_normal_page(vma, addr, ptent);
613 if (!page)
614 continue;
615
616 /*
617 * If pmd isn't transhuge but the page is THP and
618 * is owned by only this process, split it and
619 * deactivate all pages.
620 */
621 if (PageTransCompound(page)) {
622 if (page_mapcount(page) != 1)
623 goto out;
624 get_page(page);
625 if (!trylock_page(page)) {
626 put_page(page);
627 goto out;
628 }
629 pte_unmap_unlock(orig_pte, ptl);
630 if (split_huge_page(page)) {
631 unlock_page(page);
632 put_page(page);
633 pte_offset_map_lock(mm, pmd, addr, &ptl);
634 goto out;
635 }
636 unlock_page(page);
637 put_page(page);
638 pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
639 pte--;
640 addr -= PAGE_SIZE;
641 continue;
642 }
643
644 VM_BUG_ON_PAGE(PageTransCompound(page), page);
645
646 if (PageSwapCache(page) || PageDirty(page)) {
647 if (!trylock_page(page))
648 continue;
649 /*
650 * If page is shared with others, we couldn't clear
651 * PG_dirty of the page.
652 */
653 if (page_mapcount(page) != 1) {
654 unlock_page(page);
655 continue;
656 }
657
658 if (PageSwapCache(page) && !try_to_free_swap(page)) {
659 unlock_page(page);
660 continue;
661 }
662
663 ClearPageDirty(page);
664 unlock_page(page);
665 }
666
667 if (pte_young(ptent) || pte_dirty(ptent)) {
668 /*
669 * Some of architecture(ex, PPC) don't update TLB
670 * with set_pte_at and tlb_remove_tlb_entry so for
671 * the portability, remap the pte with old|clean
672 * after pte clearing.
673 */
674 ptent = ptep_get_and_clear_full(mm, addr, pte,
675 tlb->fullmm);
676
677 ptent = pte_mkold(ptent);
678 ptent = pte_mkclean(ptent);
679 set_pte_at(mm, addr, pte, ptent);
680 tlb_remove_tlb_entry(tlb, pte, addr);
681 }
682 mark_page_lazyfree(page);
683 }
684 out:
685 if (nr_swap) {
686 if (current->mm == mm)
687 sync_mm_rss(mm);
688
689 add_mm_counter(mm, MM_SWAPENTS, nr_swap);
690 }
691 arch_leave_lazy_mmu_mode();
692 pte_unmap_unlock(orig_pte, ptl);
693 cond_resched();
694 next:
695 return 0;
696 }
697
698 static const struct mm_walk_ops madvise_free_walk_ops = {
699 .pmd_entry = madvise_free_pte_range,
700 };
701
madvise_free_single_vma(struct vm_area_struct * vma,unsigned long start_addr,unsigned long end_addr)702 static int madvise_free_single_vma(struct vm_area_struct *vma,
703 unsigned long start_addr, unsigned long end_addr)
704 {
705 struct mm_struct *mm = vma->vm_mm;
706 struct mmu_notifier_range range;
707 struct mmu_gather tlb;
708
709 /* MADV_FREE works for only anon vma at the moment */
710 if (!vma_is_anonymous(vma))
711 return -EINVAL;
712
713 range.start = max(vma->vm_start, start_addr);
714 if (range.start >= vma->vm_end)
715 return -EINVAL;
716 range.end = min(vma->vm_end, end_addr);
717 if (range.end <= vma->vm_start)
718 return -EINVAL;
719 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, mm,
720 range.start, range.end);
721
722 lru_add_drain();
723 tlb_gather_mmu(&tlb, mm, range.start, range.end);
724 update_hiwater_rss(mm);
725
726 mmu_notifier_invalidate_range_start(&range);
727 tlb_start_vma(&tlb, vma);
728 walk_page_range(vma->vm_mm, range.start, range.end,
729 &madvise_free_walk_ops, &tlb);
730 tlb_end_vma(&tlb, vma);
731 mmu_notifier_invalidate_range_end(&range);
732 tlb_finish_mmu(&tlb, range.start, range.end);
733
734 return 0;
735 }
736
737 /*
738 * Application no longer needs these pages. If the pages are dirty,
739 * it's OK to just throw them away. The app will be more careful about
740 * data it wants to keep. Be sure to free swap resources too. The
741 * zap_page_range call sets things up for shrink_active_list to actually free
742 * these pages later if no one else has touched them in the meantime,
743 * although we could add these pages to a global reuse list for
744 * shrink_active_list to pick up before reclaiming other pages.
745 *
746 * NB: This interface discards data rather than pushes it out to swap,
747 * as some implementations do. This has performance implications for
748 * applications like large transactional databases which want to discard
749 * pages in anonymous maps after committing to backing store the data
750 * that was kept in them. There is no reason to write this data out to
751 * the swap area if the application is discarding it.
752 *
753 * An interface that causes the system to free clean pages and flush
754 * dirty pages is already available as msync(MS_INVALIDATE).
755 */
madvise_dontneed_single_vma(struct vm_area_struct * vma,unsigned long start,unsigned long end)756 static long madvise_dontneed_single_vma(struct vm_area_struct *vma,
757 unsigned long start, unsigned long end)
758 {
759 zap_page_range(vma, start, end - start);
760 return 0;
761 }
762
madvise_dontneed_free(struct vm_area_struct * vma,struct vm_area_struct ** prev,unsigned long start,unsigned long end,int behavior)763 static long madvise_dontneed_free(struct vm_area_struct *vma,
764 struct vm_area_struct **prev,
765 unsigned long start, unsigned long end,
766 int behavior)
767 {
768 *prev = vma;
769 if (!can_madv_lru_vma(vma))
770 return -EINVAL;
771
772 if (!userfaultfd_remove(vma, start, end)) {
773 *prev = NULL; /* mmap_sem has been dropped, prev is stale */
774
775 down_read(¤t->mm->mmap_sem);
776 vma = find_vma(current->mm, start);
777 if (!vma)
778 return -ENOMEM;
779 if (start < vma->vm_start) {
780 /*
781 * This "vma" under revalidation is the one
782 * with the lowest vma->vm_start where start
783 * is also < vma->vm_end. If start <
784 * vma->vm_start it means an hole materialized
785 * in the user address space within the
786 * virtual range passed to MADV_DONTNEED
787 * or MADV_FREE.
788 */
789 return -ENOMEM;
790 }
791 if (!can_madv_lru_vma(vma))
792 return -EINVAL;
793 if (end > vma->vm_end) {
794 /*
795 * Don't fail if end > vma->vm_end. If the old
796 * vma was splitted while the mmap_sem was
797 * released the effect of the concurrent
798 * operation may not cause madvise() to
799 * have an undefined result. There may be an
800 * adjacent next vma that we'll walk
801 * next. userfaultfd_remove() will generate an
802 * UFFD_EVENT_REMOVE repetition on the
803 * end-vma->vm_end range, but the manager can
804 * handle a repetition fine.
805 */
806 end = vma->vm_end;
807 }
808 VM_WARN_ON(start >= end);
809 }
810
811 if (behavior == MADV_DONTNEED)
812 return madvise_dontneed_single_vma(vma, start, end);
813 else if (behavior == MADV_FREE)
814 return madvise_free_single_vma(vma, start, end);
815 else
816 return -EINVAL;
817 }
818
819 /*
820 * Application wants to free up the pages and associated backing store.
821 * This is effectively punching a hole into the middle of a file.
822 */
madvise_remove(struct vm_area_struct * vma,struct vm_area_struct ** prev,unsigned long start,unsigned long end)823 static long madvise_remove(struct vm_area_struct *vma,
824 struct vm_area_struct **prev,
825 unsigned long start, unsigned long end)
826 {
827 loff_t offset;
828 int error;
829 struct file *f;
830
831 *prev = NULL; /* tell sys_madvise we drop mmap_sem */
832
833 if (vma->vm_flags & VM_LOCKED)
834 return -EINVAL;
835
836 f = vma->vm_file;
837
838 if (!f || !f->f_mapping || !f->f_mapping->host) {
839 return -EINVAL;
840 }
841
842 if ((vma->vm_flags & (VM_SHARED|VM_WRITE)) != (VM_SHARED|VM_WRITE))
843 return -EACCES;
844
845 offset = (loff_t)(start - vma->vm_start)
846 + ((loff_t)vma->vm_pgoff << PAGE_SHIFT);
847
848 /*
849 * Filesystem's fallocate may need to take i_mutex. We need to
850 * explicitly grab a reference because the vma (and hence the
851 * vma's reference to the file) can go away as soon as we drop
852 * mmap_sem.
853 */
854 get_file(f);
855 if (userfaultfd_remove(vma, start, end)) {
856 /* mmap_sem was not released by userfaultfd_remove() */
857 up_read(¤t->mm->mmap_sem);
858 }
859 error = vfs_fallocate(f,
860 FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE,
861 offset, end - start);
862 fput(f);
863 down_read(¤t->mm->mmap_sem);
864 return error;
865 }
866
867 #ifdef CONFIG_MEMORY_FAILURE
868 /*
869 * Error injection support for memory error handling.
870 */
madvise_inject_error(int behavior,unsigned long start,unsigned long end)871 static int madvise_inject_error(int behavior,
872 unsigned long start, unsigned long end)
873 {
874 struct page *page;
875 struct zone *zone;
876 unsigned int order;
877
878 if (!capable(CAP_SYS_ADMIN))
879 return -EPERM;
880
881
882 for (; start < end; start += PAGE_SIZE << order) {
883 unsigned long pfn;
884 int ret;
885
886 ret = get_user_pages_fast(start, 1, 0, &page);
887 if (ret != 1)
888 return ret;
889 pfn = page_to_pfn(page);
890
891 /*
892 * When soft offlining hugepages, after migrating the page
893 * we dissolve it, therefore in the second loop "page" will
894 * no longer be a compound page, and order will be 0.
895 */
896 order = compound_order(compound_head(page));
897
898 if (PageHWPoison(page)) {
899 put_page(page);
900 continue;
901 }
902
903 if (behavior == MADV_SOFT_OFFLINE) {
904 pr_info("Soft offlining pfn %#lx at process virtual address %#lx\n",
905 pfn, start);
906
907 ret = soft_offline_page(page, MF_COUNT_INCREASED);
908 if (ret)
909 return ret;
910 continue;
911 }
912
913 pr_info("Injecting memory failure for pfn %#lx at process virtual address %#lx\n",
914 pfn, start);
915
916 /*
917 * Drop the page reference taken by get_user_pages_fast(). In
918 * the absence of MF_COUNT_INCREASED the memory_failure()
919 * routine is responsible for pinning the page to prevent it
920 * from being released back to the page allocator.
921 */
922 put_page(page);
923 ret = memory_failure(pfn, 0);
924 if (ret)
925 return ret;
926 }
927
928 /* Ensure that all poisoned pages are removed from per-cpu lists */
929 for_each_populated_zone(zone)
930 drain_all_pages(zone);
931
932 return 0;
933 }
934 #endif
935
936 static long
madvise_vma(struct vm_area_struct * vma,struct vm_area_struct ** prev,unsigned long start,unsigned long end,int behavior)937 madvise_vma(struct vm_area_struct *vma, struct vm_area_struct **prev,
938 unsigned long start, unsigned long end, int behavior)
939 {
940 switch (behavior) {
941 case MADV_REMOVE:
942 return madvise_remove(vma, prev, start, end);
943 case MADV_WILLNEED:
944 return madvise_willneed(vma, prev, start, end);
945 case MADV_COLD:
946 return madvise_cold(vma, prev, start, end);
947 case MADV_PAGEOUT:
948 return madvise_pageout(vma, prev, start, end);
949 case MADV_FREE:
950 case MADV_DONTNEED:
951 return madvise_dontneed_free(vma, prev, start, end, behavior);
952 default:
953 return madvise_behavior(vma, prev, start, end, behavior);
954 }
955 }
956
957 static bool
madvise_behavior_valid(int behavior)958 madvise_behavior_valid(int behavior)
959 {
960 switch (behavior) {
961 case MADV_DOFORK:
962 case MADV_DONTFORK:
963 case MADV_NORMAL:
964 case MADV_SEQUENTIAL:
965 case MADV_RANDOM:
966 case MADV_REMOVE:
967 case MADV_WILLNEED:
968 case MADV_DONTNEED:
969 case MADV_FREE:
970 case MADV_COLD:
971 case MADV_PAGEOUT:
972 #ifdef CONFIG_KSM
973 case MADV_MERGEABLE:
974 case MADV_UNMERGEABLE:
975 #endif
976 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
977 case MADV_HUGEPAGE:
978 case MADV_NOHUGEPAGE:
979 #endif
980 case MADV_DONTDUMP:
981 case MADV_DODUMP:
982 case MADV_WIPEONFORK:
983 case MADV_KEEPONFORK:
984 #ifdef CONFIG_MEMORY_FAILURE
985 case MADV_SOFT_OFFLINE:
986 case MADV_HWPOISON:
987 #endif
988 return true;
989
990 default:
991 return false;
992 }
993 }
994
995 /*
996 * The madvise(2) system call.
997 *
998 * Applications can use madvise() to advise the kernel how it should
999 * handle paging I/O in this VM area. The idea is to help the kernel
1000 * use appropriate read-ahead and caching techniques. The information
1001 * provided is advisory only, and can be safely disregarded by the
1002 * kernel without affecting the correct operation of the application.
1003 *
1004 * behavior values:
1005 * MADV_NORMAL - the default behavior is to read clusters. This
1006 * results in some read-ahead and read-behind.
1007 * MADV_RANDOM - the system should read the minimum amount of data
1008 * on any access, since it is unlikely that the appli-
1009 * cation will need more than what it asks for.
1010 * MADV_SEQUENTIAL - pages in the given range will probably be accessed
1011 * once, so they can be aggressively read ahead, and
1012 * can be freed soon after they are accessed.
1013 * MADV_WILLNEED - the application is notifying the system to read
1014 * some pages ahead.
1015 * MADV_DONTNEED - the application is finished with the given range,
1016 * so the kernel can free resources associated with it.
1017 * MADV_FREE - the application marks pages in the given range as lazy free,
1018 * where actual purges are postponed until memory pressure happens.
1019 * MADV_REMOVE - the application wants to free up the given range of
1020 * pages and associated backing store.
1021 * MADV_DONTFORK - omit this area from child's address space when forking:
1022 * typically, to avoid COWing pages pinned by get_user_pages().
1023 * MADV_DOFORK - cancel MADV_DONTFORK: no longer omit this area when forking.
1024 * MADV_WIPEONFORK - present the child process with zero-filled memory in this
1025 * range after a fork.
1026 * MADV_KEEPONFORK - undo the effect of MADV_WIPEONFORK
1027 * MADV_HWPOISON - trigger memory error handler as if the given memory range
1028 * were corrupted by unrecoverable hardware memory failure.
1029 * MADV_SOFT_OFFLINE - try to soft-offline the given range of memory.
1030 * MADV_MERGEABLE - the application recommends that KSM try to merge pages in
1031 * this area with pages of identical content from other such areas.
1032 * MADV_UNMERGEABLE- cancel MADV_MERGEABLE: no longer merge pages with others.
1033 * MADV_HUGEPAGE - the application wants to back the given range by transparent
1034 * huge pages in the future. Existing pages might be coalesced and
1035 * new pages might be allocated as THP.
1036 * MADV_NOHUGEPAGE - mark the given range as not worth being backed by
1037 * transparent huge pages so the existing pages will not be
1038 * coalesced into THP and new pages will not be allocated as THP.
1039 * MADV_DONTDUMP - the application wants to prevent pages in the given range
1040 * from being included in its core dump.
1041 * MADV_DODUMP - cancel MADV_DONTDUMP: no longer exclude from core dump.
1042 *
1043 * return values:
1044 * zero - success
1045 * -EINVAL - start + len < 0, start is not page-aligned,
1046 * "behavior" is not a valid value, or application
1047 * is attempting to release locked or shared pages,
1048 * or the specified address range includes file, Huge TLB,
1049 * MAP_SHARED or VMPFNMAP range.
1050 * -ENOMEM - addresses in the specified range are not currently
1051 * mapped, or are outside the AS of the process.
1052 * -EIO - an I/O error occurred while paging in data.
1053 * -EBADF - map exists, but area maps something that isn't a file.
1054 * -EAGAIN - a kernel resource was temporarily unavailable.
1055 */
SYSCALL_DEFINE3(madvise,unsigned long,start,size_t,len_in,int,behavior)1056 SYSCALL_DEFINE3(madvise, unsigned long, start, size_t, len_in, int, behavior)
1057 {
1058 unsigned long end, tmp;
1059 struct vm_area_struct *vma, *prev;
1060 int unmapped_error = 0;
1061 int error = -EINVAL;
1062 int write;
1063 size_t len;
1064 struct blk_plug plug;
1065
1066 start = untagged_addr(start);
1067
1068 if (!madvise_behavior_valid(behavior))
1069 return error;
1070
1071 if (start & ~PAGE_MASK)
1072 return error;
1073 len = (len_in + ~PAGE_MASK) & PAGE_MASK;
1074
1075 /* Check to see whether len was rounded up from small -ve to zero */
1076 if (len_in && !len)
1077 return error;
1078
1079 end = start + len;
1080 if (end < start)
1081 return error;
1082
1083 error = 0;
1084 if (end == start)
1085 return error;
1086
1087 #ifdef CONFIG_MEMORY_FAILURE
1088 if (behavior == MADV_HWPOISON || behavior == MADV_SOFT_OFFLINE)
1089 return madvise_inject_error(behavior, start, start + len_in);
1090 #endif
1091
1092 write = madvise_need_mmap_write(behavior);
1093 if (write) {
1094 if (down_write_killable(¤t->mm->mmap_sem))
1095 return -EINTR;
1096 } else {
1097 down_read(¤t->mm->mmap_sem);
1098 }
1099
1100 /*
1101 * If the interval [start,end) covers some unmapped address
1102 * ranges, just ignore them, but return -ENOMEM at the end.
1103 * - different from the way of handling in mlock etc.
1104 */
1105 vma = find_vma_prev(current->mm, start, &prev);
1106 if (vma && start > vma->vm_start)
1107 prev = vma;
1108
1109 blk_start_plug(&plug);
1110 for (;;) {
1111 /* Still start < end. */
1112 error = -ENOMEM;
1113 if (!vma)
1114 goto out;
1115
1116 /* Here start < (end|vma->vm_end). */
1117 if (start < vma->vm_start) {
1118 unmapped_error = -ENOMEM;
1119 start = vma->vm_start;
1120 if (start >= end)
1121 goto out;
1122 }
1123
1124 /* Here vma->vm_start <= start < (end|vma->vm_end) */
1125 tmp = vma->vm_end;
1126 if (end < tmp)
1127 tmp = end;
1128
1129 /* Here vma->vm_start <= start < tmp <= (end|vma->vm_end). */
1130 error = madvise_vma(vma, &prev, start, tmp, behavior);
1131 if (error)
1132 goto out;
1133 start = tmp;
1134 if (prev && start < prev->vm_end)
1135 start = prev->vm_end;
1136 error = unmapped_error;
1137 if (start >= end)
1138 goto out;
1139 if (prev)
1140 vma = prev->vm_next;
1141 else /* madvise_remove dropped mmap_sem */
1142 vma = find_vma(current->mm, start);
1143 }
1144 out:
1145 blk_finish_plug(&plug);
1146 if (write)
1147 up_write(¤t->mm->mmap_sem);
1148 else
1149 up_read(¤t->mm->mmap_sem);
1150
1151 return error;
1152 }
1153