Lines Matching refs:vma
69 static long madvise_behavior(struct vm_area_struct *vma, in madvise_behavior() argument
73 struct mm_struct *mm = vma->vm_mm; in madvise_behavior()
76 unsigned long new_flags = vma->vm_flags; in madvise_behavior()
92 if (vma->vm_flags & VM_IO) { in madvise_behavior()
100 if (vma->vm_file || vma->vm_flags & VM_SHARED) { in madvise_behavior()
113 if (!is_vm_hugetlb_page(vma) && new_flags & VM_SPECIAL) { in madvise_behavior()
121 error = ksm_madvise(vma, start, end, behavior, &new_flags); in madvise_behavior()
127 error = hugepage_madvise(vma, &new_flags, behavior); in madvise_behavior()
133 if (new_flags == vma->vm_flags) { in madvise_behavior()
134 *prev = vma; in madvise_behavior()
138 pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT); in madvise_behavior()
139 *prev = vma_merge(mm, *prev, start, end, new_flags, vma->anon_vma, in madvise_behavior()
140 vma->vm_file, pgoff, vma_policy(vma), in madvise_behavior()
141 vma->vm_userfaultfd_ctx, vma_get_anon_name(vma)); in madvise_behavior()
143 vma = *prev; in madvise_behavior()
147 *prev = vma; in madvise_behavior()
149 if (start != vma->vm_start) { in madvise_behavior()
154 error = __split_vma(mm, vma, start, 1); in madvise_behavior()
159 if (end != vma->vm_end) { in madvise_behavior()
164 error = __split_vma(mm, vma, end, 0); in madvise_behavior()
173 vm_write_begin(vma); in madvise_behavior()
174 WRITE_ONCE(vma->vm_flags, new_flags); in madvise_behavior()
175 vm_write_end(vma); in madvise_behavior()
193 struct vm_area_struct *vma = walk->private; in swapin_walk_pmd_entry() local
205 orig_pte = pte_offset_map_lock(vma->vm_mm, pmd, start, &ptl); in swapin_walk_pmd_entry()
216 vma, index, false); in swapin_walk_pmd_entry()
228 static void force_shm_swapin_readahead(struct vm_area_struct *vma, in force_shm_swapin_readahead() argument
232 XA_STATE(xas, &mapping->i_pages, linear_page_index(vma, start)); in force_shm_swapin_readahead()
233 pgoff_t end_index = linear_page_index(vma, end + PAGE_SIZE - 1); in force_shm_swapin_readahead()
262 static long madvise_willneed(struct vm_area_struct *vma, in madvise_willneed() argument
266 struct mm_struct *mm = vma->vm_mm; in madvise_willneed()
267 struct file *file = vma->vm_file; in madvise_willneed()
270 *prev = vma; in madvise_willneed()
273 walk_page_range(vma->vm_mm, start, end, &swapin_walk_ops, vma); in madvise_willneed()
279 force_shm_swapin_readahead(vma, start, end, in madvise_willneed()
301 offset = (loff_t)(start - vma->vm_start) in madvise_willneed()
302 + ((loff_t)vma->vm_pgoff << PAGE_SHIFT); in madvise_willneed()
319 struct vm_area_struct *vma = walk->vma; in madvise_cold_or_pageout_pte_range() local
328 trace_android_vh_madvise_cold_or_pageout_abort(vma, &abort_madvise); in madvise_cold_or_pageout_pte_range()
332 trace_android_vh_madvise_cold_or_pageout(vma, &allow_shared); in madvise_cold_or_pageout_pte_range()
339 ptl = pmd_trans_huge_lock(pmd, vma); in madvise_cold_or_pageout_pte_range()
377 pmdp_invalidate(vma, addr, pmd); in madvise_cold_or_pageout_pte_range()
407 orig_pte = pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl); in madvise_cold_or_pageout_pte_range()
419 page = vm_normal_page(vma, addr, ptent); in madvise_cold_or_pageout_pte_range()
511 struct vm_area_struct *vma, in madvise_cold_page_range() argument
519 tlb_start_vma(tlb, vma); in madvise_cold_page_range()
520 walk_page_range(vma->vm_mm, addr, end, &cold_walk_ops, &walk_private); in madvise_cold_page_range()
521 tlb_end_vma(tlb, vma); in madvise_cold_page_range()
524 static long madvise_cold(struct vm_area_struct *vma, in madvise_cold() argument
528 struct mm_struct *mm = vma->vm_mm; in madvise_cold()
531 *prev = vma; in madvise_cold()
532 if (!can_madv_lru_vma(vma)) in madvise_cold()
537 madvise_cold_page_range(&tlb, vma, start_addr, end_addr); in madvise_cold()
544 struct vm_area_struct *vma, in madvise_pageout_page_range() argument
554 tlb_start_vma(tlb, vma); in madvise_pageout_page_range()
555 walk_page_range(vma->vm_mm, addr, end, &cold_walk_ops, &walk_private); in madvise_pageout_page_range()
556 tlb_end_vma(tlb, vma); in madvise_pageout_page_range()
559 static inline bool can_do_file_pageout(struct vm_area_struct *vma) in can_do_file_pageout() argument
561 if (!vma->vm_file) in can_do_file_pageout()
569 return inode_owner_or_capable(file_inode(vma->vm_file)) || in can_do_file_pageout()
570 inode_permission(file_inode(vma->vm_file), MAY_WRITE) == 0; in can_do_file_pageout()
573 static long madvise_pageout(struct vm_area_struct *vma, in madvise_pageout() argument
577 struct mm_struct *mm = vma->vm_mm; in madvise_pageout()
581 *prev = vma; in madvise_pageout()
582 if (!can_madv_lru_vma(vma)) in madvise_pageout()
591 can_pageout_file = can_do_file_pageout(vma); in madvise_pageout()
595 madvise_pageout_page_range(&tlb, vma, start_addr, end_addr, can_pageout_file); in madvise_pageout()
607 struct vm_area_struct *vma = walk->vma; in madvise_free_pte_range() local
616 if (madvise_free_huge_pmd(tlb, vma, pmd, addr, next)) in madvise_free_pte_range()
648 page = vm_normal_page(vma, addr, ptent); in madvise_free_pte_range()
738 static int madvise_free_single_vma(struct vm_area_struct *vma, in madvise_free_single_vma() argument
741 struct mm_struct *mm = vma->vm_mm; in madvise_free_single_vma()
746 if (!vma_is_anonymous(vma)) in madvise_free_single_vma()
749 range.start = max(vma->vm_start, start_addr); in madvise_free_single_vma()
750 if (range.start >= vma->vm_end) in madvise_free_single_vma()
752 range.end = min(vma->vm_end, end_addr); in madvise_free_single_vma()
753 if (range.end <= vma->vm_start) in madvise_free_single_vma()
755 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, mm, in madvise_free_single_vma()
763 tlb_start_vma(&tlb, vma); in madvise_free_single_vma()
764 walk_page_range(vma->vm_mm, range.start, range.end, in madvise_free_single_vma()
766 tlb_end_vma(&tlb, vma); in madvise_free_single_vma()
792 static long madvise_dontneed_single_vma(struct vm_area_struct *vma, in madvise_dontneed_single_vma() argument
795 zap_page_range(vma, start, end - start); in madvise_dontneed_single_vma()
799 static long madvise_dontneed_free(struct vm_area_struct *vma, in madvise_dontneed_free() argument
804 struct mm_struct *mm = vma->vm_mm; in madvise_dontneed_free()
806 *prev = vma; in madvise_dontneed_free()
807 if (!can_madv_lru_vma(vma)) in madvise_dontneed_free()
810 if (!userfaultfd_remove(vma, start, end)) { in madvise_dontneed_free()
814 vma = find_vma(mm, start); in madvise_dontneed_free()
815 if (!vma) in madvise_dontneed_free()
817 if (start < vma->vm_start) { in madvise_dontneed_free()
829 if (!can_madv_lru_vma(vma)) in madvise_dontneed_free()
831 if (end > vma->vm_end) { in madvise_dontneed_free()
844 end = vma->vm_end; in madvise_dontneed_free()
850 return madvise_dontneed_single_vma(vma, start, end); in madvise_dontneed_free()
852 return madvise_free_single_vma(vma, start, end); in madvise_dontneed_free()
861 static long madvise_remove(struct vm_area_struct *vma, in madvise_remove() argument
868 struct mm_struct *mm = vma->vm_mm; in madvise_remove()
872 if (vma->vm_flags & VM_LOCKED) in madvise_remove()
875 f = vma->vm_file; in madvise_remove()
881 if ((vma->vm_flags & (VM_SHARED|VM_WRITE)) != (VM_SHARED|VM_WRITE)) in madvise_remove()
884 offset = (loff_t)(start - vma->vm_start) in madvise_remove()
885 + ((loff_t)vma->vm_pgoff << PAGE_SHIFT); in madvise_remove()
894 if (userfaultfd_remove(vma, start, end)) { in madvise_remove()
960 madvise_vma(struct vm_area_struct *vma, struct vm_area_struct **prev, in madvise_vma() argument
965 return madvise_remove(vma, prev, start, end); in madvise_vma()
967 return madvise_willneed(vma, prev, start, end); in madvise_vma()
969 return madvise_cold(vma, prev, start, end); in madvise_vma()
971 return madvise_pageout(vma, prev, start, end); in madvise_vma()
974 return madvise_dontneed_free(vma, prev, start, end, behavior); in madvise_vma()
976 return madvise_behavior(vma, prev, start, end, behavior); in madvise_vma()
1100 struct vm_area_struct *vma, *prev; in do_madvise() local
1146 vma = find_vma_prev(mm, start, &prev); in do_madvise()
1147 if (vma && start > vma->vm_start) in do_madvise()
1148 prev = vma; in do_madvise()
1154 if (!vma) in do_madvise()
1158 if (start < vma->vm_start) { in do_madvise()
1160 start = vma->vm_start; in do_madvise()
1166 tmp = vma->vm_end; in do_madvise()
1171 error = madvise_vma(vma, &prev, start, tmp, behavior); in do_madvise()
1181 vma = prev->vm_next; in do_madvise()
1183 vma = find_vma(mm, start); in do_madvise()