Lines Matching refs:vma
92 bool userfaultfd_wp_unpopulated(struct vm_area_struct *vma) in userfaultfd_wp_unpopulated() argument
94 struct userfaultfd_ctx *ctx = vma->vm_userfaultfd_ctx.ctx; in userfaultfd_wp_unpopulated()
102 static void userfaultfd_set_vm_flags(struct vm_area_struct *vma, in userfaultfd_set_vm_flags() argument
105 const bool uffd_wp_changed = (vma->vm_flags ^ flags) & VM_UFFD_WP; in userfaultfd_set_vm_flags()
107 vm_flags_reset(vma, flags); in userfaultfd_set_vm_flags()
113 if ((vma->vm_flags & VM_SHARED) && uffd_wp_changed) in userfaultfd_set_vm_flags()
114 vma_set_page_prot(vma); in userfaultfd_set_vm_flags()
244 struct vm_area_struct *vma = vmf->vma; in userfaultfd_huge_must_wait() local
250 ptep = hugetlb_walk(vma, vmf->address, vma_mmu_pagesize(vma)); in userfaultfd_huge_must_wait()
375 struct vm_area_struct *vma = vmf->vma; in handle_userfault() local
376 struct mm_struct *mm = vma->vm_mm; in handle_userfault()
399 ctx = vma->vm_userfaultfd_ctx.ctx; in handle_userfault()
495 if (is_vm_hugetlb_page(vma)) in handle_userfault()
496 hugetlb_vma_lock_read(vma); in handle_userfault()
512 if (!is_vm_hugetlb_page(vma)) in handle_userfault()
516 if (is_vm_hugetlb_page(vma)) in handle_userfault()
517 hugetlb_vma_unlock_read(vma); in handle_userfault()
613 struct vm_area_struct *vma; in userfaultfd_event_wait_completion() local
619 for_each_vma(vmi, vma) { in userfaultfd_event_wait_completion()
620 if (vma->vm_userfaultfd_ctx.ctx == release_new_ctx) { in userfaultfd_event_wait_completion()
621 vma_start_write(vma); in userfaultfd_event_wait_completion()
622 vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX; in userfaultfd_event_wait_completion()
623 userfaultfd_set_vm_flags(vma, in userfaultfd_event_wait_completion()
624 vma->vm_flags & ~__VM_UFFD_FLAGS); in userfaultfd_event_wait_completion()
650 int dup_userfaultfd(struct vm_area_struct *vma, struct list_head *fcs) in dup_userfaultfd() argument
655 octx = vma->vm_userfaultfd_ctx.ctx; in dup_userfaultfd()
657 vma_start_write(vma); in dup_userfaultfd()
658 vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX; in dup_userfaultfd()
659 userfaultfd_set_vm_flags(vma, vma->vm_flags & ~__VM_UFFD_FLAGS); in dup_userfaultfd()
686 ctx->mm = vma->vm_mm; in dup_userfaultfd()
698 vma->vm_userfaultfd_ctx.ctx = ctx; in dup_userfaultfd()
726 void mremap_userfaultfd_prep(struct vm_area_struct *vma, in mremap_userfaultfd_prep() argument
731 ctx = vma->vm_userfaultfd_ctx.ctx; in mremap_userfaultfd_prep()
744 vma_start_write(vma); in mremap_userfaultfd_prep()
745 vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX; in mremap_userfaultfd_prep()
746 userfaultfd_set_vm_flags(vma, vma->vm_flags & ~__VM_UFFD_FLAGS); in mremap_userfaultfd_prep()
775 bool userfaultfd_remove(struct vm_area_struct *vma, in userfaultfd_remove() argument
778 struct mm_struct *mm = vma->vm_mm; in userfaultfd_remove()
782 ctx = vma->vm_userfaultfd_ctx.ctx; in userfaultfd_remove()
816 int userfaultfd_unmap_prep(struct vm_area_struct *vma, unsigned long start, in userfaultfd_unmap_prep() argument
820 struct userfaultfd_ctx *ctx = vma->vm_userfaultfd_ctx.ctx; in userfaultfd_unmap_prep()
865 struct vm_area_struct *vma, *prev; in userfaultfd_release() local
886 for_each_vma(vmi, vma) { in userfaultfd_release()
888 BUG_ON(!!vma->vm_userfaultfd_ctx.ctx ^ in userfaultfd_release()
889 !!(vma->vm_flags & __VM_UFFD_FLAGS)); in userfaultfd_release()
890 if (vma->vm_userfaultfd_ctx.ctx != ctx) { in userfaultfd_release()
891 prev = vma; in userfaultfd_release()
895 if (userfaultfd_wp(vma)) in userfaultfd_release()
896 uffd_wp_range(vma, vma->vm_start, in userfaultfd_release()
897 vma->vm_end - vma->vm_start, false); in userfaultfd_release()
898 new_flags = vma->vm_flags & ~__VM_UFFD_FLAGS; in userfaultfd_release()
899 prev = vma_merge(&vmi, mm, prev, vma->vm_start, vma->vm_end, in userfaultfd_release()
900 new_flags, vma->anon_vma, in userfaultfd_release()
901 vma->vm_file, vma->vm_pgoff, in userfaultfd_release()
902 vma_policy(vma), in userfaultfd_release()
903 NULL_VM_UFFD_CTX, anon_vma_name(vma)); in userfaultfd_release()
905 vma = prev; in userfaultfd_release()
907 prev = vma; in userfaultfd_release()
910 vma_start_write(vma); in userfaultfd_release()
911 userfaultfd_set_vm_flags(vma, new_flags); in userfaultfd_release()
912 vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX; in userfaultfd_release()
1293 struct vm_area_struct *vma, *prev, *cur; in userfaultfd_register() local
1347 vma = vma_find(&vmi, end); in userfaultfd_register()
1348 if (!vma) in userfaultfd_register()
1355 if (is_vm_hugetlb_page(vma)) { in userfaultfd_register()
1356 unsigned long vma_hpagesize = vma_kernel_pagesize(vma); in userfaultfd_register()
1367 cur = vma; in userfaultfd_register()
1430 if (vma->vm_start < start) in userfaultfd_register()
1431 prev = vma; in userfaultfd_register()
1434 for_each_vma_range(vmi, vma, end) { in userfaultfd_register()
1437 BUG_ON(!vma_can_userfault(vma, vm_flags)); in userfaultfd_register()
1438 BUG_ON(vma->vm_userfaultfd_ctx.ctx && in userfaultfd_register()
1439 vma->vm_userfaultfd_ctx.ctx != ctx); in userfaultfd_register()
1440 WARN_ON(!(vma->vm_flags & VM_MAYWRITE)); in userfaultfd_register()
1446 if (vma->vm_userfaultfd_ctx.ctx == ctx && in userfaultfd_register()
1447 (vma->vm_flags & vm_flags) == vm_flags) in userfaultfd_register()
1450 if (vma->vm_start > start) in userfaultfd_register()
1451 start = vma->vm_start; in userfaultfd_register()
1452 vma_end = min(end, vma->vm_end); in userfaultfd_register()
1454 new_flags = (vma->vm_flags & ~__VM_UFFD_FLAGS) | vm_flags; in userfaultfd_register()
1455 pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT); in userfaultfd_register()
1457 vma->anon_vma, vma->vm_file, pgoff, in userfaultfd_register()
1458 vma_policy(vma), in userfaultfd_register()
1460 anon_vma_name(vma)); in userfaultfd_register()
1463 vma = prev; in userfaultfd_register()
1466 if (vma->vm_start < start) { in userfaultfd_register()
1467 ret = split_vma(&vmi, vma, start, 1); in userfaultfd_register()
1471 if (vma->vm_end > end) { in userfaultfd_register()
1472 ret = split_vma(&vmi, vma, end, 0); in userfaultfd_register()
1482 vma_start_write(vma); in userfaultfd_register()
1483 userfaultfd_set_vm_flags(vma, new_flags); in userfaultfd_register()
1484 vma->vm_userfaultfd_ctx.ctx = ctx; in userfaultfd_register()
1486 if (is_vm_hugetlb_page(vma) && uffd_disable_huge_pmd_share(vma)) in userfaultfd_register()
1487 hugetlb_unshare_all_pmds(vma); in userfaultfd_register()
1490 prev = vma; in userfaultfd_register()
1491 start = vma->vm_end; in userfaultfd_register()
1530 struct vm_area_struct *vma, *prev, *cur; in userfaultfd_unregister() local
1559 vma = vma_find(&vmi, end); in userfaultfd_unregister()
1560 if (!vma) in userfaultfd_unregister()
1567 if (is_vm_hugetlb_page(vma)) { in userfaultfd_unregister()
1568 unsigned long vma_hpagesize = vma_kernel_pagesize(vma); in userfaultfd_unregister()
1578 cur = vma; in userfaultfd_unregister()
1601 if (vma->vm_start < start) in userfaultfd_unregister()
1602 prev = vma; in userfaultfd_unregister()
1605 for_each_vma_range(vmi, vma, end) { in userfaultfd_unregister()
1608 BUG_ON(!vma_can_userfault(vma, vma->vm_flags)); in userfaultfd_unregister()
1614 if (!vma->vm_userfaultfd_ctx.ctx) in userfaultfd_unregister()
1617 WARN_ON(!(vma->vm_flags & VM_MAYWRITE)); in userfaultfd_unregister()
1619 if (vma->vm_start > start) in userfaultfd_unregister()
1620 start = vma->vm_start; in userfaultfd_unregister()
1621 vma_end = min(end, vma->vm_end); in userfaultfd_unregister()
1623 if (userfaultfd_missing(vma)) { in userfaultfd_unregister()
1633 wake_userfault(vma->vm_userfaultfd_ctx.ctx, &range); in userfaultfd_unregister()
1637 if (userfaultfd_wp(vma)) in userfaultfd_unregister()
1638 uffd_wp_range(vma, start, vma_end - start, false); in userfaultfd_unregister()
1640 new_flags = vma->vm_flags & ~__VM_UFFD_FLAGS; in userfaultfd_unregister()
1641 pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT); in userfaultfd_unregister()
1643 vma->anon_vma, vma->vm_file, pgoff, in userfaultfd_unregister()
1644 vma_policy(vma), in userfaultfd_unregister()
1645 NULL_VM_UFFD_CTX, anon_vma_name(vma)); in userfaultfd_unregister()
1647 vma = prev; in userfaultfd_unregister()
1650 if (vma->vm_start < start) { in userfaultfd_unregister()
1651 ret = split_vma(&vmi, vma, start, 1); in userfaultfd_unregister()
1655 if (vma->vm_end > end) { in userfaultfd_unregister()
1656 ret = split_vma(&vmi, vma, end, 0); in userfaultfd_unregister()
1666 vma_start_write(vma); in userfaultfd_unregister()
1667 userfaultfd_set_vm_flags(vma, new_flags); in userfaultfd_unregister()
1668 vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX; in userfaultfd_unregister()
1671 prev = vma; in userfaultfd_unregister()
1672 start = vma->vm_end; in userfaultfd_unregister()