Lines Matching refs:vma
228 struct vm_area_struct *vma, in userfaultfd_huge_must_wait() argument
239 ptep = huge_pte_offset(mm, address, vma_mmu_pagesize(vma)); in userfaultfd_huge_must_wait()
260 struct vm_area_struct *vma, in userfaultfd_huge_must_wait() argument
354 struct mm_struct *mm = vmf->vma->vm_mm; in handle_userfault()
381 ctx = vmf->vma->vm_userfaultfd_ctx.ctx; in handle_userfault()
485 if (!is_vm_hugetlb_page(vmf->vma)) in handle_userfault()
489 must_wait = userfaultfd_huge_must_wait(ctx, vmf->vma, in handle_userfault()
637 struct vm_area_struct *vma; in userfaultfd_event_wait_completion() local
644 for (vma = mm->mmap; vma; vma = vma->vm_next) in userfaultfd_event_wait_completion()
645 if (vma->vm_userfaultfd_ctx.ctx == release_new_ctx) { in userfaultfd_event_wait_completion()
646 vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX; in userfaultfd_event_wait_completion()
647 vma->vm_flags &= ~(VM_UFFD_WP | VM_UFFD_MISSING); in userfaultfd_event_wait_completion()
671 int dup_userfaultfd(struct vm_area_struct *vma, struct list_head *fcs) in dup_userfaultfd() argument
676 octx = vma->vm_userfaultfd_ctx.ctx; in dup_userfaultfd()
678 vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX; in dup_userfaultfd()
679 vma->vm_flags &= ~(VM_UFFD_WP | VM_UFFD_MISSING); in dup_userfaultfd()
706 ctx->mm = vma->vm_mm; in dup_userfaultfd()
716 vma->vm_userfaultfd_ctx.ctx = ctx; in dup_userfaultfd()
744 void mremap_userfaultfd_prep(struct vm_area_struct *vma, in mremap_userfaultfd_prep() argument
749 ctx = vma->vm_userfaultfd_ctx.ctx; in mremap_userfaultfd_prep()
760 vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX; in mremap_userfaultfd_prep()
761 vma->vm_flags &= ~(VM_UFFD_WP | VM_UFFD_MISSING); in mremap_userfaultfd_prep()
790 bool userfaultfd_remove(struct vm_area_struct *vma, in userfaultfd_remove() argument
793 struct mm_struct *mm = vma->vm_mm; in userfaultfd_remove()
797 ctx = vma->vm_userfaultfd_ctx.ctx; in userfaultfd_remove()
829 int userfaultfd_unmap_prep(struct vm_area_struct *vma, in userfaultfd_unmap_prep() argument
833 for ( ; vma && vma->vm_start < end; vma = vma->vm_next) { in userfaultfd_unmap_prep()
835 struct userfaultfd_ctx *ctx = vma->vm_userfaultfd_ctx.ctx; in userfaultfd_unmap_prep()
879 struct vm_area_struct *vma, *prev; in userfaultfd_release() local
901 for (vma = mm->mmap; vma; vma = vma->vm_next) { in userfaultfd_release()
903 BUG_ON(!!vma->vm_userfaultfd_ctx.ctx ^ in userfaultfd_release()
904 !!(vma->vm_flags & (VM_UFFD_MISSING | VM_UFFD_WP))); in userfaultfd_release()
905 if (vma->vm_userfaultfd_ctx.ctx != ctx) { in userfaultfd_release()
906 prev = vma; in userfaultfd_release()
909 new_flags = vma->vm_flags & ~(VM_UFFD_MISSING | VM_UFFD_WP); in userfaultfd_release()
911 prev = vma_merge(mm, prev, vma->vm_start, vma->vm_end, in userfaultfd_release()
912 new_flags, vma->anon_vma, in userfaultfd_release()
913 vma->vm_file, vma->vm_pgoff, in userfaultfd_release()
914 vma_policy(vma), in userfaultfd_release()
916 vma_get_anon_name(vma)); in userfaultfd_release()
918 vma = prev; in userfaultfd_release()
920 prev = vma; in userfaultfd_release()
922 vma->vm_flags = new_flags; in userfaultfd_release()
923 vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX; in userfaultfd_release()
1297 static inline bool vma_can_userfault(struct vm_area_struct *vma) in vma_can_userfault() argument
1299 return vma_is_anonymous(vma) || is_vm_hugetlb_page(vma) || in vma_can_userfault()
1300 vma_is_shmem(vma); in vma_can_userfault()
1307 struct vm_area_struct *vma, *prev, *cur; in userfaultfd_register() local
1357 vma = find_vma_prev(mm, start, &prev); in userfaultfd_register()
1358 if (!vma) in userfaultfd_register()
1363 if (vma->vm_start >= end) in userfaultfd_register()
1370 if (is_vm_hugetlb_page(vma)) { in userfaultfd_register()
1371 unsigned long vma_hpagesize = vma_kernel_pagesize(vma); in userfaultfd_register()
1382 for (cur = vma; cur && cur->vm_start < end; cur = cur->vm_next) { in userfaultfd_register()
1440 if (vma->vm_start < start) in userfaultfd_register()
1441 prev = vma; in userfaultfd_register()
1447 BUG_ON(!vma_can_userfault(vma)); in userfaultfd_register()
1448 BUG_ON(vma->vm_userfaultfd_ctx.ctx && in userfaultfd_register()
1449 vma->vm_userfaultfd_ctx.ctx != ctx); in userfaultfd_register()
1450 WARN_ON(!(vma->vm_flags & VM_MAYWRITE)); in userfaultfd_register()
1456 if (vma->vm_userfaultfd_ctx.ctx == ctx && in userfaultfd_register()
1457 (vma->vm_flags & vm_flags) == vm_flags) in userfaultfd_register()
1460 if (vma->vm_start > start) in userfaultfd_register()
1461 start = vma->vm_start; in userfaultfd_register()
1462 vma_end = min(end, vma->vm_end); in userfaultfd_register()
1464 new_flags = (vma->vm_flags & ~vm_flags) | vm_flags; in userfaultfd_register()
1466 vma->anon_vma, vma->vm_file, vma->vm_pgoff, in userfaultfd_register()
1467 vma_policy(vma), in userfaultfd_register()
1469 vma_get_anon_name(vma)); in userfaultfd_register()
1471 vma = prev; in userfaultfd_register()
1474 if (vma->vm_start < start) { in userfaultfd_register()
1475 ret = split_vma(mm, vma, start, 1); in userfaultfd_register()
1479 if (vma->vm_end > end) { in userfaultfd_register()
1480 ret = split_vma(mm, vma, end, 0); in userfaultfd_register()
1490 vma->vm_flags = new_flags; in userfaultfd_register()
1491 vma->vm_userfaultfd_ctx.ctx = ctx; in userfaultfd_register()
1494 prev = vma; in userfaultfd_register()
1495 start = vma->vm_end; in userfaultfd_register()
1496 vma = vma->vm_next; in userfaultfd_register()
1497 } while (vma && vma->vm_start < end); in userfaultfd_register()
1520 struct vm_area_struct *vma, *prev, *cur; in userfaultfd_unregister() local
1547 vma = find_vma_prev(mm, start, &prev); in userfaultfd_unregister()
1548 if (!vma) in userfaultfd_unregister()
1553 if (vma->vm_start >= end) in userfaultfd_unregister()
1560 if (is_vm_hugetlb_page(vma)) { in userfaultfd_unregister()
1561 unsigned long vma_hpagesize = vma_kernel_pagesize(vma); in userfaultfd_unregister()
1572 for (cur = vma; cur && cur->vm_start < end; cur = cur->vm_next) { in userfaultfd_unregister()
1592 if (vma->vm_start < start) in userfaultfd_unregister()
1593 prev = vma; in userfaultfd_unregister()
1599 BUG_ON(!vma_can_userfault(vma)); in userfaultfd_unregister()
1605 if (!vma->vm_userfaultfd_ctx.ctx) in userfaultfd_unregister()
1608 WARN_ON(!(vma->vm_flags & VM_MAYWRITE)); in userfaultfd_unregister()
1610 if (vma->vm_start > start) in userfaultfd_unregister()
1611 start = vma->vm_start; in userfaultfd_unregister()
1612 vma_end = min(end, vma->vm_end); in userfaultfd_unregister()
1614 if (userfaultfd_missing(vma)) { in userfaultfd_unregister()
1624 wake_userfault(vma->vm_userfaultfd_ctx.ctx, &range); in userfaultfd_unregister()
1627 new_flags = vma->vm_flags & ~(VM_UFFD_MISSING | VM_UFFD_WP); in userfaultfd_unregister()
1629 vma->anon_vma, vma->vm_file, vma->vm_pgoff, in userfaultfd_unregister()
1630 vma_policy(vma), in userfaultfd_unregister()
1632 vma_get_anon_name(vma)); in userfaultfd_unregister()
1634 vma = prev; in userfaultfd_unregister()
1637 if (vma->vm_start < start) { in userfaultfd_unregister()
1638 ret = split_vma(mm, vma, start, 1); in userfaultfd_unregister()
1642 if (vma->vm_end > end) { in userfaultfd_unregister()
1643 ret = split_vma(mm, vma, end, 0); in userfaultfd_unregister()
1653 vma->vm_flags = new_flags; in userfaultfd_unregister()
1654 vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX; in userfaultfd_unregister()
1657 prev = vma; in userfaultfd_unregister()
1658 start = vma->vm_end; in userfaultfd_unregister()
1659 vma = vma->vm_next; in userfaultfd_unregister()
1660 } while (vma && vma->vm_start < end); in userfaultfd_unregister()