Lines Matching refs:vma
121 static bool valid_vma(struct vm_area_struct *vma, bool is_register) in valid_vma() argument
128 return vma->vm_file && (vma->vm_flags & flags) == VM_MAYEXEC; in valid_vma()
131 static unsigned long offset_to_vaddr(struct vm_area_struct *vma, loff_t offset) in offset_to_vaddr() argument
133 return vma->vm_start + offset - ((loff_t)vma->vm_pgoff << PAGE_SHIFT); in offset_to_vaddr()
136 static loff_t vaddr_to_offset(struct vm_area_struct *vma, unsigned long vaddr) in vaddr_to_offset() argument
138 return ((loff_t)vma->vm_pgoff << PAGE_SHIFT) + (vaddr - vma->vm_start); in vaddr_to_offset()
154 static int __replace_page(struct vm_area_struct *vma, unsigned long addr, in __replace_page() argument
159 struct mm_struct *mm = vma->vm_mm; in __replace_page()
160 DEFINE_FOLIO_VMA_WALK(pvmw, old_folio, vma, addr, 0); in __replace_page()
164 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, mm, addr, in __replace_page()
169 err = mem_cgroup_charge(new_folio, vma->vm_mm, GFP_KERNEL); in __replace_page()
185 page_add_new_anon_rmap(new_page, vma, addr); in __replace_page()
186 folio_add_lru_vma(new_folio, vma); in __replace_page()
196 flush_cache_page(vma, addr, pte_pfn(*pvmw.pte)); in __replace_page()
197 ptep_clear_flush_notify(vma, addr, pvmw.pte); in __replace_page()
200 mk_pte(new_page, vma->vm_page_prot)); in __replace_page()
202 page_remove_rmap(old_page, vma, false); in __replace_page()
339 struct vm_area_struct *vma) in valid_ref_ctr_vma() argument
341 unsigned long vaddr = offset_to_vaddr(vma, uprobe->ref_ctr_offset); in valid_ref_ctr_vma()
344 vma->vm_file && in valid_ref_ctr_vma()
345 file_inode(vma->vm_file) == uprobe->inode && in valid_ref_ctr_vma()
346 (vma->vm_flags & (VM_WRITE|VM_SHARED)) == VM_WRITE && in valid_ref_ctr_vma()
347 vma->vm_start <= vaddr && in valid_ref_ctr_vma()
348 vma->vm_end > vaddr; in valid_ref_ctr_vma()
369 struct vm_area_struct *vma; in __update_ref_ctr() local
377 FOLL_WRITE, &page, &vma, NULL); in __update_ref_ctr()
466 struct vm_area_struct *vma; in uprobe_write_opcode() local
479 &old_page, &vma, NULL); in uprobe_write_opcode()
506 ret = anon_vma_prepare(vma); in uprobe_write_opcode()
511 new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, vaddr); in uprobe_write_opcode()
525 index = vaddr_to_offset(vma, vaddr & PAGE_MASK) >> PAGE_SHIFT; in uprobe_write_opcode()
526 orig_page = find_get_page(vma->vm_file->f_inode->i_mapping, in uprobe_write_opcode()
543 ret = __replace_page(vma, vaddr, old_page, new_page); in uprobe_write_opcode()
894 struct vm_area_struct *vma, unsigned long vaddr) in install_breakpoint() argument
899 ret = prepare_uprobe(uprobe, vma->vm_file, mm, vaddr); in install_breakpoint()
965 struct vm_area_struct *vma; in build_map_info() local
973 vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) { in build_map_info()
974 if (!valid_vma(vma, is_register)) in build_map_info()
992 if (!mmget_not_zero(vma->vm_mm)) in build_map_info()
1000 info->mm = vma->vm_mm; in build_map_info()
1001 info->vaddr = offset_to_vaddr(vma, offset); in build_map_info()
1048 struct vm_area_struct *vma; in register_for_each_vma() local
1054 vma = find_vma(mm, info->vaddr); in register_for_each_vma()
1055 if (!vma || !valid_vma(vma, is_register) || in register_for_each_vma()
1056 file_inode(vma->vm_file) != uprobe->inode) in register_for_each_vma()
1059 if (vma->vm_start > info->vaddr || in register_for_each_vma()
1060 vaddr_to_offset(vma, info->vaddr) != uprobe->offset) in register_for_each_vma()
1067 err = install_breakpoint(uprobe, mm, vma, info->vaddr); in register_for_each_vma()
1238 struct vm_area_struct *vma; in unapply_uprobe() local
1242 for_each_vma(vmi, vma) { in unapply_uprobe()
1246 if (!valid_vma(vma, false) || in unapply_uprobe()
1247 file_inode(vma->vm_file) != uprobe->inode) in unapply_uprobe()
1250 offset = (loff_t)vma->vm_pgoff << PAGE_SHIFT; in unapply_uprobe()
1252 uprobe->offset >= offset + vma->vm_end - vma->vm_start) in unapply_uprobe()
1255 vaddr = offset_to_vaddr(vma, uprobe->offset); in unapply_uprobe()
1292 struct vm_area_struct *vma, in build_probe_list() argument
1301 min = vaddr_to_offset(vma, start); in build_probe_list()
1326 static int delayed_ref_ctr_inc(struct vm_area_struct *vma) in delayed_ref_ctr_inc() argument
1337 if (du->mm != vma->vm_mm || in delayed_ref_ctr_inc()
1338 !valid_ref_ctr_vma(du->uprobe, vma)) in delayed_ref_ctr_inc()
1341 vaddr = offset_to_vaddr(vma, du->uprobe->ref_ctr_offset); in delayed_ref_ctr_inc()
1342 ret = __update_ref_ctr(vma->vm_mm, vaddr, 1); in delayed_ref_ctr_inc()
1344 update_ref_ctr_warn(du->uprobe, vma->vm_mm, 1); in delayed_ref_ctr_inc()
1360 int uprobe_mmap(struct vm_area_struct *vma) in uprobe_mmap() argument
1369 if (vma->vm_file && in uprobe_mmap()
1370 (vma->vm_flags & (VM_WRITE|VM_SHARED)) == VM_WRITE && in uprobe_mmap()
1371 test_bit(MMF_HAS_UPROBES, &vma->vm_mm->flags)) in uprobe_mmap()
1372 delayed_ref_ctr_inc(vma); in uprobe_mmap()
1374 if (!valid_vma(vma, true)) in uprobe_mmap()
1377 inode = file_inode(vma->vm_file); in uprobe_mmap()
1382 build_probe_list(inode, vma, vma->vm_start, vma->vm_end, &tmp_list); in uprobe_mmap()
1390 filter_chain(uprobe, UPROBE_FILTER_MMAP, vma->vm_mm)) { in uprobe_mmap()
1391 unsigned long vaddr = offset_to_vaddr(vma, uprobe->offset); in uprobe_mmap()
1392 install_breakpoint(uprobe, vma->vm_mm, vma, vaddr); in uprobe_mmap()
1402 vma_has_uprobes(struct vm_area_struct *vma, unsigned long start, unsigned long end) in vma_has_uprobes() argument
1408 inode = file_inode(vma->vm_file); in vma_has_uprobes()
1410 min = vaddr_to_offset(vma, start); in vma_has_uprobes()
1423 void uprobe_munmap(struct vm_area_struct *vma, unsigned long start, unsigned long end) in uprobe_munmap() argument
1425 if (no_uprobe_events() || !valid_vma(vma, false)) in uprobe_munmap()
1428 if (!atomic_read(&vma->vm_mm->mm_users)) /* called by mmput() ? */ in uprobe_munmap()
1431 if (!test_bit(MMF_HAS_UPROBES, &vma->vm_mm->flags) || in uprobe_munmap()
1432 test_bit(MMF_RECALC_UPROBES, &vma->vm_mm->flags)) in uprobe_munmap()
1435 if (vma_has_uprobes(vma, start, end)) in uprobe_munmap()
1436 set_bit(MMF_RECALC_UPROBES, &vma->vm_mm->flags); in uprobe_munmap()
1442 struct vm_area_struct *vma; in xol_add_vma() local
1463 vma = _install_special_mapping(mm, area->vaddr, PAGE_SIZE, in xol_add_vma()
1466 if (IS_ERR(vma)) { in xol_add_vma()
1467 ret = PTR_ERR(vma); in xol_add_vma()
1991 struct vm_area_struct *vma; in mmf_recalc_uprobes() local
1993 for_each_vma(vmi, vma) { in mmf_recalc_uprobes()
1994 if (!valid_vma(vma, false)) in mmf_recalc_uprobes()
2002 if (vma_has_uprobes(vma, vma->vm_start, vma->vm_end)) in mmf_recalc_uprobes()
2047 struct vm_area_struct *vma; in find_active_uprobe() local
2050 vma = vma_lookup(mm, bp_vaddr); in find_active_uprobe()
2051 if (vma) { in find_active_uprobe()
2052 if (valid_vma(vma, false)) { in find_active_uprobe()
2053 struct inode *inode = file_inode(vma->vm_file); in find_active_uprobe()
2054 loff_t offset = vaddr_to_offset(vma, bp_vaddr); in find_active_uprobe()