Lines Matching refs:mm
83 struct mm_struct *mm; member
159 struct mm_struct *mm = vma->vm_mm; in __replace_page() local
164 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, mm, addr, in __replace_page()
189 dec_mm_counter(mm, MM_ANONPAGES); in __replace_page()
192 dec_mm_counter(mm, mm_counter_file(old_page)); in __replace_page()
193 inc_mm_counter(mm, MM_ANONPAGES); in __replace_page()
199 set_pte_at_notify(mm, addr, pvmw.pte, in __replace_page()
283 delayed_uprobe_check(struct uprobe *uprobe, struct mm_struct *mm) in delayed_uprobe_check() argument
288 if (du->uprobe == uprobe && du->mm == mm) in delayed_uprobe_check()
293 static int delayed_uprobe_add(struct uprobe *uprobe, struct mm_struct *mm) in delayed_uprobe_add() argument
297 if (delayed_uprobe_check(uprobe, mm)) in delayed_uprobe_add()
305 du->mm = mm; in delayed_uprobe_add()
318 static void delayed_uprobe_remove(struct uprobe *uprobe, struct mm_struct *mm) in delayed_uprobe_remove() argument
323 if (!uprobe && !mm) in delayed_uprobe_remove()
331 if (mm && du->mm != mm) in delayed_uprobe_remove()
352 find_ref_ctr_vma(struct uprobe *uprobe, struct mm_struct *mm) in find_ref_ctr_vma() argument
354 VMA_ITERATOR(vmi, mm, 0); in find_ref_ctr_vma()
365 __update_ref_ctr(struct mm_struct *mm, unsigned long vaddr, short d) in __update_ref_ctr() argument
376 ret = get_user_pages_remote(mm, vaddr, 1, in __update_ref_ctr()
405 struct mm_struct *mm, short d) in update_ref_ctr_warn() argument
411 (unsigned long long) uprobe->ref_ctr_offset, mm); in update_ref_ctr_warn()
414 static int update_ref_ctr(struct uprobe *uprobe, struct mm_struct *mm, in update_ref_ctr() argument
421 rc_vma = find_ref_ctr_vma(uprobe, mm); in update_ref_ctr()
425 ret = __update_ref_ctr(mm, rc_vaddr, d); in update_ref_ctr()
427 update_ref_ctr_warn(uprobe, mm, d); in update_ref_ctr()
435 ret = delayed_uprobe_add(uprobe, mm); in update_ref_ctr()
437 delayed_uprobe_remove(uprobe, mm); in update_ref_ctr()
461 int uprobe_write_opcode(struct arch_uprobe *auprobe, struct mm_struct *mm, in uprobe_write_opcode() argument
478 ret = get_user_pages_remote(mm, vaddr, 1, gup_flags, in uprobe_write_opcode()
495 ret = update_ref_ctr(uprobe, mm, is_register ? 1 : -1); in uprobe_write_opcode()
554 update_ref_ctr(uprobe, mm, -1); in uprobe_write_opcode()
558 collapse_pte_mapped_thp(mm, vaddr, false); in uprobe_write_opcode()
572 int __weak set_swbp(struct arch_uprobe *auprobe, struct mm_struct *mm, unsigned long vaddr) in set_swbp() argument
574 return uprobe_write_opcode(auprobe, mm, vaddr, UPROBE_SWBP_INSN); in set_swbp()
587 set_orig_insn(struct arch_uprobe *auprobe, struct mm_struct *mm, unsigned long vaddr) in set_orig_insn() argument
589 return uprobe_write_opcode(auprobe, mm, vaddr, in set_orig_insn()
836 struct mm_struct *mm, unsigned long vaddr) in prepare_uprobe() argument
856 ret = arch_uprobe_analyze_insn(&uprobe->arch, mm, vaddr); in prepare_uprobe()
870 enum uprobe_filter_ctx ctx, struct mm_struct *mm) in consumer_filter() argument
872 return !uc->filter || uc->filter(uc, ctx, mm); in consumer_filter()
876 enum uprobe_filter_ctx ctx, struct mm_struct *mm) in filter_chain() argument
883 ret = consumer_filter(uc, ctx, mm); in filter_chain()
893 install_breakpoint(struct uprobe *uprobe, struct mm_struct *mm, in install_breakpoint() argument
899 ret = prepare_uprobe(uprobe, vma->vm_file, mm, vaddr); in install_breakpoint()
907 first_uprobe = !test_bit(MMF_HAS_UPROBES, &mm->flags); in install_breakpoint()
909 set_bit(MMF_HAS_UPROBES, &mm->flags); in install_breakpoint()
911 ret = set_swbp(&uprobe->arch, mm, vaddr); in install_breakpoint()
913 clear_bit(MMF_RECALC_UPROBES, &mm->flags); in install_breakpoint()
915 clear_bit(MMF_HAS_UPROBES, &mm->flags); in install_breakpoint()
921 remove_breakpoint(struct uprobe *uprobe, struct mm_struct *mm, unsigned long vaddr) in remove_breakpoint() argument
923 set_bit(MMF_RECALC_UPROBES, &mm->flags); in remove_breakpoint()
924 return set_orig_insn(&uprobe->arch, mm, vaddr); in remove_breakpoint()
950 struct mm_struct *mm; member
1000 info->mm = vma->vm_mm; in build_map_info()
1010 mmput(curr->mm); in build_map_info()
1047 struct mm_struct *mm = info->mm; in register_for_each_vma() local
1053 mmap_write_lock(mm); in register_for_each_vma()
1054 vma = find_vma(mm, info->vaddr); in register_for_each_vma()
1066 UPROBE_FILTER_REGISTER, mm)) in register_for_each_vma()
1067 err = install_breakpoint(uprobe, mm, vma, info->vaddr); in register_for_each_vma()
1068 } else if (test_bit(MMF_HAS_UPROBES, &mm->flags)) { in register_for_each_vma()
1070 UPROBE_FILTER_UNREGISTER, mm)) in register_for_each_vma()
1071 err |= remove_breakpoint(uprobe, mm, info->vaddr); in register_for_each_vma()
1075 mmap_write_unlock(mm); in register_for_each_vma()
1077 mmput(mm); in register_for_each_vma()
1235 static int unapply_uprobe(struct uprobe *uprobe, struct mm_struct *mm) in unapply_uprobe() argument
1237 VMA_ITERATOR(vmi, mm, 0); in unapply_uprobe()
1241 mmap_read_lock(mm); in unapply_uprobe()
1256 err |= remove_breakpoint(uprobe, mm, vaddr); in unapply_uprobe()
1258 mmap_read_unlock(mm); in unapply_uprobe()
1337 if (du->mm != vma->vm_mm || in delayed_ref_ctr_inc()
1440 static int xol_add_vma(struct mm_struct *mm, struct xol_area *area) in xol_add_vma() argument
1445 if (mmap_write_lock_killable(mm)) in xol_add_vma()
1448 if (mm->uprobes_state.xol_area) { in xol_add_vma()
1463 vma = _install_special_mapping(mm, area->vaddr, PAGE_SIZE, in xol_add_vma()
1473 smp_store_release(&mm->uprobes_state.xol_area, area); /* ^^^ */ in xol_add_vma()
1475 mmap_write_unlock(mm); in xol_add_vma()
1482 struct mm_struct *mm = current->mm; in __create_xol_area() local
1510 if (!xol_add_vma(mm, area)) in __create_xol_area()
1530 struct mm_struct *mm = current->mm; in get_xol_area() local
1533 if (!mm->uprobes_state.xol_area) in get_xol_area()
1537 area = READ_ONCE(mm->uprobes_state.xol_area); /* ^^^ */ in get_xol_area()
1544 void uprobe_clear_state(struct mm_struct *mm) in uprobe_clear_state() argument
1546 struct xol_area *area = mm->uprobes_state.xol_area; in uprobe_clear_state()
1549 delayed_uprobe_remove(NULL, mm); in uprobe_clear_state()
1639 if (!tsk->mm || !tsk->mm->uprobes_state.xol_area || !tsk->utask) in xol_free_insn_slot()
1646 area = tsk->mm->uprobes_state.xol_area; in xol_free_insn_slot()
1800 struct mm_struct *mm = current->mm; in uprobe_copy_process() local
1808 if (mm == t->mm && !(flags & CLONE_VFORK)) in uprobe_copy_process()
1815 area = mm->uprobes_state.xol_area; in uprobe_copy_process()
1819 if (mm == t->mm) in uprobe_copy_process()
1839 area = READ_ONCE(current->mm->uprobes_state.xol_area); /* ^^^ */ in get_trampoline_vaddr()
1988 static void mmf_recalc_uprobes(struct mm_struct *mm) in mmf_recalc_uprobes() argument
1990 VMA_ITERATOR(vmi, mm, 0); in mmf_recalc_uprobes()
2006 clear_bit(MMF_HAS_UPROBES, &mm->flags); in mmf_recalc_uprobes()
2009 static int is_trap_at_addr(struct mm_struct *mm, unsigned long vaddr) in is_trap_at_addr() argument
2031 result = get_user_pages_remote(mm, vaddr, 1, FOLL_FORCE, &page, in is_trap_at_addr()
2045 struct mm_struct *mm = current->mm; in find_active_uprobe() local
2049 mmap_read_lock(mm); in find_active_uprobe()
2050 vma = vma_lookup(mm, bp_vaddr); in find_active_uprobe()
2060 *is_swbp = is_trap_at_addr(mm, bp_vaddr); in find_active_uprobe()
2065 if (!uprobe && test_and_clear_bit(MMF_RECALC_UPROBES, &mm->flags)) in find_active_uprobe()
2066 mmf_recalc_uprobes(mm); in find_active_uprobe()
2067 mmap_read_unlock(mm); in find_active_uprobe()
2099 unapply_uprobe(uprobe, current->mm); in handler_chain()
2318 if (!current->mm) in uprobe_pre_sstep_notifier()
2321 if (!test_bit(MMF_HAS_UPROBES, ¤t->mm->flags) && in uprobe_pre_sstep_notifier()
2337 if (!current->mm || !utask || !utask->active_uprobe) in uprobe_post_sstep_notifier()