Lines Matching refs:uprobe
55 struct uprobe { struct
82 struct uprobe *uprobe; member
287 delayed_uprobe_check(struct uprobe *uprobe, struct mm_struct *mm) in delayed_uprobe_check() argument
292 if (du->uprobe == uprobe && du->mm == mm) in delayed_uprobe_check()
297 static int delayed_uprobe_add(struct uprobe *uprobe, struct mm_struct *mm) in delayed_uprobe_add() argument
301 if (delayed_uprobe_check(uprobe, mm)) in delayed_uprobe_add()
308 du->uprobe = uprobe; in delayed_uprobe_add()
322 static void delayed_uprobe_remove(struct uprobe *uprobe, struct mm_struct *mm) in delayed_uprobe_remove() argument
327 if (!uprobe && !mm) in delayed_uprobe_remove()
333 if (uprobe && du->uprobe != uprobe) in delayed_uprobe_remove()
342 static bool valid_ref_ctr_vma(struct uprobe *uprobe, in valid_ref_ctr_vma() argument
345 unsigned long vaddr = offset_to_vaddr(vma, uprobe->ref_ctr_offset); in valid_ref_ctr_vma()
347 return uprobe->ref_ctr_offset && in valid_ref_ctr_vma()
349 file_inode(vma->vm_file) == uprobe->inode && in valid_ref_ctr_vma()
356 find_ref_ctr_vma(struct uprobe *uprobe, struct mm_struct *mm) in find_ref_ctr_vma() argument
361 if (valid_ref_ctr_vma(uprobe, tmp)) in find_ref_ctr_vma()
407 static void update_ref_ctr_warn(struct uprobe *uprobe, in update_ref_ctr_warn() argument
412 d > 0 ? "increment" : "decrement", uprobe->inode->i_ino, in update_ref_ctr_warn()
413 (unsigned long long) uprobe->offset, in update_ref_ctr_warn()
414 (unsigned long long) uprobe->ref_ctr_offset, mm); in update_ref_ctr_warn()
417 static int update_ref_ctr(struct uprobe *uprobe, struct mm_struct *mm, in update_ref_ctr() argument
424 rc_vma = find_ref_ctr_vma(uprobe, mm); in update_ref_ctr()
427 rc_vaddr = offset_to_vaddr(rc_vma, uprobe->ref_ctr_offset); in update_ref_ctr()
430 update_ref_ctr_warn(uprobe, mm, d); in update_ref_ctr()
438 ret = delayed_uprobe_add(uprobe, mm); in update_ref_ctr()
440 delayed_uprobe_remove(uprobe, mm); in update_ref_ctr()
466 struct uprobe *uprobe; in uprobe_write_opcode() local
474 uprobe = container_of(auprobe, struct uprobe, arch); in uprobe_write_opcode()
496 if (!ref_ctr_updated && uprobe->ref_ctr_offset) { in uprobe_write_opcode()
497 ret = update_ref_ctr(uprobe, mm, is_register ? 1 : -1); in uprobe_write_opcode()
556 update_ref_ctr(uprobe, mm, -1); in uprobe_write_opcode()
595 static struct uprobe *get_uprobe(struct uprobe *uprobe) in get_uprobe() argument
597 refcount_inc(&uprobe->ref); in get_uprobe()
598 return uprobe; in get_uprobe()
601 static void put_uprobe(struct uprobe *uprobe) in put_uprobe() argument
603 if (refcount_dec_and_test(&uprobe->ref)) { in put_uprobe()
610 delayed_uprobe_remove(uprobe, NULL); in put_uprobe()
612 kfree(uprobe); in put_uprobe()
616 static int match_uprobe(struct uprobe *l, struct uprobe *r) in match_uprobe()
633 static struct uprobe *__find_uprobe(struct inode *inode, loff_t offset) in __find_uprobe()
635 struct uprobe u = { .inode = inode, .offset = offset }; in __find_uprobe()
637 struct uprobe *uprobe; in __find_uprobe() local
641 uprobe = rb_entry(n, struct uprobe, rb_node); in __find_uprobe()
642 match = match_uprobe(&u, uprobe); in __find_uprobe()
644 return get_uprobe(uprobe); in __find_uprobe()
658 static struct uprobe *find_uprobe(struct inode *inode, loff_t offset) in find_uprobe()
660 struct uprobe *uprobe; in find_uprobe() local
663 uprobe = __find_uprobe(inode, offset); in find_uprobe()
666 return uprobe; in find_uprobe()
669 static struct uprobe *__insert_uprobe(struct uprobe *uprobe) in __insert_uprobe() argument
673 struct uprobe *u; in __insert_uprobe()
678 u = rb_entry(parent, struct uprobe, rb_node); in __insert_uprobe()
679 match = match_uprobe(uprobe, u); in __insert_uprobe()
691 rb_link_node(&uprobe->rb_node, parent, p); in __insert_uprobe()
692 rb_insert_color(&uprobe->rb_node, &uprobes_tree); in __insert_uprobe()
694 refcount_set(&uprobe->ref, 2); in __insert_uprobe()
707 static struct uprobe *insert_uprobe(struct uprobe *uprobe) in insert_uprobe() argument
709 struct uprobe *u; in insert_uprobe()
712 u = __insert_uprobe(uprobe); in insert_uprobe()
719 ref_ctr_mismatch_warn(struct uprobe *cur_uprobe, struct uprobe *uprobe) in ref_ctr_mismatch_warn() argument
723 uprobe->inode->i_ino, (unsigned long long) uprobe->offset, in ref_ctr_mismatch_warn()
725 (unsigned long long) uprobe->ref_ctr_offset); in ref_ctr_mismatch_warn()
728 static struct uprobe *alloc_uprobe(struct inode *inode, loff_t offset, in alloc_uprobe()
731 struct uprobe *uprobe, *cur_uprobe; in alloc_uprobe() local
733 uprobe = kzalloc(sizeof(struct uprobe), GFP_KERNEL); in alloc_uprobe()
734 if (!uprobe) in alloc_uprobe()
737 uprobe->inode = inode; in alloc_uprobe()
738 uprobe->offset = offset; in alloc_uprobe()
739 uprobe->ref_ctr_offset = ref_ctr_offset; in alloc_uprobe()
740 init_rwsem(&uprobe->register_rwsem); in alloc_uprobe()
741 init_rwsem(&uprobe->consumer_rwsem); in alloc_uprobe()
744 cur_uprobe = insert_uprobe(uprobe); in alloc_uprobe()
747 if (cur_uprobe->ref_ctr_offset != uprobe->ref_ctr_offset) { in alloc_uprobe()
748 ref_ctr_mismatch_warn(cur_uprobe, uprobe); in alloc_uprobe()
750 kfree(uprobe); in alloc_uprobe()
753 kfree(uprobe); in alloc_uprobe()
754 uprobe = cur_uprobe; in alloc_uprobe()
757 return uprobe; in alloc_uprobe()
760 static void consumer_add(struct uprobe *uprobe, struct uprobe_consumer *uc) in consumer_add() argument
762 down_write(&uprobe->consumer_rwsem); in consumer_add()
763 uc->next = uprobe->consumers; in consumer_add()
764 uprobe->consumers = uc; in consumer_add()
765 up_write(&uprobe->consumer_rwsem); in consumer_add()
773 static bool consumer_del(struct uprobe *uprobe, struct uprobe_consumer *uc) in consumer_del() argument
778 down_write(&uprobe->consumer_rwsem); in consumer_del()
779 for (con = &uprobe->consumers; *con; con = &(*con)->next) { in consumer_del()
786 up_write(&uprobe->consumer_rwsem); in consumer_del()
813 static int copy_insn(struct uprobe *uprobe, struct file *filp) in copy_insn() argument
815 struct address_space *mapping = uprobe->inode->i_mapping; in copy_insn()
816 loff_t offs = uprobe->offset; in copy_insn()
817 void *insn = &uprobe->arch.insn; in copy_insn()
818 int size = sizeof(uprobe->arch.insn); in copy_insn()
823 if (offs >= i_size_read(uprobe->inode)) in copy_insn()
839 static int prepare_uprobe(struct uprobe *uprobe, struct file *file, in prepare_uprobe() argument
844 if (test_bit(UPROBE_COPY_INSN, &uprobe->flags)) in prepare_uprobe()
848 down_write(&uprobe->consumer_rwsem); in prepare_uprobe()
849 if (test_bit(UPROBE_COPY_INSN, &uprobe->flags)) in prepare_uprobe()
852 ret = copy_insn(uprobe, file); in prepare_uprobe()
857 if (is_trap_insn((uprobe_opcode_t *)&uprobe->arch.insn)) in prepare_uprobe()
860 ret = arch_uprobe_analyze_insn(&uprobe->arch, mm, vaddr); in prepare_uprobe()
865 set_bit(UPROBE_COPY_INSN, &uprobe->flags); in prepare_uprobe()
868 up_write(&uprobe->consumer_rwsem); in prepare_uprobe()
879 static bool filter_chain(struct uprobe *uprobe, in filter_chain() argument
885 down_read(&uprobe->consumer_rwsem); in filter_chain()
886 for (uc = uprobe->consumers; uc; uc = uc->next) { in filter_chain()
891 up_read(&uprobe->consumer_rwsem); in filter_chain()
897 install_breakpoint(struct uprobe *uprobe, struct mm_struct *mm, in install_breakpoint() argument
903 ret = prepare_uprobe(uprobe, vma->vm_file, mm, vaddr); in install_breakpoint()
915 ret = set_swbp(&uprobe->arch, mm, vaddr); in install_breakpoint()
925 remove_breakpoint(struct uprobe *uprobe, struct mm_struct *mm, unsigned long vaddr) in remove_breakpoint() argument
928 return set_orig_insn(&uprobe->arch, mm, vaddr); in remove_breakpoint()
931 static inline bool uprobe_is_active(struct uprobe *uprobe) in uprobe_is_active() argument
933 return !RB_EMPTY_NODE(&uprobe->rb_node); in uprobe_is_active()
940 static void delete_uprobe(struct uprobe *uprobe) in delete_uprobe() argument
942 if (WARN_ON(!uprobe_is_active(uprobe))) in delete_uprobe()
946 rb_erase(&uprobe->rb_node, &uprobes_tree); in delete_uprobe()
948 RB_CLEAR_NODE(&uprobe->rb_node); /* for uprobe_is_active() */ in delete_uprobe()
949 put_uprobe(uprobe); in delete_uprobe()
1036 register_for_each_vma(struct uprobe *uprobe, struct uprobe_consumer *new) in register_for_each_vma() argument
1043 info = build_map_info(uprobe->inode->i_mapping, in register_for_each_vma()
1044 uprobe->offset, is_register); in register_for_each_vma()
1060 file_inode(vma->vm_file) != uprobe->inode) in register_for_each_vma()
1064 vaddr_to_offset(vma, info->vaddr) != uprobe->offset) in register_for_each_vma()
1071 err = install_breakpoint(uprobe, mm, vma, info->vaddr); in register_for_each_vma()
1073 if (!filter_chain(uprobe, in register_for_each_vma()
1075 err |= remove_breakpoint(uprobe, mm, info->vaddr); in register_for_each_vma()
1090 __uprobe_unregister(struct uprobe *uprobe, struct uprobe_consumer *uc) in __uprobe_unregister() argument
1094 if (WARN_ON(!consumer_del(uprobe, uc))) in __uprobe_unregister()
1097 err = register_for_each_vma(uprobe, NULL); in __uprobe_unregister()
1099 if (!uprobe->consumers && !err) in __uprobe_unregister()
1100 delete_uprobe(uprobe); in __uprobe_unregister()
1111 struct uprobe *uprobe; in uprobe_unregister() local
1113 uprobe = find_uprobe(inode, offset); in uprobe_unregister()
1114 if (WARN_ON(!uprobe)) in uprobe_unregister()
1117 down_write(&uprobe->register_rwsem); in uprobe_unregister()
1118 __uprobe_unregister(uprobe, uc); in uprobe_unregister()
1119 up_write(&uprobe->register_rwsem); in uprobe_unregister()
1120 put_uprobe(uprobe); in uprobe_unregister()
1145 struct uprobe *uprobe; in __uprobe_register() local
1169 uprobe = alloc_uprobe(inode, offset, ref_ctr_offset); in __uprobe_register()
1170 if (!uprobe) in __uprobe_register()
1172 if (IS_ERR(uprobe)) in __uprobe_register()
1173 return PTR_ERR(uprobe); in __uprobe_register()
1179 down_write(&uprobe->register_rwsem); in __uprobe_register()
1181 if (likely(uprobe_is_active(uprobe))) { in __uprobe_register()
1182 consumer_add(uprobe, uc); in __uprobe_register()
1183 ret = register_for_each_vma(uprobe, uc); in __uprobe_register()
1185 __uprobe_unregister(uprobe, uc); in __uprobe_register()
1187 up_write(&uprobe->register_rwsem); in __uprobe_register()
1188 put_uprobe(uprobe); in __uprobe_register()
1219 struct uprobe *uprobe; in uprobe_apply() local
1223 uprobe = find_uprobe(inode, offset); in uprobe_apply()
1224 if (WARN_ON(!uprobe)) in uprobe_apply()
1227 down_write(&uprobe->register_rwsem); in uprobe_apply()
1228 for (con = uprobe->consumers; con && con != uc ; con = con->next) in uprobe_apply()
1231 ret = register_for_each_vma(uprobe, add ? uc : NULL); in uprobe_apply()
1232 up_write(&uprobe->register_rwsem); in uprobe_apply()
1233 put_uprobe(uprobe); in uprobe_apply()
1238 static int unapply_uprobe(struct uprobe *uprobe, struct mm_struct *mm) in unapply_uprobe() argument
1249 file_inode(vma->vm_file) != uprobe->inode) in unapply_uprobe()
1253 if (uprobe->offset < offset || in unapply_uprobe()
1254 uprobe->offset >= offset + vma->vm_end - vma->vm_start) in unapply_uprobe()
1257 vaddr = offset_to_vaddr(vma, uprobe->offset); in unapply_uprobe()
1258 err |= remove_breakpoint(uprobe, mm, vaddr); in unapply_uprobe()
1271 struct uprobe *u = rb_entry(n, struct uprobe, rb_node); in find_node_in_range()
1300 struct uprobe *u; in build_probe_list()
1310 u = rb_entry(t, struct uprobe, rb_node); in build_probe_list()
1317 u = rb_entry(t, struct uprobe, rb_node); in build_probe_list()
1340 !valid_ref_ctr_vma(du->uprobe, vma)) in delayed_ref_ctr_inc()
1343 vaddr = offset_to_vaddr(vma, du->uprobe->ref_ctr_offset); in delayed_ref_ctr_inc()
1346 update_ref_ctr_warn(du->uprobe, vma->vm_mm, 1); in delayed_ref_ctr_inc()
1365 struct uprobe *uprobe, *u; in uprobe_mmap() local
1390 list_for_each_entry_safe(uprobe, u, &tmp_list, pending_list) { in uprobe_mmap()
1392 filter_chain(uprobe, UPROBE_FILTER_MMAP, vma->vm_mm)) { in uprobe_mmap()
1393 unsigned long vaddr = offset_to_vaddr(vma, uprobe->offset); in uprobe_mmap()
1394 install_breakpoint(uprobe, vma->vm_mm, vma, vaddr); in uprobe_mmap()
1396 put_uprobe(uprobe); in uprobe_mmap()
1611 static unsigned long xol_get_insn_slot(struct uprobe *uprobe) in xol_get_insn_slot() argument
1625 &uprobe->arch.ixol, sizeof(uprobe->arch.ixol)); in xol_get_insn_slot()
1708 put_uprobe(ri->uprobe); in free_ret_instance()
1769 get_uprobe(n->uprobe); in dup_utask()
1861 static void prepare_uretprobe(struct uprobe *uprobe, struct pt_regs *regs) in prepare_uretprobe() argument
1912 ri->uprobe = get_uprobe(uprobe); in prepare_uretprobe()
1929 pre_ssout(struct uprobe *uprobe, struct pt_regs *regs, unsigned long bp_vaddr) in pre_ssout() argument
1939 xol_vaddr = xol_get_insn_slot(uprobe); in pre_ssout()
1946 err = arch_uprobe_pre_xol(&uprobe->arch, regs); in pre_ssout()
1952 utask->active_uprobe = uprobe; in pre_ssout()
2044 static struct uprobe *find_active_uprobe(unsigned long bp_vaddr, int *is_swbp) in find_active_uprobe()
2047 struct uprobe *uprobe = NULL; in find_active_uprobe() local
2057 uprobe = find_uprobe(inode, offset); in find_active_uprobe()
2060 if (!uprobe) in find_active_uprobe()
2066 if (!uprobe && test_and_clear_bit(MMF_RECALC_UPROBES, &mm->flags)) in find_active_uprobe()
2070 return uprobe; in find_active_uprobe()
2073 static void handler_chain(struct uprobe *uprobe, struct pt_regs *regs) in handler_chain() argument
2079 down_read(&uprobe->register_rwsem); in handler_chain()
2080 for (uc = uprobe->consumers; uc; uc = uc->next) { in handler_chain()
2096 prepare_uretprobe(uprobe, regs); /* put bp at return */ in handler_chain()
2098 if (remove && uprobe->consumers) { in handler_chain()
2099 WARN_ON(!uprobe_is_active(uprobe)); in handler_chain()
2100 unapply_uprobe(uprobe, current->mm); in handler_chain()
2102 up_read(&uprobe->register_rwsem); in handler_chain()
2108 struct uprobe *uprobe = ri->uprobe; in handle_uretprobe_chain() local
2111 down_read(&uprobe->register_rwsem); in handle_uretprobe_chain()
2112 for (uc = uprobe->consumers; uc; uc = uc->next) { in handle_uretprobe_chain()
2116 up_read(&uprobe->register_rwsem); in handle_uretprobe_chain()
2190 struct uprobe *uprobe; in handle_swbp() local
2198 uprobe = find_active_uprobe(bp_vaddr, &is_swbp); in handle_swbp()
2199 if (!uprobe) { in handle_swbp()
2225 if (unlikely(!test_bit(UPROBE_COPY_INSN, &uprobe->flags))) in handle_swbp()
2241 if (arch_uprobe_ignore(&uprobe->arch, regs)) in handle_swbp()
2244 handler_chain(uprobe, regs); in handle_swbp()
2246 if (arch_uprobe_skip_sstep(&uprobe->arch, regs)) in handle_swbp()
2249 if (!pre_ssout(uprobe, regs, bp_vaddr)) in handle_swbp()
2254 put_uprobe(uprobe); in handle_swbp()
2263 struct uprobe *uprobe; in handle_singlestep() local
2266 uprobe = utask->active_uprobe; in handle_singlestep()
2268 err = arch_uprobe_post_xol(&uprobe->arch, regs); in handle_singlestep()
2270 arch_uprobe_abort_xol(&uprobe->arch, regs); in handle_singlestep()
2274 put_uprobe(uprobe); in handle_singlestep()