55 struct uprobe {  struct
82 	struct uprobe *uprobe;  argument
283 delayed_uprobe_check(struct uprobe *uprobe, struct mm_struct *mm)  in delayed_uprobe_check()  argument
288 		if (du->uprobe == uprobe && du->mm == mm)  in delayed_uprobe_check()
293 static int delayed_uprobe_add(struct uprobe *uprobe, struct mm_struct *mm)  in delayed_uprobe_add()  argument
297 	if (delayed_uprobe_check(uprobe, mm))  in delayed_uprobe_add()
304 	du->uprobe = uprobe;  in delayed_uprobe_add()
318 static void delayed_uprobe_remove(struct uprobe *uprobe, struct mm_struct *mm)  in delayed_uprobe_remove()  argument
323 	if (!uprobe && !mm)  in delayed_uprobe_remove()
329 		if (uprobe && du->uprobe != uprobe)  in delayed_uprobe_remove()
338 static bool valid_ref_ctr_vma(struct uprobe *uprobe,  in valid_ref_ctr_vma()  argument
341 	unsigned long vaddr = offset_to_vaddr(vma, uprobe->ref_ctr_offset);  in valid_ref_ctr_vma()
343 	return uprobe->ref_ctr_offset &&  in valid_ref_ctr_vma()
345 		file_inode(vma->vm_file) == uprobe->inode &&  in valid_ref_ctr_vma()
352 find_ref_ctr_vma(struct uprobe *uprobe, struct mm_struct *mm)  in find_ref_ctr_vma()  argument
358 		if (valid_ref_ctr_vma(uprobe, tmp))  in find_ref_ctr_vma()
404 static void update_ref_ctr_warn(struct uprobe *uprobe,  in update_ref_ctr_warn()  argument
409 		d > 0 ? "increment" : "decrement", uprobe->inode->i_ino,  in update_ref_ctr_warn()
410 		(unsigned long long) uprobe->offset,  in update_ref_ctr_warn()
411 		(unsigned long long) uprobe->ref_ctr_offset, mm);  in update_ref_ctr_warn()
414 static int update_ref_ctr(struct uprobe *uprobe, struct mm_struct *mm,  in update_ref_ctr()  argument
421 	rc_vma = find_ref_ctr_vma(uprobe, mm);  in update_ref_ctr()
424 		rc_vaddr = offset_to_vaddr(rc_vma, uprobe->ref_ctr_offset);  in update_ref_ctr()
427 			update_ref_ctr_warn(uprobe, mm, d);  in update_ref_ctr()
435 		ret = delayed_uprobe_add(uprobe, mm);  in update_ref_ctr()
437 		delayed_uprobe_remove(uprobe, mm);  in update_ref_ctr()
464 	struct uprobe *uprobe;  in uprobe_write_opcode()  local
472 	uprobe = container_of(auprobe, struct uprobe, arch);  in uprobe_write_opcode()
494 	if (!ref_ctr_updated && uprobe->ref_ctr_offset) {  in uprobe_write_opcode()
495 		ret = update_ref_ctr(uprobe, mm, is_register ? 1 : -1);  in uprobe_write_opcode()
554 		update_ref_ctr(uprobe, mm, -1);  in uprobe_write_opcode()
593 static struct uprobe *get_uprobe(struct uprobe *uprobe)  in get_uprobe()  argument
595 	refcount_inc(&uprobe->ref);  in get_uprobe()
596 	return uprobe;  in get_uprobe()
599 static void put_uprobe(struct uprobe *uprobe)  in put_uprobe()  argument
601 	if (refcount_dec_and_test(&uprobe->ref)) {  in put_uprobe()
608 		delayed_uprobe_remove(uprobe, NULL);  in put_uprobe()
610 		kfree(uprobe);  in put_uprobe()
616 	       const struct uprobe *r)  in uprobe_cmp()
634 	rb_entry((node), struct uprobe, rb_node)
649 	struct uprobe *u = __node_2_uprobe(a);  in __uprobe_cmp()
653 static struct uprobe *__find_uprobe(struct inode *inode, loff_t offset)  in __find_uprobe()
671 static struct uprobe *find_uprobe(struct inode *inode, loff_t offset)  in find_uprobe()
673 	struct uprobe *uprobe;  in find_uprobe()  local
676 	uprobe = __find_uprobe(inode, offset);  in find_uprobe()
679 	return uprobe;  in find_uprobe()
682 static struct uprobe *__insert_uprobe(struct uprobe *uprobe)  in __insert_uprobe()  argument
686 	node = rb_find_add(&uprobe->rb_node, &uprobes_tree, __uprobe_cmp);  in __insert_uprobe()
691 	refcount_set(&uprobe->ref, 2);  in __insert_uprobe()
703 static struct uprobe *insert_uprobe(struct uprobe *uprobe)  in insert_uprobe()  argument
705 	struct uprobe *u;  in insert_uprobe()
708 	u = __insert_uprobe(uprobe);  in insert_uprobe()
715 ref_ctr_mismatch_warn(struct uprobe *cur_uprobe, struct uprobe *uprobe)  in ref_ctr_mismatch_warn()  argument
719 		uprobe->inode->i_ino, (unsigned long long) uprobe->offset,  in ref_ctr_mismatch_warn()
721 		(unsigned long long) uprobe->ref_ctr_offset);  in ref_ctr_mismatch_warn()
724 static struct uprobe *alloc_uprobe(struct inode *inode, loff_t offset,  in alloc_uprobe()
727 	struct uprobe *uprobe, *cur_uprobe;  in alloc_uprobe()  local
729 	uprobe = kzalloc(sizeof(struct uprobe), GFP_KERNEL);  in alloc_uprobe()
730 	if (!uprobe)  in alloc_uprobe()
733 	uprobe->inode = inode;  in alloc_uprobe()
734 	uprobe->offset = offset;  in alloc_uprobe()
735 	uprobe->ref_ctr_offset = ref_ctr_offset;  in alloc_uprobe()
736 	init_rwsem(&uprobe->register_rwsem);  in alloc_uprobe()
737 	init_rwsem(&uprobe->consumer_rwsem);  in alloc_uprobe()
740 	cur_uprobe = insert_uprobe(uprobe);  in alloc_uprobe()
743 		if (cur_uprobe->ref_ctr_offset != uprobe->ref_ctr_offset) {  in alloc_uprobe()
744 			ref_ctr_mismatch_warn(cur_uprobe, uprobe);  in alloc_uprobe()
746 			kfree(uprobe);  in alloc_uprobe()
749 		kfree(uprobe);  in alloc_uprobe()
750 		uprobe = cur_uprobe;  in alloc_uprobe()
753 	return uprobe;  in alloc_uprobe()
756 static void consumer_add(struct uprobe *uprobe, struct uprobe_consumer *uc)  in consumer_add()  argument
758 	down_write(&uprobe->consumer_rwsem);  in consumer_add()
759 	uc->next = uprobe->consumers;  in consumer_add()
760 	uprobe->consumers = uc;  in consumer_add()
761 	up_write(&uprobe->consumer_rwsem);  in consumer_add()
769 static bool consumer_del(struct uprobe *uprobe, struct uprobe_consumer *uc)  in consumer_del()  argument
774 	down_write(&uprobe->consumer_rwsem);  in consumer_del()
775 	for (con = &uprobe->consumers; *con; con = &(*con)->next) {  in consumer_del()
782 	up_write(&uprobe->consumer_rwsem);  in consumer_del()
809 static int copy_insn(struct uprobe *uprobe, struct file *filp)  in copy_insn()  argument
811 	struct address_space *mapping = uprobe->inode->i_mapping;  in copy_insn()
812 	loff_t offs = uprobe->offset;  in copy_insn()
813 	void *insn = &uprobe->arch.insn;  in copy_insn()
814 	int size = sizeof(uprobe->arch.insn);  in copy_insn()
819 		if (offs >= i_size_read(uprobe->inode))  in copy_insn()
835 static int prepare_uprobe(struct uprobe *uprobe, struct file *file,  in prepare_uprobe()  argument
840 	if (test_bit(UPROBE_COPY_INSN, &uprobe->flags))  in prepare_uprobe()
844 	down_write(&uprobe->consumer_rwsem);  in prepare_uprobe()
845 	if (test_bit(UPROBE_COPY_INSN, &uprobe->flags))  in prepare_uprobe()
848 	ret = copy_insn(uprobe, file);  in prepare_uprobe()
853 	if (is_trap_insn((uprobe_opcode_t *)&uprobe->arch.insn))  in prepare_uprobe()
856 	ret = arch_uprobe_analyze_insn(&uprobe->arch, mm, vaddr);  in prepare_uprobe()
861 	set_bit(UPROBE_COPY_INSN, &uprobe->flags);  in prepare_uprobe()
864 	up_write(&uprobe->consumer_rwsem);  in prepare_uprobe()
875 static bool filter_chain(struct uprobe *uprobe,  in filter_chain()  argument
881 	down_read(&uprobe->consumer_rwsem);  in filter_chain()
882 	for (uc = uprobe->consumers; uc; uc = uc->next) {  in filter_chain()
887 	up_read(&uprobe->consumer_rwsem);  in filter_chain()
893 install_breakpoint(struct uprobe *uprobe, struct mm_struct *mm,  in install_breakpoint()  argument
899 	ret = prepare_uprobe(uprobe, vma->vm_file, mm, vaddr);  in install_breakpoint()
911 	ret = set_swbp(&uprobe->arch, mm, vaddr);  in install_breakpoint()
921 remove_breakpoint(struct uprobe *uprobe, struct mm_struct *mm, unsigned long vaddr)  in remove_breakpoint()  argument
924 	return set_orig_insn(&uprobe->arch, mm, vaddr);  in remove_breakpoint()
927 static inline bool uprobe_is_active(struct uprobe *uprobe)  in uprobe_is_active()  argument
929 	return !RB_EMPTY_NODE(&uprobe->rb_node);  in uprobe_is_active()
936 static void delete_uprobe(struct uprobe *uprobe)  in delete_uprobe()  argument
938 	if (WARN_ON(!uprobe_is_active(uprobe)))  in delete_uprobe()
942 	rb_erase(&uprobe->rb_node, &uprobes_tree);  in delete_uprobe()
944 	RB_CLEAR_NODE(&uprobe->rb_node); /* for uprobe_is_active() */  in delete_uprobe()
945 	put_uprobe(uprobe);  in delete_uprobe()
1032 register_for_each_vma(struct uprobe *uprobe, struct uprobe_consumer *new)  in register_for_each_vma()  argument
1039 	info = build_map_info(uprobe->inode->i_mapping,  in register_for_each_vma()
1040 					uprobe->offset, is_register);  in register_for_each_vma()
1056 		    file_inode(vma->vm_file) != uprobe->inode)  in register_for_each_vma()
1060 		    vaddr_to_offset(vma, info->vaddr) != uprobe->offset)  in register_for_each_vma()
1067 				err = install_breakpoint(uprobe, mm, vma, info->vaddr);  in register_for_each_vma()
1069 			if (!filter_chain(uprobe,  in register_for_each_vma()
1071 				err |= remove_breakpoint(uprobe, mm, info->vaddr);  in register_for_each_vma()
1086 __uprobe_unregister(struct uprobe *uprobe, struct uprobe_consumer *uc)  in __uprobe_unregister()  argument
1090 	if (WARN_ON(!consumer_del(uprobe, uc)))  in __uprobe_unregister()
1093 	err = register_for_each_vma(uprobe, NULL);  in __uprobe_unregister()
1095 	if (!uprobe->consumers && !err)  in __uprobe_unregister()
1096 		delete_uprobe(uprobe);  in __uprobe_unregister()
1107 	struct uprobe *uprobe;  in uprobe_unregister()  local
1109 	uprobe = find_uprobe(inode, offset);  in uprobe_unregister()
1110 	if (WARN_ON(!uprobe))  in uprobe_unregister()
1113 	down_write(&uprobe->register_rwsem);  in uprobe_unregister()
1114 	__uprobe_unregister(uprobe, uc);  in uprobe_unregister()
1115 	up_write(&uprobe->register_rwsem);  in uprobe_unregister()
1116 	put_uprobe(uprobe);  in uprobe_unregister()
1141 	struct uprobe *uprobe;  in __uprobe_register()  local
1166 	uprobe = alloc_uprobe(inode, offset, ref_ctr_offset);  in __uprobe_register()
1167 	if (!uprobe)  in __uprobe_register()
1169 	if (IS_ERR(uprobe))  in __uprobe_register()
1170 		return PTR_ERR(uprobe);  in __uprobe_register()
1176 	down_write(&uprobe->register_rwsem);  in __uprobe_register()
1178 	if (likely(uprobe_is_active(uprobe))) {  in __uprobe_register()
1179 		consumer_add(uprobe, uc);  in __uprobe_register()
1180 		ret = register_for_each_vma(uprobe, uc);  in __uprobe_register()
1182 			__uprobe_unregister(uprobe, uc);  in __uprobe_register()
1184 	up_write(&uprobe->register_rwsem);  in __uprobe_register()
1185 	put_uprobe(uprobe);  in __uprobe_register()
1216 	struct uprobe *uprobe;  in uprobe_apply()  local
1220 	uprobe = find_uprobe(inode, offset);  in uprobe_apply()
1221 	if (WARN_ON(!uprobe))  in uprobe_apply()
1224 	down_write(&uprobe->register_rwsem);  in uprobe_apply()
1225 	for (con = uprobe->consumers; con && con != uc ; con = con->next)  in uprobe_apply()
1228 		ret = register_for_each_vma(uprobe, add ? uc : NULL);  in uprobe_apply()
1229 	up_write(&uprobe->register_rwsem);  in uprobe_apply()
1230 	put_uprobe(uprobe);  in uprobe_apply()
1235 static int unapply_uprobe(struct uprobe *uprobe, struct mm_struct *mm)  in unapply_uprobe()  argument
1247 		    file_inode(vma->vm_file) != uprobe->inode)  in unapply_uprobe()
1251 		if (uprobe->offset <  offset ||  in unapply_uprobe()
1252 		    uprobe->offset >= offset + vma->vm_end - vma->vm_start)  in unapply_uprobe()
1255 		vaddr = offset_to_vaddr(vma, uprobe->offset);  in unapply_uprobe()
1256 		err |= remove_breakpoint(uprobe, mm, vaddr);  in unapply_uprobe()
1269 		struct uprobe *u = rb_entry(n, struct uprobe, rb_node);  in find_node_in_range()
1298 	struct uprobe *u;  in build_probe_list()
1308 			u = rb_entry(t, struct uprobe, rb_node);  in build_probe_list()
1315 			u = rb_entry(t, struct uprobe, rb_node);  in build_probe_list()
1338 		    !valid_ref_ctr_vma(du->uprobe, vma))  in delayed_ref_ctr_inc()
1341 		vaddr = offset_to_vaddr(vma, du->uprobe->ref_ctr_offset);  in delayed_ref_ctr_inc()
1344 			update_ref_ctr_warn(du->uprobe, vma->vm_mm, 1);  in delayed_ref_ctr_inc()
1363 	struct uprobe *uprobe, *u;  in uprobe_mmap()  local
1388 	list_for_each_entry_safe(uprobe, u, &tmp_list, pending_list) {  in uprobe_mmap()
1390 		    filter_chain(uprobe, UPROBE_FILTER_MMAP, vma->vm_mm)) {  in uprobe_mmap()
1391 			unsigned long vaddr = offset_to_vaddr(vma, uprobe->offset);  in uprobe_mmap()
1392 			install_breakpoint(uprobe, vma->vm_mm, vma, vaddr);  in uprobe_mmap()
1394 		put_uprobe(uprobe);  in uprobe_mmap()
1609 static unsigned long xol_get_insn_slot(struct uprobe *uprobe)  in xol_get_insn_slot()  argument
1623 			      &uprobe->arch.ixol, sizeof(uprobe->arch.ixol));  in xol_get_insn_slot()
1706 	put_uprobe(ri->uprobe);  in free_ret_instance()
1767 		get_uprobe(n->uprobe);  in dup_utask()
1859 static void prepare_uretprobe(struct uprobe *uprobe, struct pt_regs *regs)  in prepare_uretprobe()  argument
1910 	ri->uprobe = get_uprobe(uprobe);  in prepare_uretprobe()
1927 pre_ssout(struct uprobe *uprobe, struct pt_regs *regs, unsigned long bp_vaddr)  in pre_ssout()  argument
1937 	xol_vaddr = xol_get_insn_slot(uprobe);  in pre_ssout()
1944 	err = arch_uprobe_pre_xol(&uprobe->arch, regs);  in pre_ssout()
1950 	utask->active_uprobe = uprobe;  in pre_ssout()
2043 static struct uprobe *find_active_uprobe(unsigned long bp_vaddr, int *is_swbp)  in find_active_uprobe()
2046 	struct uprobe *uprobe = NULL;  in find_active_uprobe()  local
2056 			uprobe = find_uprobe(inode, offset);  in find_active_uprobe()
2059 		if (!uprobe)  in find_active_uprobe()
2065 	if (!uprobe && test_and_clear_bit(MMF_RECALC_UPROBES, &mm->flags))  in find_active_uprobe()
2069 	return uprobe;  in find_active_uprobe()
2072 static void handler_chain(struct uprobe *uprobe, struct pt_regs *regs)  in handler_chain()  argument
2078 	down_read(&uprobe->register_rwsem);  in handler_chain()
2079 	for (uc = uprobe->consumers; uc; uc = uc->next) {  in handler_chain()
2095 		prepare_uretprobe(uprobe, regs); /* put bp at return */  in handler_chain()
2097 	if (remove && uprobe->consumers) {  in handler_chain()
2098 		WARN_ON(!uprobe_is_active(uprobe));  in handler_chain()
2099 		unapply_uprobe(uprobe, current->mm);  in handler_chain()
2101 	up_read(&uprobe->register_rwsem);  in handler_chain()
2107 	struct uprobe *uprobe = ri->uprobe;  in handle_uretprobe_chain()  local
2110 	down_read(&uprobe->register_rwsem);  in handle_uretprobe_chain()
2111 	for (uc = uprobe->consumers; uc; uc = uc->next) {  in handle_uretprobe_chain()
2115 	up_read(&uprobe->register_rwsem);  in handle_uretprobe_chain()
2189 	struct uprobe *uprobe;  in handle_swbp()  local
2197 	uprobe = find_active_uprobe(bp_vaddr, &is_swbp);  in handle_swbp()
2198 	if (!uprobe) {  in handle_swbp()
2224 	if (unlikely(!test_bit(UPROBE_COPY_INSN, &uprobe->flags)))  in handle_swbp()
2240 	if (arch_uprobe_ignore(&uprobe->arch, regs))  in handle_swbp()
2243 	handler_chain(uprobe, regs);  in handle_swbp()
2245 	if (arch_uprobe_skip_sstep(&uprobe->arch, regs))  in handle_swbp()
2248 	if (!pre_ssout(uprobe, regs, bp_vaddr))  in handle_swbp()
2253 	put_uprobe(uprobe);  in handle_swbp()
2262 	struct uprobe *uprobe;  in handle_singlestep()  local
2265 	uprobe = utask->active_uprobe;  in handle_singlestep()
2267 		err = arch_uprobe_post_xol(&uprobe->arch, regs);  in handle_singlestep()
2269 		arch_uprobe_abort_xol(&uprobe->arch, regs);  in handle_singlestep()
2273 	put_uprobe(uprobe);  in handle_singlestep()