398 static inline void copy_kprobe(struct kprobe *ap, struct kprobe *p)  in copy_kprobe()  argument
400 	memcpy(&p->opcode, &ap->opcode, sizeof(kprobe_opcode_t));  in copy_kprobe()
401 	memcpy(&p->ainsn, &ap->ainsn, sizeof(struct arch_specific_insn));  in copy_kprobe()
753 static int reuse_unused_kprobe(struct kprobe *ap)  in reuse_unused_kprobe()  argument
761 	op = container_of(ap, struct optimized_kprobe, kp);  in reuse_unused_kprobe()
764 	ap->flags &= ~KPROBE_FLAG_DISABLED;  in reuse_unused_kprobe()
766 	if (!kprobe_optready(ap))  in reuse_unused_kprobe()
769 	optimize_kprobe(ap);  in reuse_unused_kprobe()
830 static void init_aggr_kprobe(struct kprobe *ap, struct kprobe *p);
838 	struct kprobe *ap;  in try_to_optimize_kprobe()  local
850 	ap = alloc_aggr_kprobe(p);  in try_to_optimize_kprobe()
851 	if (!ap)  in try_to_optimize_kprobe()
854 	op = container_of(ap, struct optimized_kprobe, kp);  in try_to_optimize_kprobe()
862 	init_aggr_kprobe(ap, p);  in try_to_optimize_kprobe()
863 	optimize_kprobe(ap);	/* This just kicks optimizer thread */  in try_to_optimize_kprobe()
994 static int reuse_unused_kprobe(struct kprobe *ap)  in reuse_unused_kprobe()  argument
1383 static int add_new_kprobe(struct kprobe *ap, struct kprobe *p)  in add_new_kprobe()  argument
1386 		unoptimize_kprobe(ap, true);	/* Fall back to normal kprobe */  in add_new_kprobe()
1388 	list_add_rcu(&p->list, &ap->list);  in add_new_kprobe()
1389 	if (p->post_handler && !ap->post_handler)  in add_new_kprobe()
1390 		ap->post_handler = aggr_post_handler;  in add_new_kprobe()
1399 static void init_aggr_kprobe(struct kprobe *ap, struct kprobe *p)  in init_aggr_kprobe()  argument
1402 	copy_kprobe(p, ap);  in init_aggr_kprobe()
1403 	flush_insn_slot(ap);  in init_aggr_kprobe()
1404 	ap->addr = p->addr;  in init_aggr_kprobe()
1405 	ap->flags = p->flags & ~KPROBE_FLAG_OPTIMIZED;  in init_aggr_kprobe()
1406 	ap->pre_handler = aggr_pre_handler;  in init_aggr_kprobe()
1407 	ap->fault_handler = aggr_fault_handler;  in init_aggr_kprobe()
1410 		ap->post_handler = aggr_post_handler;  in init_aggr_kprobe()
1412 	INIT_LIST_HEAD(&ap->list);  in init_aggr_kprobe()
1413 	INIT_HLIST_NODE(&ap->hlist);  in init_aggr_kprobe()
1415 	list_add_rcu(&p->list, &ap->list);  in init_aggr_kprobe()
1416 	hlist_replace_rcu(&p->hlist, &ap->hlist);  in init_aggr_kprobe()
1426 	struct kprobe *ap = orig_p;  in register_aggr_kprobe()  local
1436 		ap = alloc_aggr_kprobe(orig_p);  in register_aggr_kprobe()
1437 		if (!ap) {  in register_aggr_kprobe()
1441 		init_aggr_kprobe(ap, orig_p);  in register_aggr_kprobe()
1442 	} else if (kprobe_unused(ap)) {  in register_aggr_kprobe()
1444 		ret = reuse_unused_kprobe(ap);  in register_aggr_kprobe()
1449 	if (kprobe_gone(ap)) {  in register_aggr_kprobe()
1456 		ret = arch_prepare_kprobe(ap);  in register_aggr_kprobe()
1466 		prepare_optimized_kprobe(ap);  in register_aggr_kprobe()
1472 		ap->flags = (ap->flags & ~KPROBE_FLAG_GONE)  in register_aggr_kprobe()
1477 	copy_kprobe(ap, p);  in register_aggr_kprobe()
1478 	ret = add_new_kprobe(ap, p);  in register_aggr_kprobe()
1485 	if (ret == 0 && kprobe_disabled(ap) && !kprobe_disabled(p)) {  in register_aggr_kprobe()
1486 		ap->flags &= ~KPROBE_FLAG_DISABLED;  in register_aggr_kprobe()
1489 			ret = arm_kprobe(ap);  in register_aggr_kprobe()
1491 				ap->flags |= KPROBE_FLAG_DISABLED;  in register_aggr_kprobe()
1578 	struct kprobe *ap, *list_p;  in __get_valid_kprobe()  local
1582 	ap = get_kprobe(p->addr);  in __get_valid_kprobe()
1583 	if (unlikely(!ap))  in __get_valid_kprobe()
1586 	if (p != ap) {  in __get_valid_kprobe()
1587 		list_for_each_entry(list_p, &ap->list, list)  in __get_valid_kprobe()
1594 	return ap;  in __get_valid_kprobe()
1763 static int aggr_kprobe_disabled(struct kprobe *ap)  in aggr_kprobe_disabled()  argument
1769 	list_for_each_entry(kp, &ap->list, list)  in aggr_kprobe_disabled()
1823 	struct kprobe *ap, *list_p;  in __unregister_kprobe_top()  local
1826 	ap = __disable_kprobe(p);  in __unregister_kprobe_top()
1827 	if (IS_ERR(ap))  in __unregister_kprobe_top()
1828 		return PTR_ERR(ap);  in __unregister_kprobe_top()
1830 	if (ap == p)  in __unregister_kprobe_top()
1838 	WARN_ON(!kprobe_aggrprobe(ap));  in __unregister_kprobe_top()
1840 	if (list_is_singular(&ap->list) && kprobe_disarmed(ap))  in __unregister_kprobe_top()
1849 			list_for_each_entry(list_p, &ap->list, list) {  in __unregister_kprobe_top()
1858 			if (!kprobe_ftrace(ap))  in __unregister_kprobe_top()
1859 				ap->post_handler = NULL;  in __unregister_kprobe_top()
1867 		if (!kprobe_disabled(ap) && !kprobes_all_disarmed)  in __unregister_kprobe_top()
1872 			optimize_kprobe(ap);  in __unregister_kprobe_top()
1877 	hlist_del_rcu(&ap->hlist);  in __unregister_kprobe_top()
1883 	struct kprobe *ap;  in __unregister_kprobe_bottom()  local
1890 		ap = list_entry(p->list.next, struct kprobe, list);  in __unregister_kprobe_bottom()
1892 		free_aggr_kprobe(ap);  in __unregister_kprobe_bottom()