Lines Matching refs:tr
34 static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mutex);
38 struct bpf_trampoline *tr = ops->private; in bpf_tramp_ftrace_ops_func() local
45 lockdep_assert_held_once(&tr->mutex); in bpf_tramp_ftrace_ops_func()
52 if ((tr->flags & BPF_TRAMP_F_CALL_ORIG) && in bpf_tramp_ftrace_ops_func()
53 !(tr->flags & BPF_TRAMP_F_ORIG_STACK)) { in bpf_tramp_ftrace_ops_func()
54 if (WARN_ON_ONCE(tr->flags & BPF_TRAMP_F_SHARE_IPMODIFY)) in bpf_tramp_ftrace_ops_func()
57 tr->flags |= BPF_TRAMP_F_SHARE_IPMODIFY; in bpf_tramp_ftrace_ops_func()
76 if (!mutex_trylock(&tr->mutex)) { in bpf_tramp_ftrace_ops_func()
86 tr->flags |= BPF_TRAMP_F_SHARE_IPMODIFY; in bpf_tramp_ftrace_ops_func()
88 if ((tr->flags & BPF_TRAMP_F_CALL_ORIG) && in bpf_tramp_ftrace_ops_func()
89 !(tr->flags & BPF_TRAMP_F_ORIG_STACK)) in bpf_tramp_ftrace_ops_func()
90 ret = bpf_trampoline_update(tr, false /* lock_direct_mutex */); in bpf_tramp_ftrace_ops_func()
93 tr->flags &= ~BPF_TRAMP_F_SHARE_IPMODIFY; in bpf_tramp_ftrace_ops_func()
95 if (tr->flags & BPF_TRAMP_F_ORIG_STACK) in bpf_tramp_ftrace_ops_func()
96 ret = bpf_trampoline_update(tr, false /* lock_direct_mutex */); in bpf_tramp_ftrace_ops_func()
103 mutex_unlock(&tr->mutex); in bpf_tramp_ftrace_ops_func()
137 struct bpf_trampoline *tr; in bpf_trampoline_lookup() local
143 hlist_for_each_entry(tr, head, hlist) { in bpf_trampoline_lookup()
144 if (tr->key == key) { in bpf_trampoline_lookup()
145 refcount_inc(&tr->refcnt); in bpf_trampoline_lookup()
149 tr = kzalloc(sizeof(*tr), GFP_KERNEL); in bpf_trampoline_lookup()
150 if (!tr) in bpf_trampoline_lookup()
153 tr->fops = kzalloc(sizeof(struct ftrace_ops), GFP_KERNEL); in bpf_trampoline_lookup()
154 if (!tr->fops) { in bpf_trampoline_lookup()
155 kfree(tr); in bpf_trampoline_lookup()
156 tr = NULL; in bpf_trampoline_lookup()
159 tr->fops->private = tr; in bpf_trampoline_lookup()
160 tr->fops->ops_func = bpf_tramp_ftrace_ops_func; in bpf_trampoline_lookup()
163 tr->key = key; in bpf_trampoline_lookup()
164 INIT_HLIST_NODE(&tr->hlist); in bpf_trampoline_lookup()
165 hlist_add_head(&tr->hlist, head); in bpf_trampoline_lookup()
166 refcount_set(&tr->refcnt, 1); in bpf_trampoline_lookup()
167 mutex_init(&tr->mutex); in bpf_trampoline_lookup()
169 INIT_HLIST_HEAD(&tr->progs_hlist[i]); in bpf_trampoline_lookup()
172 return tr; in bpf_trampoline_lookup()
175 static int bpf_trampoline_module_get(struct bpf_trampoline *tr) in bpf_trampoline_module_get() argument
181 mod = __module_text_address((unsigned long) tr->func.addr); in bpf_trampoline_module_get()
185 tr->mod = mod; in bpf_trampoline_module_get()
189 static void bpf_trampoline_module_put(struct bpf_trampoline *tr) in bpf_trampoline_module_put() argument
191 module_put(tr->mod); in bpf_trampoline_module_put()
192 tr->mod = NULL; in bpf_trampoline_module_put()
195 static int unregister_fentry(struct bpf_trampoline *tr, void *old_addr) in unregister_fentry() argument
197 void *ip = tr->func.addr; in unregister_fentry()
200 if (tr->func.ftrace_managed) in unregister_fentry()
201 ret = unregister_ftrace_direct_multi(tr->fops, (long)old_addr); in unregister_fentry()
206 bpf_trampoline_module_put(tr); in unregister_fentry()
210 static int modify_fentry(struct bpf_trampoline *tr, void *old_addr, void *new_addr, in modify_fentry() argument
213 void *ip = tr->func.addr; in modify_fentry()
216 if (tr->func.ftrace_managed) { in modify_fentry()
218 ret = modify_ftrace_direct_multi(tr->fops, (long)new_addr); in modify_fentry()
220 ret = modify_ftrace_direct_multi_nolock(tr->fops, (long)new_addr); in modify_fentry()
228 static int register_fentry(struct bpf_trampoline *tr, void *new_addr) in register_fentry() argument
230 void *ip = tr->func.addr; in register_fentry()
236 if (!tr->fops) in register_fentry()
238 tr->func.ftrace_managed = true; in register_fentry()
241 if (bpf_trampoline_module_get(tr)) in register_fentry()
244 if (tr->func.ftrace_managed) { in register_fentry()
245 ftrace_set_filter_ip(tr->fops, (unsigned long)ip, 0, 1); in register_fentry()
246 ret = register_ftrace_direct_multi(tr->fops, (long)new_addr); in register_fentry()
252 bpf_trampoline_module_put(tr); in register_fentry()
257 bpf_trampoline_get_progs(const struct bpf_trampoline *tr, int *total, bool *ip_arg) in bpf_trampoline_get_progs() argument
270 tlinks[kind].nr_links = tr->progs_cnt[kind]; in bpf_trampoline_get_progs()
271 *total += tr->progs_cnt[kind]; in bpf_trampoline_get_progs()
274 hlist_for_each_entry(link, &tr->progs_hlist[kind], tramp_hlist) { in bpf_trampoline_get_progs()
421 static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mutex) in bpf_trampoline_update() argument
425 u32 orig_flags = tr->flags; in bpf_trampoline_update()
429 tlinks = bpf_trampoline_get_progs(tr, &total, &ip_arg); in bpf_trampoline_update()
434 err = unregister_fentry(tr, tr->cur_image->image); in bpf_trampoline_update()
435 bpf_tramp_image_put(tr->cur_image); in bpf_trampoline_update()
436 tr->cur_image = NULL; in bpf_trampoline_update()
437 tr->selector = 0; in bpf_trampoline_update()
441 im = bpf_tramp_image_alloc(tr->key, tr->selector); in bpf_trampoline_update()
448 tr->flags &= (BPF_TRAMP_F_SHARE_IPMODIFY | BPF_TRAMP_F_TAIL_CALL_CTX); in bpf_trampoline_update()
455 tr->flags |= BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_SKIP_FRAME; in bpf_trampoline_update()
457 tr->flags |= BPF_TRAMP_F_RESTORE_REGS; in bpf_trampoline_update()
461 tr->flags |= BPF_TRAMP_F_IP_ARG; in bpf_trampoline_update()
465 if ((tr->flags & BPF_TRAMP_F_SHARE_IPMODIFY) && in bpf_trampoline_update()
466 (tr->flags & BPF_TRAMP_F_CALL_ORIG)) in bpf_trampoline_update()
467 tr->flags |= BPF_TRAMP_F_ORIG_STACK; in bpf_trampoline_update()
471 &tr->func.model, tr->flags, tlinks, in bpf_trampoline_update()
472 tr->func.addr); in bpf_trampoline_update()
479 WARN_ON(tr->cur_image && tr->selector == 0); in bpf_trampoline_update()
480 WARN_ON(!tr->cur_image && tr->selector); in bpf_trampoline_update()
481 if (tr->cur_image) in bpf_trampoline_update()
483 err = modify_fentry(tr, tr->cur_image->image, im->image, lock_direct_mutex); in bpf_trampoline_update()
486 err = register_fentry(tr, im->image); in bpf_trampoline_update()
495 tr->fops->func = NULL; in bpf_trampoline_update()
496 tr->fops->trampoline = 0; in bpf_trampoline_update()
507 if (tr->cur_image) in bpf_trampoline_update()
508 bpf_tramp_image_put(tr->cur_image); in bpf_trampoline_update()
509 tr->cur_image = im; in bpf_trampoline_update()
510 tr->selector++; in bpf_trampoline_update()
514 tr->flags = orig_flags; in bpf_trampoline_update()
545 static int __bpf_trampoline_link_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr) in __bpf_trampoline_link_prog() argument
553 if (tr->extension_prog) in __bpf_trampoline_link_prog()
560 cnt += tr->progs_cnt[i]; in __bpf_trampoline_link_prog()
566 tr->extension_prog = link->link.prog; in __bpf_trampoline_link_prog()
567 return bpf_arch_text_poke(tr->func.addr, BPF_MOD_JUMP, NULL, in __bpf_trampoline_link_prog()
575 hlist_for_each_entry(link_exiting, &tr->progs_hlist[kind], tramp_hlist) { in __bpf_trampoline_link_prog()
582 hlist_add_head(&link->tramp_hlist, &tr->progs_hlist[kind]); in __bpf_trampoline_link_prog()
583 tr->progs_cnt[kind]++; in __bpf_trampoline_link_prog()
584 err = bpf_trampoline_update(tr, true /* lock_direct_mutex */); in __bpf_trampoline_link_prog()
587 tr->progs_cnt[kind]--; in __bpf_trampoline_link_prog()
592 int bpf_trampoline_link_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr) in bpf_trampoline_link_prog() argument
596 mutex_lock(&tr->mutex); in bpf_trampoline_link_prog()
597 err = __bpf_trampoline_link_prog(link, tr); in bpf_trampoline_link_prog()
598 mutex_unlock(&tr->mutex); in bpf_trampoline_link_prog()
602 static int __bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr) in __bpf_trampoline_unlink_prog() argument
609 WARN_ON_ONCE(!tr->extension_prog); in __bpf_trampoline_unlink_prog()
610 err = bpf_arch_text_poke(tr->func.addr, BPF_MOD_JUMP, in __bpf_trampoline_unlink_prog()
611 tr->extension_prog->bpf_func, NULL); in __bpf_trampoline_unlink_prog()
612 tr->extension_prog = NULL; in __bpf_trampoline_unlink_prog()
616 tr->progs_cnt[kind]--; in __bpf_trampoline_unlink_prog()
617 return bpf_trampoline_update(tr, true /* lock_direct_mutex */); in __bpf_trampoline_unlink_prog()
621 int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr) in bpf_trampoline_unlink_prog() argument
625 mutex_lock(&tr->mutex); in bpf_trampoline_unlink_prog()
626 err = __bpf_trampoline_unlink_prog(link, tr); in bpf_trampoline_unlink_prog()
627 mutex_unlock(&tr->mutex); in bpf_trampoline_unlink_prog()
693 static struct bpf_shim_tramp_link *cgroup_shim_find(struct bpf_trampoline *tr, in cgroup_shim_find() argument
700 hlist_for_each_entry(link, &tr->progs_hlist[kind], tramp_hlist) { in cgroup_shim_find()
716 struct bpf_trampoline *tr; in bpf_trampoline_link_cgroup_shim() local
731 tr = bpf_trampoline_get(key, &tgt_info); in bpf_trampoline_link_cgroup_shim()
732 if (!tr) in bpf_trampoline_link_cgroup_shim()
735 mutex_lock(&tr->mutex); in bpf_trampoline_link_cgroup_shim()
737 shim_link = cgroup_shim_find(tr, bpf_func); in bpf_trampoline_link_cgroup_shim()
742 mutex_unlock(&tr->mutex); in bpf_trampoline_link_cgroup_shim()
743 bpf_trampoline_put(tr); /* bpf_trampoline_get above */ in bpf_trampoline_link_cgroup_shim()
755 err = __bpf_trampoline_link_prog(&shim_link->link, tr); in bpf_trampoline_link_cgroup_shim()
759 shim_link->trampoline = tr; in bpf_trampoline_link_cgroup_shim()
762 mutex_unlock(&tr->mutex); in bpf_trampoline_link_cgroup_shim()
766 mutex_unlock(&tr->mutex); in bpf_trampoline_link_cgroup_shim()
772 bpf_trampoline_put(tr); /* bpf_trampoline_get above */ in bpf_trampoline_link_cgroup_shim()
780 struct bpf_trampoline *tr; in bpf_trampoline_unlink_cgroup_shim() local
788 tr = bpf_trampoline_lookup(key); in bpf_trampoline_unlink_cgroup_shim()
789 if (WARN_ON_ONCE(!tr)) in bpf_trampoline_unlink_cgroup_shim()
792 mutex_lock(&tr->mutex); in bpf_trampoline_unlink_cgroup_shim()
793 shim_link = cgroup_shim_find(tr, bpf_func); in bpf_trampoline_unlink_cgroup_shim()
794 mutex_unlock(&tr->mutex); in bpf_trampoline_unlink_cgroup_shim()
799 bpf_trampoline_put(tr); /* bpf_trampoline_lookup above */ in bpf_trampoline_unlink_cgroup_shim()
806 struct bpf_trampoline *tr; in bpf_trampoline_get() local
808 tr = bpf_trampoline_lookup(key); in bpf_trampoline_get()
809 if (!tr) in bpf_trampoline_get()
812 mutex_lock(&tr->mutex); in bpf_trampoline_get()
813 if (tr->func.addr) in bpf_trampoline_get()
816 memcpy(&tr->func.model, &tgt_info->fmodel, sizeof(tgt_info->fmodel)); in bpf_trampoline_get()
817 tr->func.addr = (void *)tgt_info->tgt_addr; in bpf_trampoline_get()
819 mutex_unlock(&tr->mutex); in bpf_trampoline_get()
820 return tr; in bpf_trampoline_get()
823 void bpf_trampoline_put(struct bpf_trampoline *tr) in bpf_trampoline_put() argument
827 if (!tr) in bpf_trampoline_put()
830 if (!refcount_dec_and_test(&tr->refcnt)) in bpf_trampoline_put()
832 WARN_ON_ONCE(mutex_is_locked(&tr->mutex)); in bpf_trampoline_put()
835 if (WARN_ON_ONCE(!hlist_empty(&tr->progs_hlist[i]))) in bpf_trampoline_put()
844 hlist_del(&tr->hlist); in bpf_trampoline_put()
845 if (tr->fops) { in bpf_trampoline_put()
846 ftrace_free_filter(tr->fops); in bpf_trampoline_put()
847 kfree(tr->fops); in bpf_trampoline_put()
849 kfree(tr); in bpf_trampoline_put()
1026 void notrace __bpf_tramp_enter(struct bpf_tramp_image *tr) in __bpf_tramp_enter() argument
1028 percpu_ref_get(&tr->pcref); in __bpf_tramp_enter()
1031 void notrace __bpf_tramp_exit(struct bpf_tramp_image *tr) in __bpf_tramp_exit() argument
1033 percpu_ref_put(&tr->pcref); in __bpf_tramp_exit()
1067 arch_prepare_bpf_trampoline(struct bpf_tramp_image *tr, void *image, void *image_end, in arch_prepare_bpf_trampoline() argument