Lines Matching refs:ops
98 static bool ftrace_pids_enabled(struct ftrace_ops *ops) in ftrace_pids_enabled() argument
102 if (!(ops->flags & FTRACE_OPS_FL_PID) || !ops->private) in ftrace_pids_enabled()
105 tr = ops->private; in ftrace_pids_enabled()
110 static void ftrace_update_trampoline(struct ftrace_ops *ops);
128 static inline void ftrace_ops_init(struct ftrace_ops *ops) in ftrace_ops_init() argument
131 if (!(ops->flags & FTRACE_OPS_FL_INITIALIZED)) { in ftrace_ops_init()
132 mutex_init(&ops->local_hash.regex_lock); in ftrace_ops_init()
133 ops->func_hash = &ops->local_hash; in ftrace_ops_init()
134 ops->flags |= FTRACE_OPS_FL_INITIALIZED; in ftrace_ops_init()
163 static ftrace_func_t ftrace_ops_get_list_func(struct ftrace_ops *ops) in ftrace_ops_get_list_func() argument
169 if (ops->flags & (FTRACE_OPS_FL_DYNAMIC | FTRACE_OPS_FL_RCU) || in ftrace_ops_get_list_func()
173 return ftrace_ops_get_func(ops); in ftrace_ops_get_list_func()
256 struct ftrace_ops *ops) in add_ftrace_ops() argument
258 rcu_assign_pointer(ops->next, *list); in add_ftrace_ops()
266 rcu_assign_pointer(*list, ops); in add_ftrace_ops()
270 struct ftrace_ops *ops) in remove_ftrace_ops() argument
279 lockdep_is_held(&ftrace_lock)) == ops && in remove_ftrace_ops()
280 rcu_dereference_protected(ops->next, in remove_ftrace_ops()
287 if (*p == ops) in remove_ftrace_ops()
290 if (*p != ops) in remove_ftrace_ops()
297 static void ftrace_update_trampoline(struct ftrace_ops *ops);
299 int __register_ftrace_function(struct ftrace_ops *ops) in __register_ftrace_function() argument
301 if (ops->flags & FTRACE_OPS_FL_DELETED) in __register_ftrace_function()
304 if (WARN_ON(ops->flags & FTRACE_OPS_FL_ENABLED)) in __register_ftrace_function()
313 if (ops->flags & FTRACE_OPS_FL_SAVE_REGS && in __register_ftrace_function()
314 !(ops->flags & FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED)) in __register_ftrace_function()
317 if (ops->flags & FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED) in __register_ftrace_function()
318 ops->flags |= FTRACE_OPS_FL_SAVE_REGS; in __register_ftrace_function()
320 if (!ftrace_enabled && (ops->flags & FTRACE_OPS_FL_PERMANENT)) in __register_ftrace_function()
323 if (!is_kernel_core_data((unsigned long)ops)) in __register_ftrace_function()
324 ops->flags |= FTRACE_OPS_FL_DYNAMIC; in __register_ftrace_function()
326 add_ftrace_ops(&ftrace_ops_list, ops); in __register_ftrace_function()
329 ops->saved_func = ops->func; in __register_ftrace_function()
331 if (ftrace_pids_enabled(ops)) in __register_ftrace_function()
332 ops->func = ftrace_pid_func; in __register_ftrace_function()
334 ftrace_update_trampoline(ops); in __register_ftrace_function()
342 int __unregister_ftrace_function(struct ftrace_ops *ops) in __unregister_ftrace_function() argument
346 if (WARN_ON(!(ops->flags & FTRACE_OPS_FL_ENABLED))) in __unregister_ftrace_function()
349 ret = remove_ftrace_ops(&ftrace_ops_list, ops); in __unregister_ftrace_function()
357 ops->func = ops->saved_func; in __unregister_ftrace_function()
754 struct ftrace_ops *ops, struct ftrace_regs *fregs) in function_profile_call() argument
1014 struct ftrace_ops ops; member
1255 void ftrace_free_filter(struct ftrace_ops *ops) in ftrace_free_filter() argument
1257 ftrace_ops_init(ops); in ftrace_free_filter()
1258 free_ftrace_hash(ops->func_hash->filter_hash); in ftrace_free_filter()
1259 free_ftrace_hash(ops->func_hash->notrace_hash); in ftrace_free_filter()
1354 ftrace_hash_rec_disable_modify(struct ftrace_ops *ops, int filter_hash);
1356 ftrace_hash_rec_enable_modify(struct ftrace_ops *ops, int filter_hash);
1358 static int ftrace_hash_ipmodify_update(struct ftrace_ops *ops,
1412 ftrace_hash_move(struct ftrace_ops *ops, int enable, in ftrace_hash_move() argument
1419 if (ops->flags & FTRACE_OPS_FL_IPMODIFY && !enable) in ftrace_hash_move()
1429 ret = ftrace_hash_ipmodify_update(ops, new_hash); in ftrace_hash_move()
1440 ftrace_hash_rec_disable_modify(ops, enable); in ftrace_hash_move()
1444 ftrace_hash_rec_enable_modify(ops, enable); in ftrace_hash_move()
1477 ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip, void *regs) in ftrace_ops_test() argument
1488 if (regs == NULL && (ops->flags & FTRACE_OPS_FL_SAVE_REGS)) in ftrace_ops_test()
1492 rcu_assign_pointer(hash.filter_hash, ops->func_hash->filter_hash); in ftrace_ops_test()
1493 rcu_assign_pointer(hash.notrace_hash, ops->func_hash->notrace_hash); in ftrace_ops_test()
1630 struct ftrace_ops *ops; in test_rec_ops_needs_regs() local
1633 for (ops = ftrace_ops_list; in test_rec_ops_needs_regs()
1634 ops != &ftrace_list_end; ops = ops->next) { in test_rec_ops_needs_regs()
1636 if (ftrace_ops_test(ops, rec->ip, rec)) { in test_rec_ops_needs_regs()
1637 if (ops->flags & FTRACE_OPS_FL_SAVE_REGS) { in test_rec_ops_needs_regs()
1652 ftrace_find_tramp_ops_next(struct dyn_ftrace *rec, struct ftrace_ops *ops);
1666 static bool __ftrace_hash_rec_update(struct ftrace_ops *ops, in __ftrace_hash_rec_update() argument
1679 if (!(ops->flags & FTRACE_OPS_FL_ENABLED)) in __ftrace_hash_rec_update()
1694 hash = ops->func_hash->filter_hash; in __ftrace_hash_rec_update()
1695 other_hash = ops->func_hash->notrace_hash; in __ftrace_hash_rec_update()
1700 hash = ops->func_hash->notrace_hash; in __ftrace_hash_rec_update()
1701 other_hash = ops->func_hash->filter_hash; in __ftrace_hash_rec_update()
1753 if (ops->flags & FTRACE_OPS_FL_DIRECT) in __ftrace_hash_rec_update()
1761 if (ftrace_rec_count(rec) == 1 && ops->trampoline) in __ftrace_hash_rec_update()
1776 if (ops->flags & FTRACE_OPS_FL_SAVE_REGS) in __ftrace_hash_rec_update()
1789 if (ops->flags & FTRACE_OPS_FL_DIRECT) in __ftrace_hash_rec_update()
1800 ops->flags & FTRACE_OPS_FL_SAVE_REGS) { in __ftrace_hash_rec_update()
1813 ftrace_find_tramp_ops_any_other(rec, ops)) in __ftrace_hash_rec_update()
1836 static bool ftrace_hash_rec_disable(struct ftrace_ops *ops, in ftrace_hash_rec_disable() argument
1839 return __ftrace_hash_rec_update(ops, filter_hash, 0); in ftrace_hash_rec_disable()
1842 static bool ftrace_hash_rec_enable(struct ftrace_ops *ops, in ftrace_hash_rec_enable() argument
1845 return __ftrace_hash_rec_update(ops, filter_hash, 1); in ftrace_hash_rec_enable()
1848 static void ftrace_hash_rec_update_modify(struct ftrace_ops *ops, in ftrace_hash_rec_update_modify() argument
1853 __ftrace_hash_rec_update(ops, filter_hash, inc); in ftrace_hash_rec_update_modify()
1855 if (ops->func_hash != &global_ops.local_hash) in ftrace_hash_rec_update_modify()
1864 if (op == ops) in ftrace_hash_rec_update_modify()
1871 static void ftrace_hash_rec_disable_modify(struct ftrace_ops *ops, in ftrace_hash_rec_disable_modify() argument
1874 ftrace_hash_rec_update_modify(ops, filter_hash, 0); in ftrace_hash_rec_disable_modify()
1877 static void ftrace_hash_rec_enable_modify(struct ftrace_ops *ops, in ftrace_hash_rec_enable_modify() argument
1880 ftrace_hash_rec_update_modify(ops, filter_hash, 1); in ftrace_hash_rec_enable_modify()
1899 static int __ftrace_hash_update_ipmodify(struct ftrace_ops *ops, in __ftrace_hash_update_ipmodify() argument
1909 if (!(ops->flags & FTRACE_OPS_FL_ENABLED)) in __ftrace_hash_update_ipmodify()
1912 is_ipmodify = ops->flags & FTRACE_OPS_FL_IPMODIFY; in __ftrace_hash_update_ipmodify()
1913 is_direct = ops->flags & FTRACE_OPS_FL_DIRECT; in __ftrace_hash_update_ipmodify()
1958 if (!ops->ops_func) in __ftrace_hash_update_ipmodify()
1960 ret = ops->ops_func(ops, FTRACE_OPS_CMD_ENABLE_SHARE_IPMODIFY_SELF); in __ftrace_hash_update_ipmodify()
2000 static int ftrace_hash_ipmodify_enable(struct ftrace_ops *ops) in ftrace_hash_ipmodify_enable() argument
2002 struct ftrace_hash *hash = ops->func_hash->filter_hash; in ftrace_hash_ipmodify_enable()
2007 return __ftrace_hash_update_ipmodify(ops, EMPTY_HASH, hash); in ftrace_hash_ipmodify_enable()
2011 static void ftrace_hash_ipmodify_disable(struct ftrace_ops *ops) in ftrace_hash_ipmodify_disable() argument
2013 struct ftrace_hash *hash = ops->func_hash->filter_hash; in ftrace_hash_ipmodify_disable()
2018 __ftrace_hash_update_ipmodify(ops, hash, EMPTY_HASH); in ftrace_hash_ipmodify_disable()
2021 static int ftrace_hash_ipmodify_update(struct ftrace_ops *ops, in ftrace_hash_ipmodify_update() argument
2024 struct ftrace_hash *old_hash = ops->func_hash->filter_hash; in ftrace_hash_ipmodify_update()
2032 return __ftrace_hash_update_ipmodify(ops, old_hash, new_hash); in ftrace_hash_ipmodify_update()
2114 struct ftrace_ops *ops = NULL; in ftrace_bug() local
2120 ops = ftrace_find_tramp_ops_any(rec); in ftrace_bug()
2121 if (ops) { in ftrace_bug()
2124 (void *)ops->trampoline, in ftrace_bug()
2125 (void *)ops->func); in ftrace_bug()
2126 ops = ftrace_find_tramp_ops_next(rec, ops); in ftrace_bug()
2127 } while (ops); in ftrace_bug()
2495 struct ftrace_ops *ops, struct ftrace_regs *fregs) in call_direct_funcs() argument
2534 struct ftrace_ops *ops; in ftrace_get_addr_new() local
2547 ops = ftrace_find_tramp_ops_new(rec); in ftrace_get_addr_new()
2548 if (FTRACE_WARN_ON(!ops || !ops->trampoline)) { in ftrace_get_addr_new()
2554 return ops->trampoline; in ftrace_get_addr_new()
2575 struct ftrace_ops *ops; in ftrace_get_addr_curr() local
2588 ops = ftrace_find_tramp_ops_curr(rec); in ftrace_get_addr_curr()
2589 if (FTRACE_WARN_ON(!ops)) { in ftrace_get_addr_curr()
2595 return ops->trampoline; in ftrace_get_addr_curr()
2868 static void ftrace_run_modify_code(struct ftrace_ops *ops, int command, in ftrace_run_modify_code() argument
2871 ops->flags |= FTRACE_OPS_FL_MODIFYING; in ftrace_run_modify_code()
2872 ops->old_hash.filter_hash = old_hash->filter_hash; in ftrace_run_modify_code()
2873 ops->old_hash.notrace_hash = old_hash->notrace_hash; in ftrace_run_modify_code()
2875 ops->old_hash.filter_hash = NULL; in ftrace_run_modify_code()
2876 ops->old_hash.notrace_hash = NULL; in ftrace_run_modify_code()
2877 ops->flags &= ~FTRACE_OPS_FL_MODIFYING; in ftrace_run_modify_code()
2883 void __weak arch_ftrace_trampoline_free(struct ftrace_ops *ops) in arch_ftrace_trampoline_free() argument
2890 static void ftrace_add_trampoline_to_kallsyms(struct ftrace_ops *ops) in ftrace_add_trampoline_to_kallsyms() argument
2893 list_add_rcu(&ops->list, &ftrace_ops_trampoline_list); in ftrace_add_trampoline_to_kallsyms()
2896 static void ftrace_remove_trampoline_from_kallsyms(struct ftrace_ops *ops) in ftrace_remove_trampoline_from_kallsyms() argument
2899 list_del_rcu(&ops->list); in ftrace_remove_trampoline_from_kallsyms()
2911 static void ftrace_trampoline_free(struct ftrace_ops *ops) in ftrace_trampoline_free() argument
2913 if (ops && (ops->flags & FTRACE_OPS_FL_ALLOC_TRAMP) && in ftrace_trampoline_free()
2914 ops->trampoline) { in ftrace_trampoline_free()
2919 perf_event_text_poke((void *)ops->trampoline, in ftrace_trampoline_free()
2920 (void *)ops->trampoline, in ftrace_trampoline_free()
2921 ops->trampoline_size, NULL, 0); in ftrace_trampoline_free()
2923 ops->trampoline, ops->trampoline_size, in ftrace_trampoline_free()
2926 ftrace_remove_trampoline_from_kallsyms(ops); in ftrace_trampoline_free()
2929 arch_ftrace_trampoline_free(ops); in ftrace_trampoline_free()
2952 int ftrace_startup(struct ftrace_ops *ops, int command) in ftrace_startup() argument
2959 ret = __register_ftrace_function(ops); in ftrace_startup()
2973 ops->flags |= FTRACE_OPS_FL_ENABLED | FTRACE_OPS_FL_ADDING; in ftrace_startup()
2975 ret = ftrace_hash_ipmodify_enable(ops); in ftrace_startup()
2978 __unregister_ftrace_function(ops); in ftrace_startup()
2980 ops->flags &= ~FTRACE_OPS_FL_ENABLED; in ftrace_startup()
2981 if (ops->flags & FTRACE_OPS_FL_DYNAMIC) in ftrace_startup()
2982 ftrace_trampoline_free(ops); in ftrace_startup()
2986 if (ftrace_hash_rec_enable(ops, 1)) in ftrace_startup()
2997 __unregister_ftrace_function(ops); in ftrace_startup()
3001 ops->flags &= ~FTRACE_OPS_FL_ADDING; in ftrace_startup()
3006 int ftrace_shutdown(struct ftrace_ops *ops, int command) in ftrace_shutdown() argument
3013 ret = __unregister_ftrace_function(ops); in ftrace_shutdown()
3026 ftrace_hash_ipmodify_disable(ops); in ftrace_shutdown()
3028 if (ftrace_hash_rec_disable(ops, 1)) in ftrace_shutdown()
3031 ops->flags &= ~FTRACE_OPS_FL_ENABLED; in ftrace_shutdown()
3045 ops->flags |= FTRACE_OPS_FL_REMOVING; in ftrace_shutdown()
3046 removed_ops = ops; in ftrace_shutdown()
3049 ops->old_hash.filter_hash = ops->func_hash->filter_hash; in ftrace_shutdown()
3050 ops->old_hash.notrace_hash = ops->func_hash->notrace_hash; in ftrace_shutdown()
3070 ops->old_hash.filter_hash = NULL; in ftrace_shutdown()
3071 ops->old_hash.notrace_hash = NULL; in ftrace_shutdown()
3074 ops->flags &= ~FTRACE_OPS_FL_REMOVING; in ftrace_shutdown()
3083 if (ops->flags & FTRACE_OPS_FL_DYNAMIC) { in ftrace_shutdown()
3104 ftrace_trampoline_free(ops); in ftrace_shutdown()
3115 static inline int ops_traces_mod(struct ftrace_ops *ops) in ops_traces_mod() argument
3121 return ftrace_hash_empty(ops->func_hash->filter_hash) && in ops_traces_mod()
3122 ftrace_hash_empty(ops->func_hash->notrace_hash); in ops_traces_mod()
3287 struct ftrace_ops *ops; member
3299 struct trace_array *tr = iter->ops->private; in t_probe_next()
3325 hash = iter->probe->ops.func_hash->filter_hash; in t_probe_next()
3342 hash = iter->probe->ops.func_hash->filter_hash; in t_probe_next()
3637 arch_ftrace_trampoline_func(struct ftrace_ops *ops, struct dyn_ftrace *rec) in arch_ftrace_trampoline_func() argument
3642 static void add_trampoline_func(struct seq_file *m, struct ftrace_ops *ops, in add_trampoline_func() argument
3647 ptr = arch_ftrace_trampoline_func(ops, rec); in add_trampoline_func()
3783 struct ftrace_ops *ops; in t_show() local
3791 ops = ftrace_find_tramp_ops_any(rec); in t_show()
3792 if (ops) { in t_show()
3795 (void *)ops->trampoline, in t_show()
3796 (void *)ops->func); in t_show()
3797 add_trampoline_func(m, ops, rec); in t_show()
3798 ops = ftrace_find_tramp_ops_next(rec, ops); in t_show()
3799 } while (ops); in t_show()
3844 iter->ops = &global_ops; in ftrace_avail_open()
3869 iter->ops = &global_ops; in ftrace_enabled_open()
3891 ftrace_regex_open(struct ftrace_ops *ops, int flag, in ftrace_regex_open() argument
3897 struct trace_array *tr = ops->private; in ftrace_regex_open()
3900 ftrace_ops_init(ops); in ftrace_regex_open()
3915 iter->ops = ops; in ftrace_regex_open()
3919 mutex_lock(&ops->func_hash->regex_lock); in ftrace_regex_open()
3922 hash = ops->func_hash->notrace_hash; in ftrace_regex_open()
3925 hash = ops->func_hash->filter_hash; in ftrace_regex_open()
3966 mutex_unlock(&ops->func_hash->regex_lock); in ftrace_regex_open()
3981 struct ftrace_ops *ops = inode->i_private; in ftrace_filter_open() local
3984 return ftrace_regex_open(ops, in ftrace_filter_open()
3992 struct ftrace_ops *ops = inode->i_private; in ftrace_notrace_open() local
3995 return ftrace_regex_open(ops, FTRACE_ITER_NOTRACE, in ftrace_notrace_open()
4221 static void ftrace_ops_update_code(struct ftrace_ops *ops, in ftrace_ops_update_code() argument
4229 if (ops->flags & FTRACE_OPS_FL_ENABLED) { in ftrace_ops_update_code()
4230 ftrace_run_modify_code(ops, FTRACE_UPDATE_CALLS, old_hash); in ftrace_ops_update_code()
4239 if (ops->func_hash != &global_ops.local_hash) in ftrace_ops_update_code()
4252 static int ftrace_hash_move_and_update_ops(struct ftrace_ops *ops, in ftrace_hash_move_and_update_ops() argument
4262 old_hash_ops.filter_hash = ops->func_hash->filter_hash; in ftrace_hash_move_and_update_ops()
4263 old_hash_ops.notrace_hash = ops->func_hash->notrace_hash; in ftrace_hash_move_and_update_ops()
4264 ret = ftrace_hash_move(ops, enable, orig_hash, hash); in ftrace_hash_move_and_update_ops()
4266 ftrace_ops_update_code(ops, &old_hash_ops); in ftrace_hash_move_and_update_ops()
4334 ftrace_set_regex(struct ftrace_ops *ops, unsigned char *buf, int len,
4338 static void process_mod_list(struct list_head *head, struct ftrace_ops *ops, in process_mod_list() argument
4346 mutex_lock(&ops->func_hash->regex_lock); in process_mod_list()
4349 orig_hash = &ops->func_hash->filter_hash; in process_mod_list()
4351 orig_hash = &ops->func_hash->notrace_hash; in process_mod_list()
4396 ftrace_hash_move_and_update_ops(ops, orig_hash, in process_mod_list()
4401 mutex_unlock(&ops->func_hash->regex_lock); in process_mod_list()
4418 process_mod_list(&tr->mod_trace, tr->ops, mod, true); in process_cached_mods()
4420 process_mod_list(&tr->mod_notrace, tr->ops, mod, false); in process_cached_mods()
4479 probe = container_of(op, struct ftrace_func_probe, ops); in function_trace_probe_call()
4708 probe->ops.func = function_trace_probe_call; in register_ftrace_function_probe()
4710 ftrace_ops_init(&probe->ops); in register_ftrace_function_probe()
4722 mutex_lock(&probe->ops.func_hash->regex_lock); in register_ftrace_function_probe()
4724 orig_hash = &probe->ops.func_hash->filter_hash; in register_ftrace_function_probe()
4776 ret = ftrace_hash_move_and_update_ops(&probe->ops, orig_hash, in register_ftrace_function_probe()
4784 if (!(probe->ops.flags & FTRACE_OPS_FL_ENABLED)) in register_ftrace_function_probe()
4785 ret = ftrace_startup(&probe->ops, 0); in register_ftrace_function_probe()
4793 mutex_unlock(&probe->ops.func_hash->regex_lock); in register_ftrace_function_probe()
4859 if (!(probe->ops.flags & FTRACE_OPS_FL_INITIALIZED)) in unregister_ftrace_function_probe_func()
4866 mutex_lock(&probe->ops.func_hash->regex_lock); in unregister_ftrace_function_probe_func()
4868 orig_hash = &probe->ops.func_hash->filter_hash; in unregister_ftrace_function_probe_func()
4914 ftrace_shutdown(&probe->ops, 0); in unregister_ftrace_function_probe_func()
4916 ret = ftrace_hash_move_and_update_ops(&probe->ops, orig_hash, in unregister_ftrace_function_probe_func()
4921 ftrace_run_modify_code(&probe->ops, FTRACE_UPDATE_CALLS, in unregister_ftrace_function_probe_func()
4934 mutex_unlock(&probe->ops.func_hash->regex_lock); in unregister_ftrace_function_probe_func()
5007 struct trace_array *tr = iter->ops->private; in ftrace_process_regex()
5134 ftrace_set_hash(struct ftrace_ops *ops, unsigned char *buf, int len, in ftrace_set_hash() argument
5145 mutex_lock(&ops->func_hash->regex_lock); in ftrace_set_hash()
5148 orig_hash = &ops->func_hash->filter_hash; in ftrace_set_hash()
5150 orig_hash = &ops->func_hash->notrace_hash; in ftrace_set_hash()
5173 ret = ftrace_hash_move_and_update_ops(ops, orig_hash, hash, enable); in ftrace_set_hash()
5177 mutex_unlock(&ops->func_hash->regex_lock); in ftrace_set_hash()
5184 ftrace_set_addr(struct ftrace_ops *ops, unsigned long *ips, unsigned int cnt, in ftrace_set_addr() argument
5187 return ftrace_set_hash(ops, NULL, 0, ips, cnt, remove, reset, enable); in ftrace_set_addr()
5245 static int register_ftrace_function_nolock(struct ftrace_ops *ops);
5586 static int check_direct_multi(struct ftrace_ops *ops) in check_direct_multi() argument
5588 if (!(ops->flags & FTRACE_OPS_FL_INITIALIZED)) in check_direct_multi()
5590 if ((ops->flags & MULTI_FLAGS) != MULTI_FLAGS) in check_direct_multi()
5634 int register_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr) in register_ftrace_direct_multi() argument
5640 if (ops->func || ops->trampoline) in register_ftrace_direct_multi()
5642 if (!(ops->flags & FTRACE_OPS_FL_INITIALIZED)) in register_ftrace_direct_multi()
5644 if (ops->flags & FTRACE_OPS_FL_ENABLED) in register_ftrace_direct_multi()
5647 hash = ops->func_hash->filter_hash; in register_ftrace_direct_multi()
5673 ops->func = call_direct_funcs; in register_ftrace_direct_multi()
5674 ops->flags = MULTI_FLAGS; in register_ftrace_direct_multi()
5675 ops->trampoline = FTRACE_REGS_ADDR; in register_ftrace_direct_multi()
5677 err = register_ftrace_function_nolock(ops); in register_ftrace_direct_multi()
5707 int unregister_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr) in unregister_ftrace_direct_multi() argument
5709 struct ftrace_hash *hash = ops->func_hash->filter_hash; in unregister_ftrace_direct_multi()
5712 if (check_direct_multi(ops)) in unregister_ftrace_direct_multi()
5714 if (!(ops->flags & FTRACE_OPS_FL_ENABLED)) in unregister_ftrace_direct_multi()
5718 err = unregister_ftrace_function(ops); in unregister_ftrace_direct_multi()
5723 ops->func = NULL; in unregister_ftrace_direct_multi()
5724 ops->trampoline = 0; in unregister_ftrace_direct_multi()
5730 __modify_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr) in __modify_ftrace_direct_multi() argument
5745 tmp_ops.func_hash = ops->func_hash; in __modify_ftrace_direct_multi()
5757 hash = ops->func_hash->filter_hash; in __modify_ftrace_direct_multi()
5794 int modify_ftrace_direct_multi_nolock(struct ftrace_ops *ops, unsigned long addr) in modify_ftrace_direct_multi_nolock() argument
5796 if (check_direct_multi(ops)) in modify_ftrace_direct_multi_nolock()
5798 if (!(ops->flags & FTRACE_OPS_FL_ENABLED)) in modify_ftrace_direct_multi_nolock()
5801 return __modify_ftrace_direct_multi(ops, addr); in modify_ftrace_direct_multi_nolock()
5820 int modify_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr) in modify_ftrace_direct_multi() argument
5824 if (check_direct_multi(ops)) in modify_ftrace_direct_multi()
5826 if (!(ops->flags & FTRACE_OPS_FL_ENABLED)) in modify_ftrace_direct_multi()
5830 err = __modify_ftrace_direct_multi(ops, addr); in modify_ftrace_direct_multi()
5851 int ftrace_set_filter_ip(struct ftrace_ops *ops, unsigned long ip, in ftrace_set_filter_ip() argument
5854 ftrace_ops_init(ops); in ftrace_set_filter_ip()
5855 return ftrace_set_addr(ops, &ip, 1, remove, reset, 1); in ftrace_set_filter_ip()
5874 int ftrace_set_filter_ips(struct ftrace_ops *ops, unsigned long *ips, in ftrace_set_filter_ips() argument
5877 ftrace_ops_init(ops); in ftrace_set_filter_ips()
5878 return ftrace_set_addr(ops, ips, cnt, remove, reset, 1); in ftrace_set_filter_ips()
5889 void ftrace_ops_set_global_filter(struct ftrace_ops *ops) in ftrace_ops_set_global_filter() argument
5891 if (ops->flags & FTRACE_OPS_FL_INITIALIZED) in ftrace_ops_set_global_filter()
5894 ftrace_ops_init(ops); in ftrace_ops_set_global_filter()
5895 ops->func_hash = &global_ops.local_hash; in ftrace_ops_set_global_filter()
5900 ftrace_set_regex(struct ftrace_ops *ops, unsigned char *buf, int len, in ftrace_set_regex() argument
5903 return ftrace_set_hash(ops, buf, len, NULL, 0, 0, reset, enable); in ftrace_set_regex()
5920 int ftrace_set_filter(struct ftrace_ops *ops, unsigned char *buf, in ftrace_set_filter() argument
5923 ftrace_ops_init(ops); in ftrace_set_filter()
5924 return ftrace_set_regex(ops, buf, len, reset, 1); in ftrace_set_filter()
5943 int ftrace_set_notrace(struct ftrace_ops *ops, unsigned char *buf, in ftrace_set_notrace() argument
5946 ftrace_ops_init(ops); in ftrace_set_notrace()
5947 return ftrace_set_regex(ops, buf, len, reset, 0); in ftrace_set_notrace()
6062 ftrace_set_early_filter(struct ftrace_ops *ops, char *buf, int enable) in ftrace_set_early_filter() argument
6066 ftrace_ops_init(ops); in ftrace_set_early_filter()
6070 ftrace_set_regex(ops, func, strlen(func), 0, enable); in ftrace_set_early_filter()
6112 mutex_lock(&iter->ops->func_hash->regex_lock); in ftrace_regex_release()
6118 orig_hash = &iter->ops->func_hash->filter_hash; in ftrace_regex_release()
6126 orig_hash = &iter->ops->func_hash->notrace_hash; in ftrace_regex_release()
6129 ftrace_hash_move_and_update_ops(iter->ops, orig_hash, in ftrace_regex_release()
6137 mutex_unlock(&iter->ops->func_hash->regex_lock); in ftrace_regex_release()
6584 void ftrace_create_filter_files(struct ftrace_ops *ops, in ftrace_create_filter_files() argument
6589 ops, &ftrace_filter_fops); in ftrace_create_filter_files()
6592 ops, &ftrace_notrace_fops); in ftrace_create_filter_files()
6605 void ftrace_destroy_filter_files(struct ftrace_ops *ops) in ftrace_destroy_filter_files() argument
6608 if (ops->flags & FTRACE_OPS_FL_ENABLED) in ftrace_destroy_filter_files()
6609 ftrace_shutdown(ops, 0); in ftrace_destroy_filter_files()
6610 ops->flags |= FTRACE_OPS_FL_DELETED; in ftrace_destroy_filter_files()
6611 ftrace_free_filter(ops); in ftrace_destroy_filter_files()
6839 ops_references_ip(struct ftrace_ops *ops, unsigned long ip) in ops_references_ip() argument
6842 if (!(ops->flags & FTRACE_OPS_FL_ENABLED)) in ops_references_ip()
6846 if (ops_traces_mod(ops)) in ops_references_ip()
6850 if (!ftrace_hash_empty(ops->func_hash->filter_hash) && in ops_references_ip()
6851 !__ftrace_lookup_ip(ops->func_hash->filter_hash, ip)) in ops_references_ip()
6855 if (ftrace_lookup_ip(ops->func_hash->notrace_hash, ip)) in ops_references_ip()
6870 struct ftrace_ops *ops; in referenced_filters() local
6873 for (ops = ftrace_ops_list; ops != &ftrace_list_end; ops = ops->next) { in referenced_filters()
6874 if (ops_references_ip(ops, rec->ip)) { in referenced_filters()
6875 if (WARN_ON_ONCE(ops->flags & FTRACE_OPS_FL_DIRECT)) in referenced_filters()
6877 if (WARN_ON_ONCE(ops->flags & FTRACE_OPS_FL_IPMODIFY)) in referenced_filters()
6880 if (ops->flags & FTRACE_OPS_FL_SAVE_REGS) in referenced_filters()
6882 if (cnt == 1 && ops->trampoline) in referenced_filters()
6922 if (!tr->ops || !tr->ops->func_hash) in clear_mod_from_hashes()
6924 mutex_lock(&tr->ops->func_hash->regex_lock); in clear_mod_from_hashes()
6925 clear_mod_from_hash(pg, tr->ops->func_hash->filter_hash); in clear_mod_from_hashes()
6926 clear_mod_from_hash(pg, tr->ops->func_hash->notrace_hash); in clear_mod_from_hashes()
6927 mutex_unlock(&tr->ops->func_hash->regex_lock); in clear_mod_from_hashes()
7305 if (!tr->ops || !tr->ops->func_hash) in clear_func_from_hashes()
7307 mutex_lock(&tr->ops->func_hash->regex_lock); in clear_func_from_hashes()
7308 clear_func_from_hash(func, tr->ops->func_hash->filter_hash); in clear_func_from_hashes()
7309 clear_func_from_hash(func, tr->ops->func_hash->notrace_hash); in clear_func_from_hashes()
7310 mutex_unlock(&tr->ops->func_hash->regex_lock); in clear_func_from_hashes()
7460 void __weak arch_ftrace_update_trampoline(struct ftrace_ops *ops) in arch_ftrace_update_trampoline() argument
7464 static void ftrace_update_trampoline(struct ftrace_ops *ops) in ftrace_update_trampoline() argument
7466 unsigned long trampoline = ops->trampoline; in ftrace_update_trampoline()
7468 arch_ftrace_update_trampoline(ops); in ftrace_update_trampoline()
7469 if (ops->trampoline && ops->trampoline != trampoline && in ftrace_update_trampoline()
7470 (ops->flags & FTRACE_OPS_FL_ALLOC_TRAMP)) { in ftrace_update_trampoline()
7472 ftrace_add_trampoline_to_kallsyms(ops); in ftrace_update_trampoline()
7474 ops->trampoline, ops->trampoline_size, false, in ftrace_update_trampoline()
7480 perf_event_text_poke((void *)ops->trampoline, NULL, 0, in ftrace_update_trampoline()
7481 (void *)ops->trampoline, in ftrace_update_trampoline()
7482 ops->trampoline_size); in ftrace_update_trampoline()
7510 static void ftrace_update_trampoline(struct ftrace_ops *ops) in ftrace_update_trampoline() argument
7518 tr->ops = &global_ops; in ftrace_init_global_array_ops()
7519 tr->ops->private = tr; in ftrace_init_global_array_ops()
7527 if (WARN_ON(tr->ops->func != ftrace_stub)) in ftrace_init_array_ops()
7529 tr->ops->func); in ftrace_init_array_ops()
7531 tr->ops->func = func; in ftrace_init_array_ops()
7532 tr->ops->private = tr; in ftrace_init_array_ops()
7537 tr->ops->func = ftrace_stub; in ftrace_reset_array_ops()
7645 ftrace_func_t ftrace_ops_get_func(struct ftrace_ops *ops) in ftrace_ops_get_func() argument
7651 if (ops->flags & (FTRACE_OPS_FL_RECURSION | in ftrace_ops_get_func()
7655 return ops->func; in ftrace_ops_get_func()
8117 static int prepare_direct_functions_for_ipmodify(struct ftrace_ops *ops) in prepare_direct_functions_for_ipmodify() argument
8126 if (!(ops->flags & FTRACE_OPS_FL_IPMODIFY)) in prepare_direct_functions_for_ipmodify()
8129 hash = ops->func_hash->filter_hash; in prepare_direct_functions_for_ipmodify()
8166 static void cleanup_direct_functions_after_ipmodify(struct ftrace_ops *ops) in cleanup_direct_functions_after_ipmodify() argument
8173 if (!(ops->flags & FTRACE_OPS_FL_IPMODIFY)) in cleanup_direct_functions_after_ipmodify()
8178 hash = ops->func_hash->filter_hash; in cleanup_direct_functions_after_ipmodify()
8209 static int prepare_direct_functions_for_ipmodify(struct ftrace_ops *ops) in prepare_direct_functions_for_ipmodify() argument
8214 static void cleanup_direct_functions_after_ipmodify(struct ftrace_ops *ops) in cleanup_direct_functions_after_ipmodify() argument
8226 static int register_ftrace_function_nolock(struct ftrace_ops *ops) in register_ftrace_function_nolock() argument
8230 ftrace_ops_init(ops); in register_ftrace_function_nolock()
8234 ret = ftrace_startup(ops, 0); in register_ftrace_function_nolock()
8252 int register_ftrace_function(struct ftrace_ops *ops) in register_ftrace_function() argument
8257 ret = prepare_direct_functions_for_ipmodify(ops); in register_ftrace_function()
8261 ret = register_ftrace_function_nolock(ops); in register_ftrace_function()
8275 int unregister_ftrace_function(struct ftrace_ops *ops) in unregister_ftrace_function() argument
8280 ret = ftrace_shutdown(ops, 0); in unregister_ftrace_function()
8283 cleanup_direct_functions_after_ipmodify(ops); in unregister_ftrace_function()