• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1  // SPDX-License-Identifier: GPL-2.0
2  
3  #include <linux/ftrace.h>
4  #include <linux/uaccess.h>
5  #include <asm/cacheflush.h>
6  
7  #ifndef CONFIG_DYNAMIC_FTRACE
8  extern void (*ftrace_trace_function)(unsigned long, unsigned long,
9  				     struct ftrace_ops*, struct pt_regs*);
10  extern void ftrace_graph_caller(void);
11  
ftrace_stub(unsigned long ip,unsigned long parent_ip,struct ftrace_ops * op,struct pt_regs * regs)12  noinline void __naked ftrace_stub(unsigned long ip, unsigned long parent_ip,
13  				  struct ftrace_ops *op, struct pt_regs *regs)
14  {
15  	__asm__ ("");  /* avoid to optimize as pure function */
16  }
17  
_mcount(unsigned long parent_ip)18  noinline void _mcount(unsigned long parent_ip)
19  {
20  	/* save all state by the compiler prologue */
21  
22  	unsigned long ip = (unsigned long)__builtin_return_address(0);
23  
24  	if (ftrace_trace_function != ftrace_stub)
25  		ftrace_trace_function(ip - MCOUNT_INSN_SIZE, parent_ip,
26  				      NULL, NULL);
27  
28  #ifdef CONFIG_FUNCTION_GRAPH_TRACER
29  	if (ftrace_graph_return != (trace_func_graph_ret_t)ftrace_stub
30  	    || ftrace_graph_entry != ftrace_graph_entry_stub)
31  		ftrace_graph_caller();
32  #endif
33  
34  	/* restore all state by the compiler epilogue */
35  }
36  EXPORT_SYMBOL(_mcount);
37  
38  #else /* CONFIG_DYNAMIC_FTRACE */
39  
ftrace_stub(unsigned long ip,unsigned long parent_ip,struct ftrace_ops * op,struct pt_regs * regs)40  noinline void __naked ftrace_stub(unsigned long ip, unsigned long parent_ip,
41  				  struct ftrace_ops *op, struct pt_regs *regs)
42  {
43  	__asm__ ("");  /* avoid to optimize as pure function */
44  }
45  
_mcount(unsigned long parent_ip)46  noinline void __naked _mcount(unsigned long parent_ip)
47  {
48  	__asm__ ("");  /* avoid to optimize as pure function */
49  }
50  EXPORT_SYMBOL(_mcount);
51  
52  #define XSTR(s) STR(s)
53  #define STR(s) #s
_ftrace_caller(unsigned long parent_ip)54  void _ftrace_caller(unsigned long parent_ip)
55  {
56  	/* save all state needed by the compiler prologue */
57  
58  	/*
59  	 * prepare arguments for real tracing function
60  	 * first  arg : __builtin_return_address(0) - MCOUNT_INSN_SIZE
61  	 * second arg : parent_ip
62  	 */
63  	__asm__ __volatile__ (
64  		"move $r1, %0				   \n\t"
65  		"addi $r0, %1, #-" XSTR(MCOUNT_INSN_SIZE) "\n\t"
66  		:
67  		: "r" (parent_ip), "r" (__builtin_return_address(0)));
68  
69  	/* a placeholder for the call to a real tracing function */
70  	__asm__ __volatile__ (
71  		"ftrace_call:		\n\t"
72  		"nop			\n\t"
73  		"nop			\n\t"
74  		"nop			\n\t");
75  
76  #ifdef CONFIG_FUNCTION_GRAPH_TRACER
77  	/* a placeholder for the call to ftrace_graph_caller */
78  	__asm__ __volatile__ (
79  		"ftrace_graph_call:	\n\t"
80  		"nop			\n\t"
81  		"nop			\n\t"
82  		"nop			\n\t");
83  #endif
84  	/* restore all state needed by the compiler epilogue */
85  }
86  
ftrace_dyn_arch_init(void)87  int __init ftrace_dyn_arch_init(void)
88  {
89  	return 0;
90  }
91  
ftrace_arch_code_modify_prepare(void)92  int ftrace_arch_code_modify_prepare(void)
93  {
94  	set_all_modules_text_rw();
95  	return 0;
96  }
97  
ftrace_arch_code_modify_post_process(void)98  int ftrace_arch_code_modify_post_process(void)
99  {
100  	set_all_modules_text_ro();
101  	return 0;
102  }
103  
gen_sethi_insn(unsigned long addr)104  static unsigned long gen_sethi_insn(unsigned long addr)
105  {
106  	unsigned long opcode = 0x46000000;
107  	unsigned long imm = addr >> 12;
108  	unsigned long rt_num = 0xf << 20;
109  
110  	return ENDIAN_CONVERT(opcode | rt_num | imm);
111  }
112  
gen_ori_insn(unsigned long addr)113  static unsigned long gen_ori_insn(unsigned long addr)
114  {
115  	unsigned long opcode = 0x58000000;
116  	unsigned long imm = addr & 0x0000fff;
117  	unsigned long rt_num = 0xf << 20;
118  	unsigned long ra_num = 0xf << 15;
119  
120  	return ENDIAN_CONVERT(opcode | rt_num | ra_num | imm);
121  }
122  
gen_jral_insn(unsigned long addr)123  static unsigned long gen_jral_insn(unsigned long addr)
124  {
125  	unsigned long opcode = 0x4a000001;
126  	unsigned long rt_num = 0x1e << 20;
127  	unsigned long rb_num = 0xf << 10;
128  
129  	return ENDIAN_CONVERT(opcode | rt_num | rb_num);
130  }
131  
ftrace_gen_call_insn(unsigned long * call_insns,unsigned long addr)132  static void ftrace_gen_call_insn(unsigned long *call_insns,
133  				 unsigned long addr)
134  {
135  	call_insns[0] = gen_sethi_insn(addr); /* sethi $r15, imm20u       */
136  	call_insns[1] = gen_ori_insn(addr);   /* ori   $r15, $r15, imm15u */
137  	call_insns[2] = gen_jral_insn(addr);  /* jral  $lp,  $r15         */
138  }
139  
__ftrace_modify_code(unsigned long pc,unsigned long * old_insn,unsigned long * new_insn,bool validate)140  static int __ftrace_modify_code(unsigned long pc, unsigned long *old_insn,
141  				unsigned long *new_insn, bool validate)
142  {
143  	unsigned long orig_insn[3];
144  
145  	if (validate) {
146  		if (probe_kernel_read(orig_insn, (void *)pc, MCOUNT_INSN_SIZE))
147  			return -EFAULT;
148  		if (memcmp(orig_insn, old_insn, MCOUNT_INSN_SIZE))
149  			return -EINVAL;
150  	}
151  
152  	if (probe_kernel_write((void *)pc, new_insn, MCOUNT_INSN_SIZE))
153  		return -EPERM;
154  
155  	return 0;
156  }
157  
ftrace_modify_code(unsigned long pc,unsigned long * old_insn,unsigned long * new_insn,bool validate)158  static int ftrace_modify_code(unsigned long pc, unsigned long *old_insn,
159  			      unsigned long *new_insn, bool validate)
160  {
161  	int ret;
162  
163  	ret = __ftrace_modify_code(pc, old_insn, new_insn, validate);
164  	if (ret)
165  		return ret;
166  
167  	flush_icache_range(pc, pc + MCOUNT_INSN_SIZE);
168  
169  	return ret;
170  }
171  
ftrace_update_ftrace_func(ftrace_func_t func)172  int ftrace_update_ftrace_func(ftrace_func_t func)
173  {
174  	unsigned long pc = (unsigned long)&ftrace_call;
175  	unsigned long old_insn[3] = {INSN_NOP, INSN_NOP, INSN_NOP};
176  	unsigned long new_insn[3] = {INSN_NOP, INSN_NOP, INSN_NOP};
177  
178  	if (func != ftrace_stub)
179  		ftrace_gen_call_insn(new_insn, (unsigned long)func);
180  
181  	return ftrace_modify_code(pc, old_insn, new_insn, false);
182  }
183  
ftrace_make_call(struct dyn_ftrace * rec,unsigned long addr)184  int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
185  {
186  	unsigned long pc = rec->ip;
187  	unsigned long nop_insn[3] = {INSN_NOP, INSN_NOP, INSN_NOP};
188  	unsigned long call_insn[3] = {INSN_NOP, INSN_NOP, INSN_NOP};
189  
190  	ftrace_gen_call_insn(call_insn, addr);
191  
192  	return ftrace_modify_code(pc, nop_insn, call_insn, true);
193  }
194  
ftrace_make_nop(struct module * mod,struct dyn_ftrace * rec,unsigned long addr)195  int ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec,
196  		    unsigned long addr)
197  {
198  	unsigned long pc = rec->ip;
199  	unsigned long nop_insn[3] = {INSN_NOP, INSN_NOP, INSN_NOP};
200  	unsigned long call_insn[3] = {INSN_NOP, INSN_NOP, INSN_NOP};
201  
202  	ftrace_gen_call_insn(call_insn, addr);
203  
204  	return ftrace_modify_code(pc, call_insn, nop_insn, true);
205  }
206  #endif /* CONFIG_DYNAMIC_FTRACE */
207  
208  #ifdef CONFIG_FUNCTION_GRAPH_TRACER
prepare_ftrace_return(unsigned long * parent,unsigned long self_addr,unsigned long frame_pointer)209  void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr,
210  			   unsigned long frame_pointer)
211  {
212  	unsigned long return_hooker = (unsigned long)&return_to_handler;
213  	unsigned long old;
214  
215  	if (unlikely(atomic_read(&current->tracing_graph_pause)))
216  		return;
217  
218  	old = *parent;
219  
220  	if (!function_graph_enter(old, self_addr, frame_pointer, NULL))
221  		*parent = return_hooker;
222  }
223  
ftrace_graph_caller(void)224  noinline void ftrace_graph_caller(void)
225  {
226  	unsigned long *parent_ip =
227  		(unsigned long *)(__builtin_frame_address(2) - 4);
228  
229  	unsigned long selfpc =
230  		(unsigned long)(__builtin_return_address(1) - MCOUNT_INSN_SIZE);
231  
232  	unsigned long frame_pointer =
233  		(unsigned long)__builtin_frame_address(3);
234  
235  	prepare_ftrace_return(parent_ip, selfpc, frame_pointer);
236  }
237  
238  extern unsigned long ftrace_return_to_handler(unsigned long frame_pointer);
return_to_handler(void)239  void __naked return_to_handler(void)
240  {
241  	__asm__ __volatile__ (
242  		/* save state needed by the ABI     */
243  		"smw.adm $r0,[$sp],$r1,#0x0  \n\t"
244  
245  		/* get original return address      */
246  		"move $r0, $fp               \n\t"
247  		"bal ftrace_return_to_handler\n\t"
248  		"move $lp, $r0               \n\t"
249  
250  		/* restore state nedded by the ABI  */
251  		"lmw.bim $r0,[$sp],$r1,#0x0  \n\t");
252  }
253  
254  #ifdef CONFIG_DYNAMIC_FTRACE
255  extern unsigned long ftrace_graph_call;
256  
ftrace_modify_graph_caller(bool enable)257  static int ftrace_modify_graph_caller(bool enable)
258  {
259  	unsigned long pc = (unsigned long)&ftrace_graph_call;
260  	unsigned long nop_insn[3] = {INSN_NOP, INSN_NOP, INSN_NOP};
261  	unsigned long call_insn[3] = {INSN_NOP, INSN_NOP, INSN_NOP};
262  
263  	ftrace_gen_call_insn(call_insn, (unsigned long)ftrace_graph_caller);
264  
265  	if (enable)
266  		return ftrace_modify_code(pc, nop_insn, call_insn, true);
267  	else
268  		return ftrace_modify_code(pc, call_insn, nop_insn, true);
269  }
270  
ftrace_enable_ftrace_graph_caller(void)271  int ftrace_enable_ftrace_graph_caller(void)
272  {
273  	return ftrace_modify_graph_caller(true);
274  }
275  
ftrace_disable_ftrace_graph_caller(void)276  int ftrace_disable_ftrace_graph_caller(void)
277  {
278  	return ftrace_modify_graph_caller(false);
279  }
280  #endif /* CONFIG_DYNAMIC_FTRACE */
281  
282  #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
283  
284  
285  #ifdef CONFIG_TRACE_IRQFLAGS
__trace_hardirqs_off(void)286  noinline void __trace_hardirqs_off(void)
287  {
288  	trace_hardirqs_off();
289  }
__trace_hardirqs_on(void)290  noinline void __trace_hardirqs_on(void)
291  {
292  	trace_hardirqs_on();
293  }
294  #endif /* CONFIG_TRACE_IRQFLAGS */
295