1 // SPDX-License-Identifier: GPL-2.0 2 3 #include <linux/ftrace.h> 4 #include <linux/uaccess.h> 5 #include <asm/cacheflush.h> 6 7 #ifndef CONFIG_DYNAMIC_FTRACE 8 extern void (*ftrace_trace_function)(unsigned long, unsigned long, 9 struct ftrace_ops*, struct pt_regs*); 10 extern void ftrace_graph_caller(void); 11 ftrace_stub(unsigned long ip,unsigned long parent_ip,struct ftrace_ops * op,struct pt_regs * regs)12 noinline void __naked ftrace_stub(unsigned long ip, unsigned long parent_ip, 13 struct ftrace_ops *op, struct pt_regs *regs) 14 { 15 __asm__ (""); /* avoid to optimize as pure function */ 16 } 17 _mcount(unsigned long parent_ip)18 noinline void _mcount(unsigned long parent_ip) 19 { 20 /* save all state by the compiler prologue */ 21 22 unsigned long ip = (unsigned long)__builtin_return_address(0); 23 24 if (ftrace_trace_function != ftrace_stub) 25 ftrace_trace_function(ip - MCOUNT_INSN_SIZE, parent_ip, 26 NULL, NULL); 27 28 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 29 if (ftrace_graph_return != (trace_func_graph_ret_t)ftrace_stub 30 || ftrace_graph_entry != ftrace_graph_entry_stub) 31 ftrace_graph_caller(); 32 #endif 33 34 /* restore all state by the compiler epilogue */ 35 } 36 EXPORT_SYMBOL(_mcount); 37 38 #else /* CONFIG_DYNAMIC_FTRACE */ 39 ftrace_stub(unsigned long ip,unsigned long parent_ip,struct ftrace_ops * op,struct pt_regs * regs)40 noinline void __naked ftrace_stub(unsigned long ip, unsigned long parent_ip, 41 struct ftrace_ops *op, struct pt_regs *regs) 42 { 43 __asm__ (""); /* avoid to optimize as pure function */ 44 } 45 _mcount(unsigned long parent_ip)46 noinline void __naked _mcount(unsigned long parent_ip) 47 { 48 __asm__ (""); /* avoid to optimize as pure function */ 49 } 50 EXPORT_SYMBOL(_mcount); 51 52 #define XSTR(s) STR(s) 53 #define STR(s) #s _ftrace_caller(unsigned long parent_ip)54 void _ftrace_caller(unsigned long parent_ip) 55 { 56 /* save all state needed by the compiler prologue */ 57 58 /* 59 * prepare arguments for real tracing function 60 * first arg : __builtin_return_address(0) - MCOUNT_INSN_SIZE 61 * second arg : parent_ip 62 */ 63 __asm__ __volatile__ ( 64 "move $r1, %0 \n\t" 65 "addi $r0, %1, #-" XSTR(MCOUNT_INSN_SIZE) "\n\t" 66 : 67 : "r" (parent_ip), "r" (__builtin_return_address(0))); 68 69 /* a placeholder for the call to a real tracing function */ 70 __asm__ __volatile__ ( 71 "ftrace_call: \n\t" 72 "nop \n\t" 73 "nop \n\t" 74 "nop \n\t"); 75 76 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 77 /* a placeholder for the call to ftrace_graph_caller */ 78 __asm__ __volatile__ ( 79 "ftrace_graph_call: \n\t" 80 "nop \n\t" 81 "nop \n\t" 82 "nop \n\t"); 83 #endif 84 /* restore all state needed by the compiler epilogue */ 85 } 86 ftrace_dyn_arch_init(void)87 int __init ftrace_dyn_arch_init(void) 88 { 89 return 0; 90 } 91 ftrace_arch_code_modify_prepare(void)92 int ftrace_arch_code_modify_prepare(void) 93 { 94 set_all_modules_text_rw(); 95 return 0; 96 } 97 ftrace_arch_code_modify_post_process(void)98 int ftrace_arch_code_modify_post_process(void) 99 { 100 set_all_modules_text_ro(); 101 return 0; 102 } 103 gen_sethi_insn(unsigned long addr)104 static unsigned long gen_sethi_insn(unsigned long addr) 105 { 106 unsigned long opcode = 0x46000000; 107 unsigned long imm = addr >> 12; 108 unsigned long rt_num = 0xf << 20; 109 110 return ENDIAN_CONVERT(opcode | rt_num | imm); 111 } 112 gen_ori_insn(unsigned long addr)113 static unsigned long gen_ori_insn(unsigned long addr) 114 { 115 unsigned long opcode = 0x58000000; 116 unsigned long imm = addr & 0x0000fff; 117 unsigned long rt_num = 0xf << 20; 118 unsigned long ra_num = 0xf << 15; 119 120 return ENDIAN_CONVERT(opcode | rt_num | ra_num | imm); 121 } 122 gen_jral_insn(unsigned long addr)123 static unsigned long gen_jral_insn(unsigned long addr) 124 { 125 unsigned long opcode = 0x4a000001; 126 unsigned long rt_num = 0x1e << 20; 127 unsigned long rb_num = 0xf << 10; 128 129 return ENDIAN_CONVERT(opcode | rt_num | rb_num); 130 } 131 ftrace_gen_call_insn(unsigned long * call_insns,unsigned long addr)132 static void ftrace_gen_call_insn(unsigned long *call_insns, 133 unsigned long addr) 134 { 135 call_insns[0] = gen_sethi_insn(addr); /* sethi $r15, imm20u */ 136 call_insns[1] = gen_ori_insn(addr); /* ori $r15, $r15, imm15u */ 137 call_insns[2] = gen_jral_insn(addr); /* jral $lp, $r15 */ 138 } 139 __ftrace_modify_code(unsigned long pc,unsigned long * old_insn,unsigned long * new_insn,bool validate)140 static int __ftrace_modify_code(unsigned long pc, unsigned long *old_insn, 141 unsigned long *new_insn, bool validate) 142 { 143 unsigned long orig_insn[3]; 144 145 if (validate) { 146 if (probe_kernel_read(orig_insn, (void *)pc, MCOUNT_INSN_SIZE)) 147 return -EFAULT; 148 if (memcmp(orig_insn, old_insn, MCOUNT_INSN_SIZE)) 149 return -EINVAL; 150 } 151 152 if (probe_kernel_write((void *)pc, new_insn, MCOUNT_INSN_SIZE)) 153 return -EPERM; 154 155 return 0; 156 } 157 ftrace_modify_code(unsigned long pc,unsigned long * old_insn,unsigned long * new_insn,bool validate)158 static int ftrace_modify_code(unsigned long pc, unsigned long *old_insn, 159 unsigned long *new_insn, bool validate) 160 { 161 int ret; 162 163 ret = __ftrace_modify_code(pc, old_insn, new_insn, validate); 164 if (ret) 165 return ret; 166 167 flush_icache_range(pc, pc + MCOUNT_INSN_SIZE); 168 169 return ret; 170 } 171 ftrace_update_ftrace_func(ftrace_func_t func)172 int ftrace_update_ftrace_func(ftrace_func_t func) 173 { 174 unsigned long pc = (unsigned long)&ftrace_call; 175 unsigned long old_insn[3] = {INSN_NOP, INSN_NOP, INSN_NOP}; 176 unsigned long new_insn[3] = {INSN_NOP, INSN_NOP, INSN_NOP}; 177 178 if (func != ftrace_stub) 179 ftrace_gen_call_insn(new_insn, (unsigned long)func); 180 181 return ftrace_modify_code(pc, old_insn, new_insn, false); 182 } 183 ftrace_make_call(struct dyn_ftrace * rec,unsigned long addr)184 int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) 185 { 186 unsigned long pc = rec->ip; 187 unsigned long nop_insn[3] = {INSN_NOP, INSN_NOP, INSN_NOP}; 188 unsigned long call_insn[3] = {INSN_NOP, INSN_NOP, INSN_NOP}; 189 190 ftrace_gen_call_insn(call_insn, addr); 191 192 return ftrace_modify_code(pc, nop_insn, call_insn, true); 193 } 194 ftrace_make_nop(struct module * mod,struct dyn_ftrace * rec,unsigned long addr)195 int ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec, 196 unsigned long addr) 197 { 198 unsigned long pc = rec->ip; 199 unsigned long nop_insn[3] = {INSN_NOP, INSN_NOP, INSN_NOP}; 200 unsigned long call_insn[3] = {INSN_NOP, INSN_NOP, INSN_NOP}; 201 202 ftrace_gen_call_insn(call_insn, addr); 203 204 return ftrace_modify_code(pc, call_insn, nop_insn, true); 205 } 206 #endif /* CONFIG_DYNAMIC_FTRACE */ 207 208 #ifdef CONFIG_FUNCTION_GRAPH_TRACER prepare_ftrace_return(unsigned long * parent,unsigned long self_addr,unsigned long frame_pointer)209 void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr, 210 unsigned long frame_pointer) 211 { 212 unsigned long return_hooker = (unsigned long)&return_to_handler; 213 unsigned long old; 214 215 if (unlikely(atomic_read(¤t->tracing_graph_pause))) 216 return; 217 218 old = *parent; 219 220 if (!function_graph_enter(old, self_addr, frame_pointer, NULL)) 221 *parent = return_hooker; 222 } 223 ftrace_graph_caller(void)224 noinline void ftrace_graph_caller(void) 225 { 226 unsigned long *parent_ip = 227 (unsigned long *)(__builtin_frame_address(2) - 4); 228 229 unsigned long selfpc = 230 (unsigned long)(__builtin_return_address(1) - MCOUNT_INSN_SIZE); 231 232 unsigned long frame_pointer = 233 (unsigned long)__builtin_frame_address(3); 234 235 prepare_ftrace_return(parent_ip, selfpc, frame_pointer); 236 } 237 238 extern unsigned long ftrace_return_to_handler(unsigned long frame_pointer); return_to_handler(void)239 void __naked return_to_handler(void) 240 { 241 __asm__ __volatile__ ( 242 /* save state needed by the ABI */ 243 "smw.adm $r0,[$sp],$r1,#0x0 \n\t" 244 245 /* get original return address */ 246 "move $r0, $fp \n\t" 247 "bal ftrace_return_to_handler\n\t" 248 "move $lp, $r0 \n\t" 249 250 /* restore state nedded by the ABI */ 251 "lmw.bim $r0,[$sp],$r1,#0x0 \n\t"); 252 } 253 254 #ifdef CONFIG_DYNAMIC_FTRACE 255 extern unsigned long ftrace_graph_call; 256 ftrace_modify_graph_caller(bool enable)257 static int ftrace_modify_graph_caller(bool enable) 258 { 259 unsigned long pc = (unsigned long)&ftrace_graph_call; 260 unsigned long nop_insn[3] = {INSN_NOP, INSN_NOP, INSN_NOP}; 261 unsigned long call_insn[3] = {INSN_NOP, INSN_NOP, INSN_NOP}; 262 263 ftrace_gen_call_insn(call_insn, (unsigned long)ftrace_graph_caller); 264 265 if (enable) 266 return ftrace_modify_code(pc, nop_insn, call_insn, true); 267 else 268 return ftrace_modify_code(pc, call_insn, nop_insn, true); 269 } 270 ftrace_enable_ftrace_graph_caller(void)271 int ftrace_enable_ftrace_graph_caller(void) 272 { 273 return ftrace_modify_graph_caller(true); 274 } 275 ftrace_disable_ftrace_graph_caller(void)276 int ftrace_disable_ftrace_graph_caller(void) 277 { 278 return ftrace_modify_graph_caller(false); 279 } 280 #endif /* CONFIG_DYNAMIC_FTRACE */ 281 282 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 283 284 285 #ifdef CONFIG_TRACE_IRQFLAGS __trace_hardirqs_off(void)286 noinline void __trace_hardirqs_off(void) 287 { 288 trace_hardirqs_off(); 289 } __trace_hardirqs_on(void)290 noinline void __trace_hardirqs_on(void) 291 { 292 trace_hardirqs_on(); 293 } 294 #endif /* CONFIG_TRACE_IRQFLAGS */ 295