1/* SPDX-License-Identifier: GPL-2.0-only */ 2/* 3 * arch/arm64/kernel/entry-ftrace.S 4 * 5 * Copyright (C) 2013 Linaro Limited 6 * Author: AKASHI Takahiro <takahiro.akashi@linaro.org> 7 */ 8 9#include <linux/linkage.h> 10#include <asm/asm-offsets.h> 11#include <asm/assembler.h> 12#include <asm/ftrace.h> 13#include <asm/insn.h> 14 15#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 16/* 17 * Due to -fpatchable-function-entry=2, the compiler has placed two NOPs before 18 * the regular function prologue. For an enabled callsite, ftrace_init_nop() and 19 * ftrace_make_call() have patched those NOPs to: 20 * 21 * MOV X9, LR 22 * BL <entry> 23 * 24 * ... where <entry> is either ftrace_caller or ftrace_regs_caller. 25 * 26 * Each instrumented function follows the AAPCS, so here x0-x8 and x18-x30 are 27 * live (x18 holds the Shadow Call Stack pointer), and x9-x17 are safe to 28 * clobber. 29 * 30 * We save the callsite's context into a pt_regs before invoking any ftrace 31 * callbacks. So that we can get a sensible backtrace, we create a stack record 32 * for the callsite and the ftrace entry assembly. This is not sufficient for 33 * reliable stacktrace: until we create the callsite stack record, its caller 34 * is missing from the LR and existing chain of frame records. 35 */ 36 .macro ftrace_regs_entry, allregs=0 37 /* Make room for pt_regs, plus a callee frame */ 38 sub sp, sp, #(PT_REGS_SIZE + 16) 39 40 /* Save function arguments (and x9 for simplicity) */ 41 stp x0, x1, [sp, #S_X0] 42 stp x2, x3, [sp, #S_X2] 43 stp x4, x5, [sp, #S_X4] 44 stp x6, x7, [sp, #S_X6] 45 stp x8, x9, [sp, #S_X8] 46 47 /* Optionally save the callee-saved registers, always save the FP */ 48 .if \allregs == 1 49 stp x10, x11, [sp, #S_X10] 50 stp x12, x13, [sp, #S_X12] 51 stp x14, x15, [sp, #S_X14] 52 stp x16, x17, [sp, #S_X16] 53 stp x18, x19, [sp, #S_X18] 54 stp x20, x21, [sp, #S_X20] 55 stp x22, x23, [sp, #S_X22] 56 stp x24, x25, [sp, #S_X24] 57 stp x26, x27, [sp, #S_X26] 58 stp x28, x29, [sp, #S_X28] 59 .else 60 str x29, [sp, #S_FP] 61 .endif 62 63 /* Save the callsite's SP and LR */ 64 add x10, sp, #(PT_REGS_SIZE + 16) 65 stp x9, x10, [sp, #S_LR] 66 67 /* Save the PC after the ftrace callsite */ 68 str x30, [sp, #S_PC] 69 70 /* Create a frame record for the callsite above pt_regs */ 71 stp x29, x9, [sp, #PT_REGS_SIZE] 72 add x29, sp, #PT_REGS_SIZE 73 74 /* Create our frame record within pt_regs. */ 75 stp x29, x30, [sp, #S_STACKFRAME] 76 add x29, sp, #S_STACKFRAME 77 .endm 78 79SYM_CODE_START(ftrace_regs_caller) 80#ifdef BTI_C 81 BTI_C 82#endif 83 ftrace_regs_entry 1 84 b ftrace_common 85SYM_CODE_END(ftrace_regs_caller) 86 87SYM_CODE_START(ftrace_caller) 88#ifdef BTI_C 89 BTI_C 90#endif 91 ftrace_regs_entry 0 92 b ftrace_common 93SYM_CODE_END(ftrace_caller) 94 95SYM_CODE_START(ftrace_common) 96 sub x0, x30, #AARCH64_INSN_SIZE // ip (callsite's BL insn) 97 mov x1, x9 // parent_ip (callsite's LR) 98 ldr_l x2, function_trace_op // op 99 mov x3, sp // regs 100 101SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL) 102 bl ftrace_stub 103 104#ifdef CONFIG_FUNCTION_GRAPH_TRACER 105SYM_INNER_LABEL(ftrace_graph_call, SYM_L_GLOBAL) // ftrace_graph_caller(); 106 nop // If enabled, this will be replaced 107 // "b ftrace_graph_caller" 108#endif 109 110/* 111 * At the callsite x0-x8 and x19-x30 were live. Any C code will have preserved 112 * x19-x29 per the AAPCS, and we created frame records upon entry, so we need 113 * to restore x0-x8, x29, and x30. 114 */ 115ftrace_common_return: 116 /* Restore function arguments */ 117 ldp x0, x1, [sp] 118 ldp x2, x3, [sp, #S_X2] 119 ldp x4, x5, [sp, #S_X4] 120 ldp x6, x7, [sp, #S_X6] 121 ldr x8, [sp, #S_X8] 122 123 /* Restore the callsite's FP, LR, PC */ 124 ldr x29, [sp, #S_FP] 125 ldr x30, [sp, #S_LR] 126 ldr x9, [sp, #S_PC] 127 128 /* Restore the callsite's SP */ 129 add sp, sp, #PT_REGS_SIZE + 16 130 131 ret x9 132SYM_CODE_END(ftrace_common) 133 134#ifdef CONFIG_FUNCTION_GRAPH_TRACER 135SYM_CODE_START(ftrace_graph_caller) 136 ldr x0, [sp, #S_PC] 137 sub x0, x0, #AARCH64_INSN_SIZE // ip (callsite's BL insn) 138 add x1, sp, #S_LR // parent_ip (callsite's LR) 139 ldr x2, [sp, #PT_REGS_SIZE] // parent fp (callsite's FP) 140 bl prepare_ftrace_return 141 b ftrace_common_return 142SYM_CODE_END(ftrace_graph_caller) 143#endif 144 145#else /* CONFIG_DYNAMIC_FTRACE_WITH_REGS */ 146 147/* 148 * Gcc with -pg will put the following code in the beginning of each function: 149 * mov x0, x30 150 * bl _mcount 151 * [function's body ...] 152 * "bl _mcount" may be replaced to "bl ftrace_caller" or NOP if dynamic 153 * ftrace is enabled. 154 * 155 * Please note that x0 as an argument will not be used here because we can 156 * get lr(x30) of instrumented function at any time by winding up call stack 157 * as long as the kernel is compiled without -fomit-frame-pointer. 158 * (or CONFIG_FRAME_POINTER, this is forced on arm64) 159 * 160 * stack layout after mcount_enter in _mcount(): 161 * 162 * current sp/fp => 0:+-----+ 163 * in _mcount() | x29 | -> instrumented function's fp 164 * +-----+ 165 * | x30 | -> _mcount()'s lr (= instrumented function's pc) 166 * old sp => +16:+-----+ 167 * when instrumented | | 168 * function calls | ... | 169 * _mcount() | | 170 * | | 171 * instrumented => +xx:+-----+ 172 * function's fp | x29 | -> parent's fp 173 * +-----+ 174 * | x30 | -> instrumented function's lr (= parent's pc) 175 * +-----+ 176 * | ... | 177 */ 178 179 .macro mcount_enter 180 stp x29, x30, [sp, #-16]! 181 mov x29, sp 182 .endm 183 184 .macro mcount_exit 185 ldp x29, x30, [sp], #16 186 ret 187 .endm 188 189 .macro mcount_adjust_addr rd, rn 190 sub \rd, \rn, #AARCH64_INSN_SIZE 191 .endm 192 193 /* for instrumented function's parent */ 194 .macro mcount_get_parent_fp reg 195 ldr \reg, [x29] 196 ldr \reg, [\reg] 197 .endm 198 199 /* for instrumented function */ 200 .macro mcount_get_pc0 reg 201 mcount_adjust_addr \reg, x30 202 .endm 203 204 .macro mcount_get_pc reg 205 ldr \reg, [x29, #8] 206 mcount_adjust_addr \reg, \reg 207 .endm 208 209 .macro mcount_get_lr reg 210 ldr \reg, [x29] 211 ldr \reg, [\reg, #8] 212 .endm 213 214 .macro mcount_get_lr_addr reg 215 ldr \reg, [x29] 216 add \reg, \reg, #8 217 .endm 218 219#ifndef CONFIG_DYNAMIC_FTRACE 220/* 221 * void _mcount(unsigned long return_address) 222 * @return_address: return address to instrumented function 223 * 224 * This function makes calls, if enabled, to: 225 * - tracer function to probe instrumented function's entry, 226 * - ftrace_graph_caller to set up an exit hook 227 */ 228SYM_FUNC_START(_mcount) 229 mcount_enter 230 231 ldr_l x2, ftrace_trace_function 232 adr x0, ftrace_stub 233 cmp x0, x2 // if (ftrace_trace_function 234 b.eq skip_ftrace_call // != ftrace_stub) { 235 236 mcount_get_pc x0 // function's pc 237 mcount_get_lr x1 // function's lr (= parent's pc) 238 blr x2 // (*ftrace_trace_function)(pc, lr); 239 240skip_ftrace_call: // } 241#ifdef CONFIG_FUNCTION_GRAPH_TRACER 242 ldr_l x2, ftrace_graph_return 243 cmp x0, x2 // if ((ftrace_graph_return 244 b.ne ftrace_graph_caller // != ftrace_stub) 245 246 ldr_l x2, ftrace_graph_entry // || (ftrace_graph_entry 247 adr_l x0, ftrace_graph_entry_stub // != ftrace_graph_entry_stub)) 248 cmp x0, x2 249 b.ne ftrace_graph_caller // ftrace_graph_caller(); 250#endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 251 mcount_exit 252SYM_FUNC_END(_mcount) 253EXPORT_SYMBOL(_mcount) 254NOKPROBE(_mcount) 255 256#else /* CONFIG_DYNAMIC_FTRACE */ 257/* 258 * _mcount() is used to build the kernel with -pg option, but all the branch 259 * instructions to _mcount() are replaced to NOP initially at kernel start up, 260 * and later on, NOP to branch to ftrace_caller() when enabled or branch to 261 * NOP when disabled per-function base. 262 */ 263SYM_FUNC_START(_mcount) 264 ret 265SYM_FUNC_END(_mcount) 266EXPORT_SYMBOL(_mcount) 267NOKPROBE(_mcount) 268 269/* 270 * void ftrace_caller(unsigned long return_address) 271 * @return_address: return address to instrumented function 272 * 273 * This function is a counterpart of _mcount() in 'static' ftrace, and 274 * makes calls to: 275 * - tracer function to probe instrumented function's entry, 276 * - ftrace_graph_caller to set up an exit hook 277 */ 278SYM_FUNC_START(ftrace_caller) 279 mcount_enter 280 281 mcount_get_pc0 x0 // function's pc 282 mcount_get_lr x1 // function's lr 283 284SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL) // tracer(pc, lr); 285 nop // This will be replaced with "bl xxx" 286 // where xxx can be any kind of tracer. 287 288#ifdef CONFIG_FUNCTION_GRAPH_TRACER 289SYM_INNER_LABEL(ftrace_graph_call, SYM_L_GLOBAL) // ftrace_graph_caller(); 290 nop // If enabled, this will be replaced 291 // "b ftrace_graph_caller" 292#endif 293 294 mcount_exit 295SYM_FUNC_END(ftrace_caller) 296#endif /* CONFIG_DYNAMIC_FTRACE */ 297 298#ifdef CONFIG_FUNCTION_GRAPH_TRACER 299/* 300 * void ftrace_graph_caller(void) 301 * 302 * Called from _mcount() or ftrace_caller() when function_graph tracer is 303 * selected. 304 * This function w/ prepare_ftrace_return() fakes link register's value on 305 * the call stack in order to intercept instrumented function's return path 306 * and run return_to_handler() later on its exit. 307 */ 308SYM_FUNC_START(ftrace_graph_caller) 309 mcount_get_pc x0 // function's pc 310 mcount_get_lr_addr x1 // pointer to function's saved lr 311 mcount_get_parent_fp x2 // parent's fp 312 bl prepare_ftrace_return // prepare_ftrace_return(pc, &lr, fp) 313 314 mcount_exit 315SYM_FUNC_END(ftrace_graph_caller) 316#endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 317#endif /* CONFIG_DYNAMIC_FTRACE_WITH_REGS */ 318 319SYM_FUNC_START(ftrace_stub) 320 ret 321SYM_FUNC_END(ftrace_stub) 322 323#ifdef CONFIG_FUNCTION_GRAPH_TRACER 324/* 325 * void return_to_handler(void) 326 * 327 * Run ftrace_return_to_handler() before going back to parent. 328 * @fp is checked against the value passed by ftrace_graph_caller(). 329 */ 330SYM_CODE_START(return_to_handler) 331 /* save return value regs */ 332 sub sp, sp, #64 333 stp x0, x1, [sp] 334 stp x2, x3, [sp, #16] 335 stp x4, x5, [sp, #32] 336 stp x6, x7, [sp, #48] 337 338 mov x0, x29 // parent's fp 339 bl ftrace_return_to_handler// addr = ftrace_return_to_hander(fp); 340 mov x30, x0 // restore the original return address 341 342 /* restore return value regs */ 343 ldp x0, x1, [sp] 344 ldp x2, x3, [sp, #16] 345 ldp x4, x5, [sp, #32] 346 ldp x6, x7, [sp, #48] 347 add sp, sp, #64 348 349 ret 350SYM_CODE_END(return_to_handler) 351#endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 352