1/* 2 * arch/arm64/kernel/entry-ftrace.S 3 * 4 * Copyright (C) 2013 Linaro Limited 5 * Author: AKASHI Takahiro <takahiro.akashi@linaro.org> 6 * 7 * This program is free software; you can redistribute it and/or modify 8 * it under the terms of the GNU General Public License version 2 as 9 * published by the Free Software Foundation. 10 */ 11 12#include <linux/linkage.h> 13#include <asm/ftrace.h> 14#include <asm/insn.h> 15 16/* 17 * Gcc with -pg will put the following code in the beginning of each function: 18 * mov x0, x30 19 * bl _mcount 20 * [function's body ...] 21 * "bl _mcount" may be replaced to "bl ftrace_caller" or NOP if dynamic 22 * ftrace is enabled. 23 * 24 * Please note that x0 as an argument will not be used here because we can 25 * get lr(x30) of instrumented function at any time by winding up call stack 26 * as long as the kernel is compiled without -fomit-frame-pointer. 27 * (or CONFIG_FRAME_POINTER, this is forced on arm64) 28 * 29 * stack layout after mcount_enter in _mcount(): 30 * 31 * current sp/fp => 0:+-----+ 32 * in _mcount() | x29 | -> instrumented function's fp 33 * +-----+ 34 * | x30 | -> _mcount()'s lr (= instrumented function's pc) 35 * old sp => +16:+-----+ 36 * when instrumented | | 37 * function calls | ... | 38 * _mcount() | | 39 * | | 40 * instrumented => +xx:+-----+ 41 * function's fp | x29 | -> parent's fp 42 * +-----+ 43 * | x30 | -> instrumented function's lr (= parent's pc) 44 * +-----+ 45 * | ... | 46 */ 47 48 .macro mcount_enter 49 stp x29, x30, [sp, #-16]! 50 mov x29, sp 51 .endm 52 53 .macro mcount_exit 54 ldp x29, x30, [sp], #16 55 ret 56 .endm 57 58 .macro mcount_adjust_addr rd, rn 59 sub \rd, \rn, #AARCH64_INSN_SIZE 60 .endm 61 62 /* for instrumented function's parent */ 63 .macro mcount_get_parent_fp reg 64 ldr \reg, [x29] 65 ldr \reg, [\reg] 66 .endm 67 68 /* for instrumented function */ 69 .macro mcount_get_pc0 reg 70 mcount_adjust_addr \reg, x30 71 .endm 72 73 .macro mcount_get_pc reg 74 ldr \reg, [x29, #8] 75 mcount_adjust_addr \reg, \reg 76 .endm 77 78 .macro mcount_get_lr reg 79 ldr \reg, [x29] 80 ldr \reg, [\reg, #8] 81 .endm 82 83 .macro mcount_get_lr_addr reg 84 ldr \reg, [x29] 85 add \reg, \reg, #8 86 .endm 87 88#ifndef CONFIG_DYNAMIC_FTRACE 89/* 90 * void _mcount(unsigned long return_address) 91 * @return_address: return address to instrumented function 92 * 93 * This function makes calls, if enabled, to: 94 * - tracer function to probe instrumented function's entry, 95 * - ftrace_graph_caller to set up an exit hook 96 */ 97ENTRY(_mcount) 98 mcount_enter 99 100 adrp x0, ftrace_trace_function 101 ldr x2, [x0, #:lo12:ftrace_trace_function] 102 adr x0, ftrace_stub 103 cmp x0, x2 // if (ftrace_trace_function 104 b.eq skip_ftrace_call // != ftrace_stub) { 105 106 mcount_get_pc x0 // function's pc 107 mcount_get_lr x1 // function's lr (= parent's pc) 108 blr x2 // (*ftrace_trace_function)(pc, lr); 109 110#ifndef CONFIG_FUNCTION_GRAPH_TRACER 111skip_ftrace_call: // return; 112 mcount_exit // } 113#else 114 mcount_exit // return; 115 // } 116skip_ftrace_call: 117 adrp x1, ftrace_graph_return 118 ldr x2, [x1, #:lo12:ftrace_graph_return] 119 cmp x0, x2 // if ((ftrace_graph_return 120 b.ne ftrace_graph_caller // != ftrace_stub) 121 122 adrp x1, ftrace_graph_entry // || (ftrace_graph_entry 123 adrp x0, ftrace_graph_entry_stub // != ftrace_graph_entry_stub)) 124 ldr x2, [x1, #:lo12:ftrace_graph_entry] 125 add x0, x0, #:lo12:ftrace_graph_entry_stub 126 cmp x0, x2 127 b.ne ftrace_graph_caller // ftrace_graph_caller(); 128 129 mcount_exit 130#endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 131ENDPROC(_mcount) 132 133#else /* CONFIG_DYNAMIC_FTRACE */ 134/* 135 * _mcount() is used to build the kernel with -pg option, but all the branch 136 * instructions to _mcount() are replaced to NOP initially at kernel start up, 137 * and later on, NOP to branch to ftrace_caller() when enabled or branch to 138 * NOP when disabled per-function base. 139 */ 140ENTRY(_mcount) 141 ret 142ENDPROC(_mcount) 143 144/* 145 * void ftrace_caller(unsigned long return_address) 146 * @return_address: return address to instrumented function 147 * 148 * This function is a counterpart of _mcount() in 'static' ftrace, and 149 * makes calls to: 150 * - tracer function to probe instrumented function's entry, 151 * - ftrace_graph_caller to set up an exit hook 152 */ 153ENTRY(ftrace_caller) 154 mcount_enter 155 156 mcount_get_pc0 x0 // function's pc 157 mcount_get_lr x1 // function's lr 158 159 .global ftrace_call 160ftrace_call: // tracer(pc, lr); 161 nop // This will be replaced with "bl xxx" 162 // where xxx can be any kind of tracer. 163 164#ifdef CONFIG_FUNCTION_GRAPH_TRACER 165 .global ftrace_graph_call 166ftrace_graph_call: // ftrace_graph_caller(); 167 nop // If enabled, this will be replaced 168 // "b ftrace_graph_caller" 169#endif 170 171 mcount_exit 172ENDPROC(ftrace_caller) 173#endif /* CONFIG_DYNAMIC_FTRACE */ 174 175ENTRY(ftrace_stub) 176 ret 177ENDPROC(ftrace_stub) 178 179#ifdef CONFIG_FUNCTION_GRAPH_TRACER 180 /* save return value regs*/ 181 .macro save_return_regs 182 sub sp, sp, #64 183 stp x0, x1, [sp] 184 stp x2, x3, [sp, #16] 185 stp x4, x5, [sp, #32] 186 stp x6, x7, [sp, #48] 187 .endm 188 189 /* restore return value regs*/ 190 .macro restore_return_regs 191 ldp x0, x1, [sp] 192 ldp x2, x3, [sp, #16] 193 ldp x4, x5, [sp, #32] 194 ldp x6, x7, [sp, #48] 195 add sp, sp, #64 196 .endm 197 198/* 199 * void ftrace_graph_caller(void) 200 * 201 * Called from _mcount() or ftrace_caller() when function_graph tracer is 202 * selected. 203 * This function w/ prepare_ftrace_return() fakes link register's value on 204 * the call stack in order to intercept instrumented function's return path 205 * and run return_to_handler() later on its exit. 206 */ 207ENTRY(ftrace_graph_caller) 208 mcount_get_lr_addr x0 // pointer to function's saved lr 209 mcount_get_pc x1 // function's pc 210 mcount_get_parent_fp x2 // parent's fp 211 bl prepare_ftrace_return // prepare_ftrace_return(&lr, pc, fp) 212 213 mcount_exit 214ENDPROC(ftrace_graph_caller) 215 216/* 217 * void return_to_handler(void) 218 * 219 * Run ftrace_return_to_handler() before going back to parent. 220 * @fp is checked against the value passed by ftrace_graph_caller() 221 * only when CONFIG_HAVE_FUNCTION_GRAPH_FP_TEST is enabled. 222 */ 223ENTRY(return_to_handler) 224 save_return_regs 225 mov x0, x29 // parent's fp 226 bl ftrace_return_to_handler// addr = ftrace_return_to_hander(fp); 227 mov x30, x0 // restore the original return address 228 restore_return_regs 229 ret 230END(return_to_handler) 231#endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 232