1/* 2 * Split from ftrace_64.S 3 * 4 * This program is free software; you can redistribute it and/or 5 * modify it under the terms of the GNU General Public License 6 * as published by the Free Software Foundation; either version 7 * 2 of the License, or (at your option) any later version. 8 */ 9 10#include <linux/magic.h> 11#include <asm/ppc_asm.h> 12#include <asm/asm-offsets.h> 13#include <asm/ftrace.h> 14#include <asm/ppc-opcode.h> 15#include <asm/export.h> 16#include <asm/thread_info.h> 17#include <asm/bug.h> 18#include <asm/ptrace.h> 19 20#ifdef CONFIG_DYNAMIC_FTRACE 21/* 22 * 23 * ftrace_caller() is the function that replaces _mcount() when ftrace is 24 * active. 25 * 26 * We arrive here after a function A calls function B, and we are the trace 27 * function for B. When we enter r1 points to A's stack frame, B has not yet 28 * had a chance to allocate one yet. 29 * 30 * Additionally r2 may point either to the TOC for A, or B, depending on 31 * whether B did a TOC setup sequence before calling us. 32 * 33 * On entry the LR points back to the _mcount() call site, and r0 holds the 34 * saved LR as it was on entry to B, ie. the original return address at the 35 * call site in A. 36 * 37 * Our job is to save the register state into a struct pt_regs (on the stack) 38 * and then arrange for the ftrace function to be called. 39 */ 40_GLOBAL(ftrace_caller) 41 /* Save the original return address in A's stack frame */ 42 std r0,LRSAVE(r1) 43 44 /* Create our stack frame + pt_regs */ 45 stdu r1,-SWITCH_FRAME_SIZE(r1) 46 47 /* Save all gprs to pt_regs */ 48 SAVE_GPR(0, r1) 49 SAVE_10GPRS(2, r1) 50 SAVE_10GPRS(12, r1) 51 SAVE_10GPRS(22, r1) 52 53 /* Save previous stack pointer (r1) */ 54 addi r8, r1, SWITCH_FRAME_SIZE 55 std r8, GPR1(r1) 56 57 /* Load special regs for save below */ 58 mfmsr r8 59 mfctr r9 60 mfxer r10 61 mfcr r11 62 63 /* Get the _mcount() call site out of LR */ 64 mflr r7 65 /* Save it as pt_regs->nip */ 66 std r7, _NIP(r1) 67 /* Save the read LR in pt_regs->link */ 68 std r0, _LINK(r1) 69 70 /* Save callee's TOC in the ABI compliant location */ 71 std r2, 24(r1) 72 ld r2,PACATOC(r13) /* get kernel TOC in r2 */ 73 74 addis r3,r2,function_trace_op@toc@ha 75 addi r3,r3,function_trace_op@toc@l 76 ld r5,0(r3) 77 78#ifdef CONFIG_LIVEPATCH 79 mr r14,r7 /* remember old NIP */ 80#endif 81 /* Calculate ip from nip-4 into r3 for call below */ 82 subi r3, r7, MCOUNT_INSN_SIZE 83 84 /* Put the original return address in r4 as parent_ip */ 85 mr r4, r0 86 87 /* Save special regs */ 88 std r8, _MSR(r1) 89 std r9, _CTR(r1) 90 std r10, _XER(r1) 91 std r11, _CCR(r1) 92 93 /* Load &pt_regs in r6 for call below */ 94 addi r6, r1 ,STACK_FRAME_OVERHEAD 95 96 /* ftrace_call(r3, r4, r5, r6) */ 97.globl ftrace_call 98ftrace_call: 99 bl ftrace_stub 100 nop 101 102 /* Load the possibly modified NIP */ 103 ld r15, _NIP(r1) 104 105#ifdef CONFIG_LIVEPATCH 106 cmpd r14, r15 /* has NIP been altered? */ 107#endif 108 109#if defined(CONFIG_LIVEPATCH) && defined(CONFIG_KPROBES_ON_FTRACE) 110 /* NIP has not been altered, skip over further checks */ 111 beq 1f 112 113 /* Check if there is an active kprobe on us */ 114 subi r3, r14, 4 115 bl is_current_kprobe_addr 116 nop 117 118 /* 119 * If r3 == 1, then this is a kprobe/jprobe. 120 * else, this is livepatched function. 121 * 122 * The conditional branch for livepatch_handler below will use the 123 * result of this comparison. For kprobe/jprobe, we just need to branch to 124 * the new NIP, not call livepatch_handler. The branch below is bne, so we 125 * want CR0[EQ] to be true if this is a kprobe/jprobe. Which means we want 126 * CR0[EQ] = (r3 == 1). 127 */ 128 cmpdi r3, 1 1291: 130#endif 131 132 /* Load CTR with the possibly modified NIP */ 133 mtctr r15 134 135 /* Restore gprs */ 136 REST_GPR(0,r1) 137 REST_10GPRS(2,r1) 138 REST_10GPRS(12,r1) 139 REST_10GPRS(22,r1) 140 141 /* Restore possibly modified LR */ 142 ld r0, _LINK(r1) 143 mtlr r0 144 145 /* Restore callee's TOC */ 146 ld r2, 24(r1) 147 148 /* Pop our stack frame */ 149 addi r1, r1, SWITCH_FRAME_SIZE 150 151#ifdef CONFIG_LIVEPATCH 152 /* 153 * Based on the cmpd or cmpdi above, if the NIP was altered and we're 154 * not on a kprobe/jprobe, then handle livepatch. 155 */ 156 bne- livepatch_handler 157#endif 158 159#ifdef CONFIG_FUNCTION_GRAPH_TRACER 160.globl ftrace_graph_call 161ftrace_graph_call: 162 b ftrace_graph_stub 163_GLOBAL(ftrace_graph_stub) 164#endif 165 166 bctr /* jump after _mcount site */ 167 168_GLOBAL(ftrace_stub) 169 blr 170 171#ifdef CONFIG_LIVEPATCH 172 /* 173 * This function runs in the mcount context, between two functions. As 174 * such it can only clobber registers which are volatile and used in 175 * function linkage. 176 * 177 * We get here when a function A, calls another function B, but B has 178 * been live patched with a new function C. 179 * 180 * On entry: 181 * - we have no stack frame and can not allocate one 182 * - LR points back to the original caller (in A) 183 * - CTR holds the new NIP in C 184 * - r0, r11 & r12 are free 185 */ 186livepatch_handler: 187 CURRENT_THREAD_INFO(r12, r1) 188 189 /* Allocate 3 x 8 bytes */ 190 ld r11, TI_livepatch_sp(r12) 191 addi r11, r11, 24 192 std r11, TI_livepatch_sp(r12) 193 194 /* Save toc & real LR on livepatch stack */ 195 std r2, -24(r11) 196 mflr r12 197 std r12, -16(r11) 198 199 /* Store stack end marker */ 200 lis r12, STACK_END_MAGIC@h 201 ori r12, r12, STACK_END_MAGIC@l 202 std r12, -8(r11) 203 204 /* Put ctr in r12 for global entry and branch there */ 205 mfctr r12 206 bctrl 207 208 /* 209 * Now we are returning from the patched function to the original 210 * caller A. We are free to use r11, r12 and we can use r2 until we 211 * restore it. 212 */ 213 214 CURRENT_THREAD_INFO(r12, r1) 215 216 ld r11, TI_livepatch_sp(r12) 217 218 /* Check stack marker hasn't been trashed */ 219 lis r2, STACK_END_MAGIC@h 220 ori r2, r2, STACK_END_MAGIC@l 221 ld r12, -8(r11) 2221: tdne r12, r2 223 EMIT_BUG_ENTRY 1b, __FILE__, __LINE__ - 1, 0 224 225 /* Restore LR & toc from livepatch stack */ 226 ld r12, -16(r11) 227 mtlr r12 228 ld r2, -24(r11) 229 230 /* Pop livepatch stack frame */ 231 CURRENT_THREAD_INFO(r12, r1) 232 subi r11, r11, 24 233 std r11, TI_livepatch_sp(r12) 234 235 /* Return to original caller of live patched function */ 236 blr 237#endif /* CONFIG_LIVEPATCH */ 238 239#endif /* CONFIG_DYNAMIC_FTRACE */ 240 241#ifdef CONFIG_FUNCTION_GRAPH_TRACER 242_GLOBAL(ftrace_graph_caller) 243 stdu r1, -112(r1) 244 /* with -mprofile-kernel, parameter regs are still alive at _mcount */ 245 std r10, 104(r1) 246 std r9, 96(r1) 247 std r8, 88(r1) 248 std r7, 80(r1) 249 std r6, 72(r1) 250 std r5, 64(r1) 251 std r4, 56(r1) 252 std r3, 48(r1) 253 254 /* Save callee's TOC in the ABI compliant location */ 255 std r2, 24(r1) 256 ld r2, PACATOC(r13) /* get kernel TOC in r2 */ 257 258 mfctr r4 /* ftrace_caller has moved local addr here */ 259 std r4, 40(r1) 260 mflr r3 /* ftrace_caller has restored LR from stack */ 261 subi r4, r4, MCOUNT_INSN_SIZE 262 263 bl prepare_ftrace_return 264 nop 265 266 /* 267 * prepare_ftrace_return gives us the address we divert to. 268 * Change the LR to this. 269 */ 270 mtlr r3 271 272 ld r0, 40(r1) 273 mtctr r0 274 ld r10, 104(r1) 275 ld r9, 96(r1) 276 ld r8, 88(r1) 277 ld r7, 80(r1) 278 ld r6, 72(r1) 279 ld r5, 64(r1) 280 ld r4, 56(r1) 281 ld r3, 48(r1) 282 283 /* Restore callee's TOC */ 284 ld r2, 24(r1) 285 286 addi r1, r1, 112 287 mflr r0 288 std r0, LRSAVE(r1) 289 bctr 290#endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 291