1/* 2 * PowerPC version 3 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org) 4 * Rewritten by Cort Dougan (cort@fsmlabs.com) for PReP 5 * Copyright (C) 1996 Cort Dougan <cort@fsmlabs.com> 6 * Adapted for Power Macintosh by Paul Mackerras. 7 * Low-level exception handlers and MMU support 8 * rewritten by Paul Mackerras. 9 * Copyright (C) 1996 Paul Mackerras. 10 * MPC8xx modifications Copyright (C) 1997 Dan Malek (dmalek@jlc.net). 11 * 12 * This file contains the system call entry code, context switch 13 * code, and exception/interrupt return code for PowerPC. 14 * 15 * This program is free software; you can redistribute it and/or 16 * modify it under the terms of the GNU General Public License 17 * as published by the Free Software Foundation; either version 18 * 2 of the License, or (at your option) any later version. 19 * 20 */ 21 22#include <linux/errno.h> 23#include <linux/sys.h> 24#include <linux/threads.h> 25#include <asm/reg.h> 26#include <asm/page.h> 27#include <asm/mmu.h> 28#include <asm/cputable.h> 29#include <asm/thread_info.h> 30#include <asm/ppc_asm.h> 31#include <asm/asm-offsets.h> 32#include <asm/unistd.h> 33#include <asm/ftrace.h> 34#include <asm/ptrace.h> 35 36#undef SHOW_SYSCALLS 37#undef SHOW_SYSCALLS_TASK 38 39/* 40 * MSR_KERNEL is > 0x10000 on 4xx/Book-E since it include MSR_CE. 41 */ 42#if MSR_KERNEL >= 0x10000 43#define LOAD_MSR_KERNEL(r, x) lis r,(x)@h; ori r,r,(x)@l 44#else 45#define LOAD_MSR_KERNEL(r, x) li r,(x) 46#endif 47 48#ifdef CONFIG_BOOKE 49 .globl mcheck_transfer_to_handler 50mcheck_transfer_to_handler: 51 mfspr r0,SPRN_DSRR0 52 stw r0,_DSRR0(r11) 53 mfspr r0,SPRN_DSRR1 54 stw r0,_DSRR1(r11) 55 /* fall through */ 56 57 .globl debug_transfer_to_handler 58debug_transfer_to_handler: 59 mfspr r0,SPRN_CSRR0 60 stw r0,_CSRR0(r11) 61 mfspr r0,SPRN_CSRR1 62 stw r0,_CSRR1(r11) 63 /* fall through */ 64 65 .globl crit_transfer_to_handler 66crit_transfer_to_handler: 67#ifdef CONFIG_PPC_BOOK3E_MMU 68 mfspr r0,SPRN_MAS0 69 stw r0,MAS0(r11) 70 mfspr r0,SPRN_MAS1 71 stw r0,MAS1(r11) 72 mfspr r0,SPRN_MAS2 73 stw r0,MAS2(r11) 74 mfspr r0,SPRN_MAS3 75 stw r0,MAS3(r11) 76 mfspr r0,SPRN_MAS6 77 stw r0,MAS6(r11) 78#ifdef CONFIG_PHYS_64BIT 79 mfspr r0,SPRN_MAS7 80 stw r0,MAS7(r11) 81#endif /* CONFIG_PHYS_64BIT */ 82#endif /* CONFIG_PPC_BOOK3E_MMU */ 83#ifdef CONFIG_44x 84 mfspr r0,SPRN_MMUCR 85 stw r0,MMUCR(r11) 86#endif 87 mfspr r0,SPRN_SRR0 88 stw r0,_SRR0(r11) 89 mfspr r0,SPRN_SRR1 90 stw r0,_SRR1(r11) 91 92 mfspr r8,SPRN_SPRG_THREAD 93 lwz r0,KSP_LIMIT(r8) 94 stw r0,SAVED_KSP_LIMIT(r11) 95 rlwimi r0,r1,0,0,(31-THREAD_SHIFT) 96 stw r0,KSP_LIMIT(r8) 97 /* fall through */ 98#endif 99 100#ifdef CONFIG_40x 101 .globl crit_transfer_to_handler 102crit_transfer_to_handler: 103 lwz r0,crit_r10@l(0) 104 stw r0,GPR10(r11) 105 lwz r0,crit_r11@l(0) 106 stw r0,GPR11(r11) 107 mfspr r0,SPRN_SRR0 108 stw r0,crit_srr0@l(0) 109 mfspr r0,SPRN_SRR1 110 stw r0,crit_srr1@l(0) 111 112 mfspr r8,SPRN_SPRG_THREAD 113 lwz r0,KSP_LIMIT(r8) 114 stw r0,saved_ksp_limit@l(0) 115 rlwimi r0,r1,0,0,(31-THREAD_SHIFT) 116 stw r0,KSP_LIMIT(r8) 117 /* fall through */ 118#endif 119 120/* 121 * This code finishes saving the registers to the exception frame 122 * and jumps to the appropriate handler for the exception, turning 123 * on address translation. 124 * Note that we rely on the caller having set cr0.eq iff the exception 125 * occurred in kernel mode (i.e. MSR:PR = 0). 126 */ 127 .globl transfer_to_handler_full 128transfer_to_handler_full: 129 SAVE_NVGPRS(r11) 130 /* fall through */ 131 132 .globl transfer_to_handler 133transfer_to_handler: 134 stw r2,GPR2(r11) 135 stw r12,_NIP(r11) 136 stw r9,_MSR(r11) 137 andi. r2,r9,MSR_PR 138 mfctr r12 139 mfspr r2,SPRN_XER 140 stw r12,_CTR(r11) 141 stw r2,_XER(r11) 142 mfspr r12,SPRN_SPRG_THREAD 143 addi r2,r12,-THREAD 144 tovirt(r2,r2) /* set r2 to current */ 145 beq 2f /* if from user, fix up THREAD.regs */ 146 addi r11,r1,STACK_FRAME_OVERHEAD 147 stw r11,PT_REGS(r12) 148#if defined(CONFIG_40x) || defined(CONFIG_BOOKE) 149 /* Check to see if the dbcr0 register is set up to debug. Use the 150 internal debug mode bit to do this. */ 151 lwz r12,THREAD_DBCR0(r12) 152 andis. r12,r12,DBCR0_IDM@h 153 beq+ 3f 154 /* From user and task is ptraced - load up global dbcr0 */ 155 li r12,-1 /* clear all pending debug events */ 156 mtspr SPRN_DBSR,r12 157 lis r11,global_dbcr0@ha 158 tophys(r11,r11) 159 addi r11,r11,global_dbcr0@l 160#ifdef CONFIG_SMP 161 rlwinm r9,r1,0,0,(31-THREAD_SHIFT) 162 lwz r9,TI_CPU(r9) 163 slwi r9,r9,3 164 add r11,r11,r9 165#endif 166 lwz r12,0(r11) 167 mtspr SPRN_DBCR0,r12 168 lwz r12,4(r11) 169 addi r12,r12,-1 170 stw r12,4(r11) 171#endif 172 b 3f 173 1742: /* if from kernel, check interrupted DOZE/NAP mode and 175 * check for stack overflow 176 */ 177 lwz r9,KSP_LIMIT(r12) 178 cmplw r1,r9 /* if r1 <= ksp_limit */ 179 ble- stack_ovf /* then the kernel stack overflowed */ 1805: 181#if defined(CONFIG_6xx) || defined(CONFIG_E500) 182 rlwinm r9,r1,0,0,31-THREAD_SHIFT 183 tophys(r9,r9) /* check local flags */ 184 lwz r12,TI_LOCAL_FLAGS(r9) 185 mtcrf 0x01,r12 186 bt- 31-TLF_NAPPING,4f 187 bt- 31-TLF_SLEEPING,7f 188#endif /* CONFIG_6xx || CONFIG_E500 */ 189 .globl transfer_to_handler_cont 190transfer_to_handler_cont: 1913: 192 mflr r9 193 lwz r11,0(r9) /* virtual address of handler */ 194 lwz r9,4(r9) /* where to go when done */ 195#ifdef CONFIG_TRACE_IRQFLAGS 196 lis r12,reenable_mmu@h 197 ori r12,r12,reenable_mmu@l 198 mtspr SPRN_SRR0,r12 199 mtspr SPRN_SRR1,r10 200 SYNC 201 RFI 202reenable_mmu: /* re-enable mmu so we can */ 203 mfmsr r10 204 lwz r12,_MSR(r1) 205 xor r10,r10,r12 206 andi. r10,r10,MSR_EE /* Did EE change? */ 207 beq 1f 208 209 /* 210 * The trace_hardirqs_off will use CALLER_ADDR0 and CALLER_ADDR1. 211 * If from user mode there is only one stack frame on the stack, and 212 * accessing CALLER_ADDR1 will cause oops. So we need create a dummy 213 * stack frame to make trace_hardirqs_off happy. 214 * 215 * This is handy because we also need to save a bunch of GPRs, 216 * r3 can be different from GPR3(r1) at this point, r9 and r11 217 * contains the old MSR and handler address respectively, 218 * r4 & r5 can contain page fault arguments that need to be passed 219 * along as well. r12, CCR, CTR, XER etc... are left clobbered as 220 * they aren't useful past this point (aren't syscall arguments), 221 * the rest is restored from the exception frame. 222 */ 223 stwu r1,-32(r1) 224 stw r9,8(r1) 225 stw r11,12(r1) 226 stw r3,16(r1) 227 stw r4,20(r1) 228 stw r5,24(r1) 229 andi. r12,r12,MSR_PR 230 b 11f 231 bl trace_hardirqs_off 232 b 12f 23311: 234 bl trace_hardirqs_off 23512: 236 lwz r5,24(r1) 237 lwz r4,20(r1) 238 lwz r3,16(r1) 239 lwz r11,12(r1) 240 lwz r9,8(r1) 241 addi r1,r1,32 242 lwz r0,GPR0(r1) 243 lwz r6,GPR6(r1) 244 lwz r7,GPR7(r1) 245 lwz r8,GPR8(r1) 2461: mtctr r11 247 mtlr r9 248 bctr /* jump to handler */ 249#else /* CONFIG_TRACE_IRQFLAGS */ 250 mtspr SPRN_SRR0,r11 251 mtspr SPRN_SRR1,r10 252 mtlr r9 253 SYNC 254 RFI /* jump to handler, enable MMU */ 255#endif /* CONFIG_TRACE_IRQFLAGS */ 256 257#if defined (CONFIG_6xx) || defined(CONFIG_E500) 2584: rlwinm r12,r12,0,~_TLF_NAPPING 259 stw r12,TI_LOCAL_FLAGS(r9) 260 b power_save_ppc32_restore 261 2627: rlwinm r12,r12,0,~_TLF_SLEEPING 263 stw r12,TI_LOCAL_FLAGS(r9) 264 lwz r9,_MSR(r11) /* if sleeping, clear MSR.EE */ 265 rlwinm r9,r9,0,~MSR_EE 266 lwz r12,_LINK(r11) /* and return to address in LR */ 267 b fast_exception_return 268#endif 269 270/* 271 * On kernel stack overflow, load up an initial stack pointer 272 * and call StackOverflow(regs), which should not return. 273 */ 274stack_ovf: 275 /* sometimes we use a statically-allocated stack, which is OK. */ 276 lis r12,_end@h 277 ori r12,r12,_end@l 278 cmplw r1,r12 279 ble 5b /* r1 <= &_end is OK */ 280 SAVE_NVGPRS(r11) 281 addi r3,r1,STACK_FRAME_OVERHEAD 282 lis r1,init_thread_union@ha 283 addi r1,r1,init_thread_union@l 284 addi r1,r1,THREAD_SIZE-STACK_FRAME_OVERHEAD 285 lis r9,StackOverflow@ha 286 addi r9,r9,StackOverflow@l 287 LOAD_MSR_KERNEL(r10,MSR_KERNEL) 288 FIX_SRR1(r10,r12) 289 mtspr SPRN_SRR0,r9 290 mtspr SPRN_SRR1,r10 291 SYNC 292 RFI 293 294/* 295 * Handle a system call. 296 */ 297 .stabs "arch/powerpc/kernel/",N_SO,0,0,0f 298 .stabs "entry_32.S",N_SO,0,0,0f 2990: 300 301_GLOBAL(DoSyscall) 302 stw r3,ORIG_GPR3(r1) 303 li r12,0 304 stw r12,RESULT(r1) 305 lwz r11,_CCR(r1) /* Clear SO bit in CR */ 306 rlwinm r11,r11,0,4,2 307 stw r11,_CCR(r1) 308#ifdef SHOW_SYSCALLS 309 bl do_show_syscall 310#endif /* SHOW_SYSCALLS */ 311#ifdef CONFIG_TRACE_IRQFLAGS 312 /* Return from syscalls can (and generally will) hard enable 313 * interrupts. You aren't supposed to call a syscall with 314 * interrupts disabled in the first place. However, to ensure 315 * that we get it right vs. lockdep if it happens, we force 316 * that hard enable here with appropriate tracing if we see 317 * that we have been called with interrupts off 318 */ 319 mfmsr r11 320 andi. r12,r11,MSR_EE 321 bne+ 1f 322 /* We came in with interrupts disabled, we enable them now */ 323 bl trace_hardirqs_on 324 mfmsr r11 325 lwz r0,GPR0(r1) 326 lwz r3,GPR3(r1) 327 lwz r4,GPR4(r1) 328 ori r11,r11,MSR_EE 329 lwz r5,GPR5(r1) 330 lwz r6,GPR6(r1) 331 lwz r7,GPR7(r1) 332 lwz r8,GPR8(r1) 333 mtmsr r11 3341: 335#endif /* CONFIG_TRACE_IRQFLAGS */ 336 rlwinm r10,r1,0,0,(31-THREAD_SHIFT) /* current_thread_info() */ 337 lwz r11,TI_FLAGS(r10) 338 andi. r11,r11,_TIF_SYSCALL_T_OR_A 339 bne- syscall_dotrace 340syscall_dotrace_cont: 341 cmplwi 0,r0,NR_syscalls 342 lis r10,sys_call_table@h 343 ori r10,r10,sys_call_table@l 344 slwi r0,r0,2 345 bge- 66f 346 lwzx r10,r10,r0 /* Fetch system call handler [ptr] */ 347 mtlr r10 348 addi r9,r1,STACK_FRAME_OVERHEAD 349 PPC440EP_ERR42 350 blrl /* Call handler */ 351 .globl ret_from_syscall 352ret_from_syscall: 353#ifdef SHOW_SYSCALLS 354 bl do_show_syscall_exit 355#endif 356 mr r6,r3 357 rlwinm r12,r1,0,0,(31-THREAD_SHIFT) /* current_thread_info() */ 358 /* disable interrupts so current_thread_info()->flags can't change */ 359 LOAD_MSR_KERNEL(r10,MSR_KERNEL) /* doesn't include MSR_EE */ 360 /* Note: We don't bother telling lockdep about it */ 361 SYNC 362 MTMSRD(r10) 363 lwz r9,TI_FLAGS(r12) 364 li r8,-_LAST_ERRNO 365 andi. r0,r9,(_TIF_SYSCALL_T_OR_A|_TIF_SINGLESTEP|_TIF_USER_WORK_MASK|_TIF_PERSYSCALL_MASK) 366 bne- syscall_exit_work 367 cmplw 0,r3,r8 368 blt+ syscall_exit_cont 369 lwz r11,_CCR(r1) /* Load CR */ 370 neg r3,r3 371 oris r11,r11,0x1000 /* Set SO bit in CR */ 372 stw r11,_CCR(r1) 373syscall_exit_cont: 374 lwz r8,_MSR(r1) 375#ifdef CONFIG_TRACE_IRQFLAGS 376 /* If we are going to return from the syscall with interrupts 377 * off, we trace that here. It shouldn't happen though but we 378 * want to catch the bugger if it does right ? 379 */ 380 andi. r10,r8,MSR_EE 381 bne+ 1f 382 stw r3,GPR3(r1) 383 bl trace_hardirqs_off 384 lwz r3,GPR3(r1) 3851: 386#endif /* CONFIG_TRACE_IRQFLAGS */ 387#if defined(CONFIG_4xx) || defined(CONFIG_BOOKE) 388 /* If the process has its own DBCR0 value, load it up. The internal 389 debug mode bit tells us that dbcr0 should be loaded. */ 390 lwz r0,THREAD+THREAD_DBCR0(r2) 391 andis. r10,r0,DBCR0_IDM@h 392 bnel- load_dbcr0 393#endif 394#ifdef CONFIG_44x 395BEGIN_MMU_FTR_SECTION 396 lis r4,icache_44x_need_flush@ha 397 lwz r5,icache_44x_need_flush@l(r4) 398 cmplwi cr0,r5,0 399 bne- 2f 4001: 401END_MMU_FTR_SECTION_IFCLR(MMU_FTR_TYPE_47x) 402#endif /* CONFIG_44x */ 403BEGIN_FTR_SECTION 404 lwarx r7,0,r1 405END_FTR_SECTION_IFSET(CPU_FTR_NEED_PAIRED_STWCX) 406 stwcx. r0,0,r1 /* to clear the reservation */ 407 lwz r4,_LINK(r1) 408 lwz r5,_CCR(r1) 409 mtlr r4 410 mtcr r5 411 lwz r7,_NIP(r1) 412 FIX_SRR1(r8, r0) 413 lwz r2,GPR2(r1) 414 lwz r1,GPR1(r1) 415 mtspr SPRN_SRR0,r7 416 mtspr SPRN_SRR1,r8 417 SYNC 418 RFI 419#ifdef CONFIG_44x 4202: li r7,0 421 iccci r0,r0 422 stw r7,icache_44x_need_flush@l(r4) 423 b 1b 424#endif /* CONFIG_44x */ 425 42666: li r3,-ENOSYS 427 b ret_from_syscall 428 429 .globl ret_from_fork 430ret_from_fork: 431 REST_NVGPRS(r1) 432 bl schedule_tail 433 li r3,0 434 b ret_from_syscall 435 436/* Traced system call support */ 437syscall_dotrace: 438 SAVE_NVGPRS(r1) 439 li r0,0xc00 440 stw r0,_TRAP(r1) 441 addi r3,r1,STACK_FRAME_OVERHEAD 442 bl do_syscall_trace_enter 443 /* 444 * Restore argument registers possibly just changed. 445 * We use the return value of do_syscall_trace_enter 446 * for call number to look up in the table (r0). 447 */ 448 mr r0,r3 449 lwz r3,GPR3(r1) 450 lwz r4,GPR4(r1) 451 lwz r5,GPR5(r1) 452 lwz r6,GPR6(r1) 453 lwz r7,GPR7(r1) 454 lwz r8,GPR8(r1) 455 REST_NVGPRS(r1) 456 b syscall_dotrace_cont 457 458syscall_exit_work: 459 andi. r0,r9,_TIF_RESTOREALL 460 beq+ 0f 461 REST_NVGPRS(r1) 462 b 2f 4630: cmplw 0,r3,r8 464 blt+ 1f 465 andi. r0,r9,_TIF_NOERROR 466 bne- 1f 467 lwz r11,_CCR(r1) /* Load CR */ 468 neg r3,r3 469 oris r11,r11,0x1000 /* Set SO bit in CR */ 470 stw r11,_CCR(r1) 471 4721: stw r6,RESULT(r1) /* Save result */ 473 stw r3,GPR3(r1) /* Update return value */ 4742: andi. r0,r9,(_TIF_PERSYSCALL_MASK) 475 beq 4f 476 477 /* Clear per-syscall TIF flags if any are set. */ 478 479 li r11,_TIF_PERSYSCALL_MASK 480 addi r12,r12,TI_FLAGS 4813: lwarx r8,0,r12 482 andc r8,r8,r11 483#ifdef CONFIG_IBM405_ERR77 484 dcbt 0,r12 485#endif 486 stwcx. r8,0,r12 487 bne- 3b 488 subi r12,r12,TI_FLAGS 489 4904: /* Anything which requires enabling interrupts? */ 491 andi. r0,r9,(_TIF_SYSCALL_T_OR_A|_TIF_SINGLESTEP) 492 beq ret_from_except 493 494 /* Re-enable interrupts. There is no need to trace that with 495 * lockdep as we are supposed to have IRQs on at this point 496 */ 497 ori r10,r10,MSR_EE 498 SYNC 499 MTMSRD(r10) 500 501 /* Save NVGPRS if they're not saved already */ 502 lwz r4,_TRAP(r1) 503 andi. r4,r4,1 504 beq 5f 505 SAVE_NVGPRS(r1) 506 li r4,0xc00 507 stw r4,_TRAP(r1) 5085: 509 addi r3,r1,STACK_FRAME_OVERHEAD 510 bl do_syscall_trace_leave 511 b ret_from_except_full 512 513#ifdef SHOW_SYSCALLS 514do_show_syscall: 515#ifdef SHOW_SYSCALLS_TASK 516 lis r11,show_syscalls_task@ha 517 lwz r11,show_syscalls_task@l(r11) 518 cmp 0,r2,r11 519 bnelr 520#endif 521 stw r31,GPR31(r1) 522 mflr r31 523 lis r3,7f@ha 524 addi r3,r3,7f@l 525 lwz r4,GPR0(r1) 526 lwz r5,GPR3(r1) 527 lwz r6,GPR4(r1) 528 lwz r7,GPR5(r1) 529 lwz r8,GPR6(r1) 530 lwz r9,GPR7(r1) 531 bl printk 532 lis r3,77f@ha 533 addi r3,r3,77f@l 534 lwz r4,GPR8(r1) 535 mr r5,r2 536 bl printk 537 lwz r0,GPR0(r1) 538 lwz r3,GPR3(r1) 539 lwz r4,GPR4(r1) 540 lwz r5,GPR5(r1) 541 lwz r6,GPR6(r1) 542 lwz r7,GPR7(r1) 543 lwz r8,GPR8(r1) 544 mtlr r31 545 lwz r31,GPR31(r1) 546 blr 547 548do_show_syscall_exit: 549#ifdef SHOW_SYSCALLS_TASK 550 lis r11,show_syscalls_task@ha 551 lwz r11,show_syscalls_task@l(r11) 552 cmp 0,r2,r11 553 bnelr 554#endif 555 stw r31,GPR31(r1) 556 mflr r31 557 stw r3,RESULT(r1) /* Save result */ 558 mr r4,r3 559 lis r3,79f@ha 560 addi r3,r3,79f@l 561 bl printk 562 lwz r3,RESULT(r1) 563 mtlr r31 564 lwz r31,GPR31(r1) 565 blr 566 5677: .string "syscall %d(%x, %x, %x, %x, %x, " 56877: .string "%x), current=%p\n" 56979: .string " -> %x\n" 570 .align 2,0 571 572#ifdef SHOW_SYSCALLS_TASK 573 .data 574 .globl show_syscalls_task 575show_syscalls_task: 576 .long -1 577 .text 578#endif 579#endif /* SHOW_SYSCALLS */ 580 581/* 582 * The fork/clone functions need to copy the full register set into 583 * the child process. Therefore we need to save all the nonvolatile 584 * registers (r13 - r31) before calling the C code. 585 */ 586 .globl ppc_fork 587ppc_fork: 588 SAVE_NVGPRS(r1) 589 lwz r0,_TRAP(r1) 590 rlwinm r0,r0,0,0,30 /* clear LSB to indicate full */ 591 stw r0,_TRAP(r1) /* register set saved */ 592 b sys_fork 593 594 .globl ppc_vfork 595ppc_vfork: 596 SAVE_NVGPRS(r1) 597 lwz r0,_TRAP(r1) 598 rlwinm r0,r0,0,0,30 /* clear LSB to indicate full */ 599 stw r0,_TRAP(r1) /* register set saved */ 600 b sys_vfork 601 602 .globl ppc_clone 603ppc_clone: 604 SAVE_NVGPRS(r1) 605 lwz r0,_TRAP(r1) 606 rlwinm r0,r0,0,0,30 /* clear LSB to indicate full */ 607 stw r0,_TRAP(r1) /* register set saved */ 608 b sys_clone 609 610 .globl ppc_swapcontext 611ppc_swapcontext: 612 SAVE_NVGPRS(r1) 613 lwz r0,_TRAP(r1) 614 rlwinm r0,r0,0,0,30 /* clear LSB to indicate full */ 615 stw r0,_TRAP(r1) /* register set saved */ 616 b sys_swapcontext 617 618/* 619 * Top-level page fault handling. 620 * This is in assembler because if do_page_fault tells us that 621 * it is a bad kernel page fault, we want to save the non-volatile 622 * registers before calling bad_page_fault. 623 */ 624 .globl handle_page_fault 625handle_page_fault: 626 stw r4,_DAR(r1) 627 addi r3,r1,STACK_FRAME_OVERHEAD 628 bl do_page_fault 629 cmpwi r3,0 630 beq+ ret_from_except 631 SAVE_NVGPRS(r1) 632 lwz r0,_TRAP(r1) 633 clrrwi r0,r0,1 634 stw r0,_TRAP(r1) 635 mr r5,r3 636 addi r3,r1,STACK_FRAME_OVERHEAD 637 lwz r4,_DAR(r1) 638 bl bad_page_fault 639 b ret_from_except_full 640 641/* 642 * This routine switches between two different tasks. The process 643 * state of one is saved on its kernel stack. Then the state 644 * of the other is restored from its kernel stack. The memory 645 * management hardware is updated to the second process's state. 646 * Finally, we can return to the second process. 647 * On entry, r3 points to the THREAD for the current task, r4 648 * points to the THREAD for the new task. 649 * 650 * This routine is always called with interrupts disabled. 651 * 652 * Note: there are two ways to get to the "going out" portion 653 * of this code; either by coming in via the entry (_switch) 654 * or via "fork" which must set up an environment equivalent 655 * to the "_switch" path. If you change this , you'll have to 656 * change the fork code also. 657 * 658 * The code which creates the new task context is in 'copy_thread' 659 * in arch/ppc/kernel/process.c 660 */ 661_GLOBAL(_switch) 662 stwu r1,-INT_FRAME_SIZE(r1) 663 mflr r0 664 stw r0,INT_FRAME_SIZE+4(r1) 665 /* r3-r12 are caller saved -- Cort */ 666 SAVE_NVGPRS(r1) 667 stw r0,_NIP(r1) /* Return to switch caller */ 668 mfmsr r11 669 li r0,MSR_FP /* Disable floating-point */ 670#ifdef CONFIG_ALTIVEC 671BEGIN_FTR_SECTION 672 oris r0,r0,MSR_VEC@h /* Disable altivec */ 673 mfspr r12,SPRN_VRSAVE /* save vrsave register value */ 674 stw r12,THREAD+THREAD_VRSAVE(r2) 675END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) 676#endif /* CONFIG_ALTIVEC */ 677#ifdef CONFIG_SPE 678BEGIN_FTR_SECTION 679 oris r0,r0,MSR_SPE@h /* Disable SPE */ 680 mfspr r12,SPRN_SPEFSCR /* save spefscr register value */ 681 stw r12,THREAD+THREAD_SPEFSCR(r2) 682END_FTR_SECTION_IFSET(CPU_FTR_SPE) 683#endif /* CONFIG_SPE */ 684 and. r0,r0,r11 /* FP or altivec or SPE enabled? */ 685 beq+ 1f 686 andc r11,r11,r0 687 MTMSRD(r11) 688 isync 6891: stw r11,_MSR(r1) 690 mfcr r10 691 stw r10,_CCR(r1) 692 stw r1,KSP(r3) /* Set old stack pointer */ 693 694#ifdef CONFIG_SMP 695 /* We need a sync somewhere here to make sure that if the 696 * previous task gets rescheduled on another CPU, it sees all 697 * stores it has performed on this one. 698 */ 699 sync 700#endif /* CONFIG_SMP */ 701 702 tophys(r0,r4) 703 CLR_TOP32(r0) 704 mtspr SPRN_SPRG_THREAD,r0 /* Update current THREAD phys addr */ 705 lwz r1,KSP(r4) /* Load new stack pointer */ 706 707 /* save the old current 'last' for return value */ 708 mr r3,r2 709 addi r2,r4,-THREAD /* Update current */ 710 711#ifdef CONFIG_ALTIVEC 712BEGIN_FTR_SECTION 713 lwz r0,THREAD+THREAD_VRSAVE(r2) 714 mtspr SPRN_VRSAVE,r0 /* if G4, restore VRSAVE reg */ 715END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) 716#endif /* CONFIG_ALTIVEC */ 717#ifdef CONFIG_SPE 718BEGIN_FTR_SECTION 719 lwz r0,THREAD+THREAD_SPEFSCR(r2) 720 mtspr SPRN_SPEFSCR,r0 /* restore SPEFSCR reg */ 721END_FTR_SECTION_IFSET(CPU_FTR_SPE) 722#endif /* CONFIG_SPE */ 723 724 lwz r0,_CCR(r1) 725 mtcrf 0xFF,r0 726 /* r3-r12 are destroyed -- Cort */ 727 REST_NVGPRS(r1) 728 729 lwz r4,_NIP(r1) /* Return to _switch caller in new task */ 730 mtlr r4 731 addi r1,r1,INT_FRAME_SIZE 732 blr 733 734 .globl fast_exception_return 735fast_exception_return: 736#if !(defined(CONFIG_4xx) || defined(CONFIG_BOOKE)) 737 andi. r10,r9,MSR_RI /* check for recoverable interrupt */ 738 beq 1f /* if not, we've got problems */ 739#endif 740 7412: REST_4GPRS(3, r11) 742 lwz r10,_CCR(r11) 743 REST_GPR(1, r11) 744 mtcr r10 745 lwz r10,_LINK(r11) 746 mtlr r10 747 REST_GPR(10, r11) 748 mtspr SPRN_SRR1,r9 749 mtspr SPRN_SRR0,r12 750 REST_GPR(9, r11) 751 REST_GPR(12, r11) 752 lwz r11,GPR11(r11) 753 SYNC 754 RFI 755 756#if !(defined(CONFIG_4xx) || defined(CONFIG_BOOKE)) 757/* check if the exception happened in a restartable section */ 7581: lis r3,exc_exit_restart_end@ha 759 addi r3,r3,exc_exit_restart_end@l 760 cmplw r12,r3 761 bge 3f 762 lis r4,exc_exit_restart@ha 763 addi r4,r4,exc_exit_restart@l 764 cmplw r12,r4 765 blt 3f 766 lis r3,fee_restarts@ha 767 tophys(r3,r3) 768 lwz r5,fee_restarts@l(r3) 769 addi r5,r5,1 770 stw r5,fee_restarts@l(r3) 771 mr r12,r4 /* restart at exc_exit_restart */ 772 b 2b 773 774 .section .bss 775 .align 2 776fee_restarts: 777 .space 4 778 .previous 779 780/* aargh, a nonrecoverable interrupt, panic */ 781/* aargh, we don't know which trap this is */ 782/* but the 601 doesn't implement the RI bit, so assume it's OK */ 7833: 784BEGIN_FTR_SECTION 785 b 2b 786END_FTR_SECTION_IFSET(CPU_FTR_601) 787 li r10,-1 788 stw r10,_TRAP(r11) 789 addi r3,r1,STACK_FRAME_OVERHEAD 790 lis r10,MSR_KERNEL@h 791 ori r10,r10,MSR_KERNEL@l 792 bl transfer_to_handler_full 793 .long nonrecoverable_exception 794 .long ret_from_except 795#endif 796 797 .globl ret_from_except_full 798ret_from_except_full: 799 REST_NVGPRS(r1) 800 /* fall through */ 801 802 .globl ret_from_except 803ret_from_except: 804 /* Hard-disable interrupts so that current_thread_info()->flags 805 * can't change between when we test it and when we return 806 * from the interrupt. */ 807 /* Note: We don't bother telling lockdep about it */ 808 LOAD_MSR_KERNEL(r10,MSR_KERNEL) 809 SYNC /* Some chip revs have problems here... */ 810 MTMSRD(r10) /* disable interrupts */ 811 812 lwz r3,_MSR(r1) /* Returning to user mode? */ 813 andi. r0,r3,MSR_PR 814 beq resume_kernel 815 816user_exc_return: /* r10 contains MSR_KERNEL here */ 817 /* Check current_thread_info()->flags */ 818 rlwinm r9,r1,0,0,(31-THREAD_SHIFT) 819 lwz r9,TI_FLAGS(r9) 820 andi. r0,r9,_TIF_USER_WORK_MASK 821 bne do_work 822 823restore_user: 824#if defined(CONFIG_4xx) || defined(CONFIG_BOOKE) 825 /* Check whether this process has its own DBCR0 value. The internal 826 debug mode bit tells us that dbcr0 should be loaded. */ 827 lwz r0,THREAD+THREAD_DBCR0(r2) 828 andis. r10,r0,DBCR0_IDM@h 829 bnel- load_dbcr0 830#endif 831 832#ifdef CONFIG_PREEMPT 833 b restore 834 835/* N.B. the only way to get here is from the beq following ret_from_except. */ 836resume_kernel: 837 /* check current_thread_info->preempt_count */ 838 rlwinm r9,r1,0,0,(31-THREAD_SHIFT) 839 lwz r0,TI_PREEMPT(r9) 840 cmpwi 0,r0,0 /* if non-zero, just restore regs and return */ 841 bne restore 842 lwz r0,TI_FLAGS(r9) 843 andi. r0,r0,_TIF_NEED_RESCHED 844 beq+ restore 845 andi. r0,r3,MSR_EE /* interrupts off? */ 846 beq restore /* don't schedule if so */ 847#ifdef CONFIG_TRACE_IRQFLAGS 848 /* Lockdep thinks irqs are enabled, we need to call 849 * preempt_schedule_irq with IRQs off, so we inform lockdep 850 * now that we -did- turn them off already 851 */ 852 bl trace_hardirqs_off 853#endif 8541: bl preempt_schedule_irq 855 rlwinm r9,r1,0,0,(31-THREAD_SHIFT) 856 lwz r3,TI_FLAGS(r9) 857 andi. r0,r3,_TIF_NEED_RESCHED 858 bne- 1b 859#ifdef CONFIG_TRACE_IRQFLAGS 860 /* And now, to properly rebalance the above, we tell lockdep they 861 * are being turned back on, which will happen when we return 862 */ 863 bl trace_hardirqs_on 864#endif 865#else 866resume_kernel: 867#endif /* CONFIG_PREEMPT */ 868 869 /* interrupts are hard-disabled at this point */ 870restore: 871#ifdef CONFIG_44x 872BEGIN_MMU_FTR_SECTION 873 b 1f 874END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_47x) 875 lis r4,icache_44x_need_flush@ha 876 lwz r5,icache_44x_need_flush@l(r4) 877 cmplwi cr0,r5,0 878 beq+ 1f 879 li r6,0 880 iccci r0,r0 881 stw r6,icache_44x_need_flush@l(r4) 8821: 883#endif /* CONFIG_44x */ 884 885 lwz r9,_MSR(r1) 886#ifdef CONFIG_TRACE_IRQFLAGS 887 /* Lockdep doesn't know about the fact that IRQs are temporarily turned 888 * off in this assembly code while peeking at TI_FLAGS() and such. However 889 * we need to inform it if the exception turned interrupts off, and we 890 * are about to trun them back on. 891 * 892 * The problem here sadly is that we don't know whether the exceptions was 893 * one that turned interrupts off or not. So we always tell lockdep about 894 * turning them on here when we go back to wherever we came from with EE 895 * on, even if that may meen some redudant calls being tracked. Maybe later 896 * we could encode what the exception did somewhere or test the exception 897 * type in the pt_regs but that sounds overkill 898 */ 899 andi. r10,r9,MSR_EE 900 beq 1f 901 /* 902 * Since the ftrace irqsoff latency trace checks CALLER_ADDR1, 903 * which is the stack frame here, we need to force a stack frame 904 * in case we came from user space. 905 */ 906 stwu r1,-32(r1) 907 mflr r0 908 stw r0,4(r1) 909 stwu r1,-32(r1) 910 bl trace_hardirqs_on 911 lwz r1,0(r1) 912 lwz r1,0(r1) 913 lwz r9,_MSR(r1) 9141: 915#endif /* CONFIG_TRACE_IRQFLAGS */ 916 917 lwz r0,GPR0(r1) 918 lwz r2,GPR2(r1) 919 REST_4GPRS(3, r1) 920 REST_2GPRS(7, r1) 921 922 lwz r10,_XER(r1) 923 lwz r11,_CTR(r1) 924 mtspr SPRN_XER,r10 925 mtctr r11 926 927 PPC405_ERR77(0,r1) 928BEGIN_FTR_SECTION 929 lwarx r11,0,r1 930END_FTR_SECTION_IFSET(CPU_FTR_NEED_PAIRED_STWCX) 931 stwcx. r0,0,r1 /* to clear the reservation */ 932 933#if !(defined(CONFIG_4xx) || defined(CONFIG_BOOKE)) 934 andi. r10,r9,MSR_RI /* check if this exception occurred */ 935 beql nonrecoverable /* at a bad place (MSR:RI = 0) */ 936 937 lwz r10,_CCR(r1) 938 lwz r11,_LINK(r1) 939 mtcrf 0xFF,r10 940 mtlr r11 941 942 /* 943 * Once we put values in SRR0 and SRR1, we are in a state 944 * where exceptions are not recoverable, since taking an 945 * exception will trash SRR0 and SRR1. Therefore we clear the 946 * MSR:RI bit to indicate this. If we do take an exception, 947 * we can't return to the point of the exception but we 948 * can restart the exception exit path at the label 949 * exc_exit_restart below. -- paulus 950 */ 951 LOAD_MSR_KERNEL(r10,MSR_KERNEL & ~MSR_RI) 952 SYNC 953 MTMSRD(r10) /* clear the RI bit */ 954 .globl exc_exit_restart 955exc_exit_restart: 956 lwz r12,_NIP(r1) 957 FIX_SRR1(r9,r10) 958 mtspr SPRN_SRR0,r12 959 mtspr SPRN_SRR1,r9 960 REST_4GPRS(9, r1) 961 lwz r1,GPR1(r1) 962 .globl exc_exit_restart_end 963exc_exit_restart_end: 964 SYNC 965 RFI 966 967#else /* !(CONFIG_4xx || CONFIG_BOOKE) */ 968 /* 969 * This is a bit different on 4xx/Book-E because it doesn't have 970 * the RI bit in the MSR. 971 * The TLB miss handler checks if we have interrupted 972 * the exception exit path and restarts it if so 973 * (well maybe one day it will... :). 974 */ 975 lwz r11,_LINK(r1) 976 mtlr r11 977 lwz r10,_CCR(r1) 978 mtcrf 0xff,r10 979 REST_2GPRS(9, r1) 980 .globl exc_exit_restart 981exc_exit_restart: 982 lwz r11,_NIP(r1) 983 lwz r12,_MSR(r1) 984exc_exit_start: 985 mtspr SPRN_SRR0,r11 986 mtspr SPRN_SRR1,r12 987 REST_2GPRS(11, r1) 988 lwz r1,GPR1(r1) 989 .globl exc_exit_restart_end 990exc_exit_restart_end: 991 PPC405_ERR77_SYNC 992 rfi 993 b . /* prevent prefetch past rfi */ 994 995/* 996 * Returning from a critical interrupt in user mode doesn't need 997 * to be any different from a normal exception. For a critical 998 * interrupt in the kernel, we just return (without checking for 999 * preemption) since the interrupt may have happened at some crucial 1000 * place (e.g. inside the TLB miss handler), and because we will be 1001 * running with r1 pointing into critical_stack, not the current 1002 * process's kernel stack (and therefore current_thread_info() will 1003 * give the wrong answer). 1004 * We have to restore various SPRs that may have been in use at the 1005 * time of the critical interrupt. 1006 * 1007 */ 1008#ifdef CONFIG_40x 1009#define PPC_40x_TURN_OFF_MSR_DR \ 1010 /* avoid any possible TLB misses here by turning off MSR.DR, we \ 1011 * assume the instructions here are mapped by a pinned TLB entry */ \ 1012 li r10,MSR_IR; \ 1013 mtmsr r10; \ 1014 isync; \ 1015 tophys(r1, r1); 1016#else 1017#define PPC_40x_TURN_OFF_MSR_DR 1018#endif 1019 1020#define RET_FROM_EXC_LEVEL(exc_lvl_srr0, exc_lvl_srr1, exc_lvl_rfi) \ 1021 REST_NVGPRS(r1); \ 1022 lwz r3,_MSR(r1); \ 1023 andi. r3,r3,MSR_PR; \ 1024 LOAD_MSR_KERNEL(r10,MSR_KERNEL); \ 1025 bne user_exc_return; \ 1026 lwz r0,GPR0(r1); \ 1027 lwz r2,GPR2(r1); \ 1028 REST_4GPRS(3, r1); \ 1029 REST_2GPRS(7, r1); \ 1030 lwz r10,_XER(r1); \ 1031 lwz r11,_CTR(r1); \ 1032 mtspr SPRN_XER,r10; \ 1033 mtctr r11; \ 1034 PPC405_ERR77(0,r1); \ 1035 stwcx. r0,0,r1; /* to clear the reservation */ \ 1036 lwz r11,_LINK(r1); \ 1037 mtlr r11; \ 1038 lwz r10,_CCR(r1); \ 1039 mtcrf 0xff,r10; \ 1040 PPC_40x_TURN_OFF_MSR_DR; \ 1041 lwz r9,_DEAR(r1); \ 1042 lwz r10,_ESR(r1); \ 1043 mtspr SPRN_DEAR,r9; \ 1044 mtspr SPRN_ESR,r10; \ 1045 lwz r11,_NIP(r1); \ 1046 lwz r12,_MSR(r1); \ 1047 mtspr exc_lvl_srr0,r11; \ 1048 mtspr exc_lvl_srr1,r12; \ 1049 lwz r9,GPR9(r1); \ 1050 lwz r12,GPR12(r1); \ 1051 lwz r10,GPR10(r1); \ 1052 lwz r11,GPR11(r1); \ 1053 lwz r1,GPR1(r1); \ 1054 PPC405_ERR77_SYNC; \ 1055 exc_lvl_rfi; \ 1056 b .; /* prevent prefetch past exc_lvl_rfi */ 1057 1058#define RESTORE_xSRR(exc_lvl_srr0, exc_lvl_srr1) \ 1059 lwz r9,_##exc_lvl_srr0(r1); \ 1060 lwz r10,_##exc_lvl_srr1(r1); \ 1061 mtspr SPRN_##exc_lvl_srr0,r9; \ 1062 mtspr SPRN_##exc_lvl_srr1,r10; 1063 1064#if defined(CONFIG_PPC_BOOK3E_MMU) 1065#ifdef CONFIG_PHYS_64BIT 1066#define RESTORE_MAS7 \ 1067 lwz r11,MAS7(r1); \ 1068 mtspr SPRN_MAS7,r11; 1069#else 1070#define RESTORE_MAS7 1071#endif /* CONFIG_PHYS_64BIT */ 1072#define RESTORE_MMU_REGS \ 1073 lwz r9,MAS0(r1); \ 1074 lwz r10,MAS1(r1); \ 1075 lwz r11,MAS2(r1); \ 1076 mtspr SPRN_MAS0,r9; \ 1077 lwz r9,MAS3(r1); \ 1078 mtspr SPRN_MAS1,r10; \ 1079 lwz r10,MAS6(r1); \ 1080 mtspr SPRN_MAS2,r11; \ 1081 mtspr SPRN_MAS3,r9; \ 1082 mtspr SPRN_MAS6,r10; \ 1083 RESTORE_MAS7; 1084#elif defined(CONFIG_44x) 1085#define RESTORE_MMU_REGS \ 1086 lwz r9,MMUCR(r1); \ 1087 mtspr SPRN_MMUCR,r9; 1088#else 1089#define RESTORE_MMU_REGS 1090#endif 1091 1092#ifdef CONFIG_40x 1093 .globl ret_from_crit_exc 1094ret_from_crit_exc: 1095 mfspr r9,SPRN_SPRG_THREAD 1096 lis r10,saved_ksp_limit@ha; 1097 lwz r10,saved_ksp_limit@l(r10); 1098 tovirt(r9,r9); 1099 stw r10,KSP_LIMIT(r9) 1100 lis r9,crit_srr0@ha; 1101 lwz r9,crit_srr0@l(r9); 1102 lis r10,crit_srr1@ha; 1103 lwz r10,crit_srr1@l(r10); 1104 mtspr SPRN_SRR0,r9; 1105 mtspr SPRN_SRR1,r10; 1106 RET_FROM_EXC_LEVEL(SPRN_CSRR0, SPRN_CSRR1, PPC_RFCI) 1107#endif /* CONFIG_40x */ 1108 1109#ifdef CONFIG_BOOKE 1110 .globl ret_from_crit_exc 1111ret_from_crit_exc: 1112 mfspr r9,SPRN_SPRG_THREAD 1113 lwz r10,SAVED_KSP_LIMIT(r1) 1114 stw r10,KSP_LIMIT(r9) 1115 RESTORE_xSRR(SRR0,SRR1); 1116 RESTORE_MMU_REGS; 1117 RET_FROM_EXC_LEVEL(SPRN_CSRR0, SPRN_CSRR1, PPC_RFCI) 1118 1119 .globl ret_from_debug_exc 1120ret_from_debug_exc: 1121 mfspr r9,SPRN_SPRG_THREAD 1122 lwz r10,SAVED_KSP_LIMIT(r1) 1123 stw r10,KSP_LIMIT(r9) 1124 lwz r9,THREAD_INFO-THREAD(r9) 1125 rlwinm r10,r1,0,0,(31-THREAD_SHIFT) 1126 lwz r10,TI_PREEMPT(r10) 1127 stw r10,TI_PREEMPT(r9) 1128 RESTORE_xSRR(SRR0,SRR1); 1129 RESTORE_xSRR(CSRR0,CSRR1); 1130 RESTORE_MMU_REGS; 1131 RET_FROM_EXC_LEVEL(SPRN_DSRR0, SPRN_DSRR1, PPC_RFDI) 1132 1133 .globl ret_from_mcheck_exc 1134ret_from_mcheck_exc: 1135 mfspr r9,SPRN_SPRG_THREAD 1136 lwz r10,SAVED_KSP_LIMIT(r1) 1137 stw r10,KSP_LIMIT(r9) 1138 RESTORE_xSRR(SRR0,SRR1); 1139 RESTORE_xSRR(CSRR0,CSRR1); 1140 RESTORE_xSRR(DSRR0,DSRR1); 1141 RESTORE_MMU_REGS; 1142 RET_FROM_EXC_LEVEL(SPRN_MCSRR0, SPRN_MCSRR1, PPC_RFMCI) 1143#endif /* CONFIG_BOOKE */ 1144 1145/* 1146 * Load the DBCR0 value for a task that is being ptraced, 1147 * having first saved away the global DBCR0. Note that r0 1148 * has the dbcr0 value to set upon entry to this. 1149 */ 1150load_dbcr0: 1151 mfmsr r10 /* first disable debug exceptions */ 1152 rlwinm r10,r10,0,~MSR_DE 1153 mtmsr r10 1154 isync 1155 mfspr r10,SPRN_DBCR0 1156 lis r11,global_dbcr0@ha 1157 addi r11,r11,global_dbcr0@l 1158#ifdef CONFIG_SMP 1159 rlwinm r9,r1,0,0,(31-THREAD_SHIFT) 1160 lwz r9,TI_CPU(r9) 1161 slwi r9,r9,3 1162 add r11,r11,r9 1163#endif 1164 stw r10,0(r11) 1165 mtspr SPRN_DBCR0,r0 1166 lwz r10,4(r11) 1167 addi r10,r10,1 1168 stw r10,4(r11) 1169 li r11,-1 1170 mtspr SPRN_DBSR,r11 /* clear all pending debug events */ 1171 blr 1172 1173 .section .bss 1174 .align 4 1175global_dbcr0: 1176 .space 8*NR_CPUS 1177 .previous 1178#endif /* !(CONFIG_4xx || CONFIG_BOOKE) */ 1179 1180do_work: /* r10 contains MSR_KERNEL here */ 1181 andi. r0,r9,_TIF_NEED_RESCHED 1182 beq do_user_signal 1183 1184do_resched: /* r10 contains MSR_KERNEL here */ 1185 /* Note: We don't need to inform lockdep that we are enabling 1186 * interrupts here. As far as it knows, they are already enabled 1187 */ 1188 ori r10,r10,MSR_EE 1189 SYNC 1190 MTMSRD(r10) /* hard-enable interrupts */ 1191 bl schedule 1192recheck: 1193 /* Note: And we don't tell it we are disabling them again 1194 * neither. Those disable/enable cycles used to peek at 1195 * TI_FLAGS aren't advertised. 1196 */ 1197 LOAD_MSR_KERNEL(r10,MSR_KERNEL) 1198 SYNC 1199 MTMSRD(r10) /* disable interrupts */ 1200 rlwinm r9,r1,0,0,(31-THREAD_SHIFT) 1201 lwz r9,TI_FLAGS(r9) 1202 andi. r0,r9,_TIF_NEED_RESCHED 1203 bne- do_resched 1204 andi. r0,r9,_TIF_USER_WORK_MASK 1205 beq restore_user 1206do_user_signal: /* r10 contains MSR_KERNEL here */ 1207 ori r10,r10,MSR_EE 1208 SYNC 1209 MTMSRD(r10) /* hard-enable interrupts */ 1210 /* save r13-r31 in the exception frame, if not already done */ 1211 lwz r3,_TRAP(r1) 1212 andi. r0,r3,1 1213 beq 2f 1214 SAVE_NVGPRS(r1) 1215 rlwinm r3,r3,0,0,30 1216 stw r3,_TRAP(r1) 12172: addi r3,r1,STACK_FRAME_OVERHEAD 1218 mr r4,r9 1219 bl do_notify_resume 1220 REST_NVGPRS(r1) 1221 b recheck 1222 1223/* 1224 * We come here when we are at the end of handling an exception 1225 * that occurred at a place where taking an exception will lose 1226 * state information, such as the contents of SRR0 and SRR1. 1227 */ 1228nonrecoverable: 1229 lis r10,exc_exit_restart_end@ha 1230 addi r10,r10,exc_exit_restart_end@l 1231 cmplw r12,r10 1232 bge 3f 1233 lis r11,exc_exit_restart@ha 1234 addi r11,r11,exc_exit_restart@l 1235 cmplw r12,r11 1236 blt 3f 1237 lis r10,ee_restarts@ha 1238 lwz r12,ee_restarts@l(r10) 1239 addi r12,r12,1 1240 stw r12,ee_restarts@l(r10) 1241 mr r12,r11 /* restart at exc_exit_restart */ 1242 blr 12433: /* OK, we can't recover, kill this process */ 1244 /* but the 601 doesn't implement the RI bit, so assume it's OK */ 1245BEGIN_FTR_SECTION 1246 blr 1247END_FTR_SECTION_IFSET(CPU_FTR_601) 1248 lwz r3,_TRAP(r1) 1249 andi. r0,r3,1 1250 beq 4f 1251 SAVE_NVGPRS(r1) 1252 rlwinm r3,r3,0,0,30 1253 stw r3,_TRAP(r1) 12544: addi r3,r1,STACK_FRAME_OVERHEAD 1255 bl nonrecoverable_exception 1256 /* shouldn't return */ 1257 b 4b 1258 1259 .section .bss 1260 .align 2 1261ee_restarts: 1262 .space 4 1263 .previous 1264 1265/* 1266 * PROM code for specific machines follows. Put it 1267 * here so it's easy to add arch-specific sections later. 1268 * -- Cort 1269 */ 1270#ifdef CONFIG_PPC_RTAS 1271/* 1272 * On CHRP, the Run-Time Abstraction Services (RTAS) have to be 1273 * called with the MMU off. 1274 */ 1275_GLOBAL(enter_rtas) 1276 stwu r1,-INT_FRAME_SIZE(r1) 1277 mflr r0 1278 stw r0,INT_FRAME_SIZE+4(r1) 1279 LOAD_REG_ADDR(r4, rtas) 1280 lis r6,1f@ha /* physical return address for rtas */ 1281 addi r6,r6,1f@l 1282 tophys(r6,r6) 1283 tophys(r7,r1) 1284 lwz r8,RTASENTRY(r4) 1285 lwz r4,RTASBASE(r4) 1286 mfmsr r9 1287 stw r9,8(r1) 1288 LOAD_MSR_KERNEL(r0,MSR_KERNEL) 1289 SYNC /* disable interrupts so SRR0/1 */ 1290 MTMSRD(r0) /* don't get trashed */ 1291 li r9,MSR_KERNEL & ~(MSR_IR|MSR_DR) 1292 mtlr r6 1293 mtspr SPRN_SPRG_RTAS,r7 1294 mtspr SPRN_SRR0,r8 1295 mtspr SPRN_SRR1,r9 1296 RFI 12971: tophys(r9,r1) 1298 lwz r8,INT_FRAME_SIZE+4(r9) /* get return address */ 1299 lwz r9,8(r9) /* original msr value */ 1300 FIX_SRR1(r9,r0) 1301 addi r1,r1,INT_FRAME_SIZE 1302 li r0,0 1303 mtspr SPRN_SPRG_RTAS,r0 1304 mtspr SPRN_SRR0,r8 1305 mtspr SPRN_SRR1,r9 1306 RFI /* return to caller */ 1307 1308 .globl machine_check_in_rtas 1309machine_check_in_rtas: 1310 twi 31,0,0 1311 /* XXX load up BATs and panic */ 1312 1313#endif /* CONFIG_PPC_RTAS */ 1314 1315#ifdef CONFIG_FUNCTION_TRACER 1316#ifdef CONFIG_DYNAMIC_FTRACE 1317_GLOBAL(mcount) 1318_GLOBAL(_mcount) 1319 /* 1320 * It is required that _mcount on PPC32 must preserve the 1321 * link register. But we have r0 to play with. We use r0 1322 * to push the return address back to the caller of mcount 1323 * into the ctr register, restore the link register and 1324 * then jump back using the ctr register. 1325 */ 1326 mflr r0 1327 mtctr r0 1328 lwz r0, 4(r1) 1329 mtlr r0 1330 bctr 1331 1332_GLOBAL(ftrace_caller) 1333 MCOUNT_SAVE_FRAME 1334 /* r3 ends up with link register */ 1335 subi r3, r3, MCOUNT_INSN_SIZE 1336.globl ftrace_call 1337ftrace_call: 1338 bl ftrace_stub 1339 nop 1340#ifdef CONFIG_FUNCTION_GRAPH_TRACER 1341.globl ftrace_graph_call 1342ftrace_graph_call: 1343 b ftrace_graph_stub 1344_GLOBAL(ftrace_graph_stub) 1345#endif 1346 MCOUNT_RESTORE_FRAME 1347 /* old link register ends up in ctr reg */ 1348 bctr 1349#else 1350_GLOBAL(mcount) 1351_GLOBAL(_mcount) 1352 1353 MCOUNT_SAVE_FRAME 1354 1355 subi r3, r3, MCOUNT_INSN_SIZE 1356 LOAD_REG_ADDR(r5, ftrace_trace_function) 1357 lwz r5,0(r5) 1358 1359 mtctr r5 1360 bctrl 1361 nop 1362 1363#ifdef CONFIG_FUNCTION_GRAPH_TRACER 1364 b ftrace_graph_caller 1365#endif 1366 MCOUNT_RESTORE_FRAME 1367 bctr 1368#endif 1369 1370_GLOBAL(ftrace_stub) 1371 blr 1372 1373#ifdef CONFIG_FUNCTION_GRAPH_TRACER 1374_GLOBAL(ftrace_graph_caller) 1375 /* load r4 with local address */ 1376 lwz r4, 44(r1) 1377 subi r4, r4, MCOUNT_INSN_SIZE 1378 1379 /* get the parent address */ 1380 addi r3, r1, 52 1381 1382 bl prepare_ftrace_return 1383 nop 1384 1385 MCOUNT_RESTORE_FRAME 1386 /* old link register ends up in ctr reg */ 1387 bctr 1388 1389_GLOBAL(return_to_handler) 1390 /* need to save return values */ 1391 stwu r1, -32(r1) 1392 stw r3, 20(r1) 1393 stw r4, 16(r1) 1394 stw r31, 12(r1) 1395 mr r31, r1 1396 1397 bl ftrace_return_to_handler 1398 nop 1399 1400 /* return value has real return address */ 1401 mtlr r3 1402 1403 lwz r3, 20(r1) 1404 lwz r4, 16(r1) 1405 lwz r31,12(r1) 1406 lwz r1, 0(r1) 1407 1408 /* Jump back to real return address */ 1409 blr 1410#endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 1411 1412#endif /* CONFIG_MCOUNT */ 1413