• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 #ifndef _ASM_POWERPC_EXCEPTION_H
2 #define _ASM_POWERPC_EXCEPTION_H
3 /*
4  * Extracted from head_64.S
5  *
6  *  PowerPC version
7  *    Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
8  *
9  *  Rewritten by Cort Dougan (cort@cs.nmt.edu) for PReP
10  *    Copyright (C) 1996 Cort Dougan <cort@cs.nmt.edu>
11  *  Adapted for Power Macintosh by Paul Mackerras.
12  *  Low-level exception handlers and MMU support
13  *  rewritten by Paul Mackerras.
14  *    Copyright (C) 1996 Paul Mackerras.
15  *
16  *  Adapted for 64bit PowerPC by Dave Engebretsen, Peter Bergner, and
17  *    Mike Corrigan {engebret|bergner|mikejc}@us.ibm.com
18  *
19  *  This file contains the low-level support and setup for the
20  *  PowerPC-64 platform, including trap and interrupt dispatch.
21  *
22  *  This program is free software; you can redistribute it and/or
23  *  modify it under the terms of the GNU General Public License
24  *  as published by the Free Software Foundation; either version
25  *  2 of the License, or (at your option) any later version.
26  */
27 /*
28  * The following macros define the code that appears as
29  * the prologue to each of the exception handlers.  They
30  * are split into two parts to allow a single kernel binary
31  * to be used for pSeries and iSeries.
32  *
33  * We make as much of the exception code common between native
34  * exception handlers (including pSeries LPAR) and iSeries LPAR
35  * implementations as possible.
36  */
37 #include <asm/head-64.h>
38 
39 /* PACA save area offsets (exgen, exmc, etc) */
40 #define EX_R9		0
41 #define EX_R10		8
42 #define EX_R11		16
43 #define EX_R12		24
44 #define EX_R13		32
45 #define EX_DAR		40
46 #define EX_DSISR	48
47 #define EX_CCR		52
48 #define EX_CFAR		56
49 #define EX_PPR		64
50 #if defined(CONFIG_RELOCATABLE)
51 #define EX_CTR		72
52 #define EX_SIZE		10	/* size in u64 units */
53 #else
54 #define EX_SIZE		9	/* size in u64 units */
55 #endif
56 
57 /*
58  * EX_LR is only used in EXSLB and where it does not overlap with EX_DAR
59  * EX_CCR similarly with DSISR, but being 4 byte registers there is a hole
60  * in the save area so it's not necessary to overlap them. Could be used
61  * for future savings though if another 4 byte register was to be saved.
62  */
63 #define EX_LR		EX_DAR
64 
65 /*
66  * EX_R3 is only used by the bad_stack handler. bad_stack reloads and
67  * saves DAR from SPRN_DAR, and EX_DAR is not used. So EX_R3 can overlap
68  * with EX_DAR.
69  */
70 #define EX_R3		EX_DAR
71 
72 #define STF_ENTRY_BARRIER_SLOT						\
73 	STF_ENTRY_BARRIER_FIXUP_SECTION;				\
74 	nop;								\
75 	nop;								\
76 	nop
77 
78 #define STF_EXIT_BARRIER_SLOT						\
79 	STF_EXIT_BARRIER_FIXUP_SECTION;					\
80 	nop;								\
81 	nop;								\
82 	nop;								\
83 	nop;								\
84 	nop;								\
85 	nop
86 
87 /*
88  * r10 must be free to use, r13 must be paca
89  */
90 #define INTERRUPT_TO_KERNEL						\
91 	STF_ENTRY_BARRIER_SLOT
92 
93 /*
94  * Macros for annotating the expected destination of (h)rfid
95  *
96  * The nop instructions allow us to insert one or more instructions to flush the
97  * L1-D cache when returning to userspace or a guest.
98  */
99 #define RFI_FLUSH_SLOT							\
100 	RFI_FLUSH_FIXUP_SECTION;					\
101 	nop;								\
102 	nop;								\
103 	nop
104 
105 #define RFI_TO_KERNEL							\
106 	rfid
107 
108 #define RFI_TO_USER							\
109 	STF_EXIT_BARRIER_SLOT;						\
110 	RFI_FLUSH_SLOT;							\
111 	rfid;								\
112 	b	rfi_flush_fallback
113 
114 #define RFI_TO_USER_OR_KERNEL						\
115 	STF_EXIT_BARRIER_SLOT;						\
116 	RFI_FLUSH_SLOT;							\
117 	rfid;								\
118 	b	rfi_flush_fallback
119 
120 #define RFI_TO_GUEST							\
121 	STF_EXIT_BARRIER_SLOT;						\
122 	RFI_FLUSH_SLOT;							\
123 	rfid;								\
124 	b	rfi_flush_fallback
125 
126 #define HRFI_TO_KERNEL							\
127 	hrfid
128 
129 #define HRFI_TO_USER							\
130 	STF_EXIT_BARRIER_SLOT;						\
131 	RFI_FLUSH_SLOT;							\
132 	hrfid;								\
133 	b	hrfi_flush_fallback
134 
135 #define HRFI_TO_USER_OR_KERNEL						\
136 	STF_EXIT_BARRIER_SLOT;						\
137 	RFI_FLUSH_SLOT;							\
138 	hrfid;								\
139 	b	hrfi_flush_fallback
140 
141 #define HRFI_TO_GUEST							\
142 	STF_EXIT_BARRIER_SLOT;						\
143 	RFI_FLUSH_SLOT;							\
144 	hrfid;								\
145 	b	hrfi_flush_fallback
146 
147 #define HRFI_TO_UNKNOWN							\
148 	STF_EXIT_BARRIER_SLOT;						\
149 	RFI_FLUSH_SLOT;							\
150 	hrfid;								\
151 	b	hrfi_flush_fallback
152 
153 #ifdef CONFIG_RELOCATABLE
154 #define __EXCEPTION_RELON_PROLOG_PSERIES_1(label, h)			\
155 	mfspr	r11,SPRN_##h##SRR0;	/* save SRR0 */			\
156 	LOAD_HANDLER(r12,label);					\
157 	mtctr	r12;							\
158 	mfspr	r12,SPRN_##h##SRR1;	/* and SRR1 */			\
159 	li	r10,MSR_RI;						\
160 	mtmsrd 	r10,1;			/* Set RI (EE=0) */		\
161 	bctr;
162 #else
163 /* If not relocatable, we can jump directly -- and save messing with LR */
164 #define __EXCEPTION_RELON_PROLOG_PSERIES_1(label, h)			\
165 	mfspr	r11,SPRN_##h##SRR0;	/* save SRR0 */			\
166 	mfspr	r12,SPRN_##h##SRR1;	/* and SRR1 */			\
167 	li	r10,MSR_RI;						\
168 	mtmsrd 	r10,1;			/* Set RI (EE=0) */		\
169 	b	label;
170 #endif
171 #define EXCEPTION_RELON_PROLOG_PSERIES_1(label, h)			\
172 	__EXCEPTION_RELON_PROLOG_PSERIES_1(label, h)			\
173 
174 /*
175  * As EXCEPTION_PROLOG_PSERIES(), except we've already got relocation on
176  * so no need to rfid.  Save lr in case we're CONFIG_RELOCATABLE, in which
177  * case EXCEPTION_RELON_PROLOG_PSERIES_1 will be using lr.
178  */
179 #define EXCEPTION_RELON_PROLOG_PSERIES(area, label, h, extra, vec)	\
180 	EXCEPTION_PROLOG_0(area);					\
181 	EXCEPTION_PROLOG_1(area, extra, vec);				\
182 	EXCEPTION_RELON_PROLOG_PSERIES_1(label, h)
183 
184 /*
185  * We're short on space and time in the exception prolog, so we can't
186  * use the normal LOAD_REG_IMMEDIATE macro to load the address of label.
187  * Instead we get the base of the kernel from paca->kernelbase and or in the low
188  * part of label. This requires that the label be within 64KB of kernelbase, and
189  * that kernelbase be 64K aligned.
190  */
191 #define LOAD_HANDLER(reg, label)					\
192 	ld	reg,PACAKBASE(r13);	/* get high part of &label */	\
193 	ori	reg,reg,FIXED_SYMBOL_ABS_ADDR(label);
194 
195 #define __LOAD_HANDLER(reg, label)					\
196 	ld	reg,PACAKBASE(r13);					\
197 	ori	reg,reg,(ABS_ADDR(label))@l;
198 
199 /*
200  * Branches from unrelocated code (e.g., interrupts) to labels outside
201  * head-y require >64K offsets.
202  */
203 #define __LOAD_FAR_HANDLER(reg, label)					\
204 	ld	reg,PACAKBASE(r13);					\
205 	ori	reg,reg,(ABS_ADDR(label))@l;				\
206 	addis	reg,reg,(ABS_ADDR(label))@h;
207 
208 /* Exception register prefixes */
209 #define EXC_HV	H
210 #define EXC_STD
211 
212 #if defined(CONFIG_RELOCATABLE)
213 /*
214  * If we support interrupts with relocation on AND we're a relocatable kernel,
215  * we need to use CTR to get to the 2nd level handler.  So, save/restore it
216  * when required.
217  */
218 #define SAVE_CTR(reg, area)	mfctr	reg ; 	std	reg,area+EX_CTR(r13)
219 #define GET_CTR(reg, area) 			ld	reg,area+EX_CTR(r13)
220 #define RESTORE_CTR(reg, area)	ld	reg,area+EX_CTR(r13) ; mtctr reg
221 #else
222 /* ...else CTR is unused and in register. */
223 #define SAVE_CTR(reg, area)
224 #define GET_CTR(reg, area) 	mfctr	reg
225 #define RESTORE_CTR(reg, area)
226 #endif
227 
228 /*
229  * PPR save/restore macros used in exceptions_64s.S
230  * Used for P7 or later processors
231  */
232 #define SAVE_PPR(area, ra, rb)						\
233 BEGIN_FTR_SECTION_NESTED(940)						\
234 	ld	ra,PACACURRENT(r13);					\
235 	ld	rb,area+EX_PPR(r13);	/* Read PPR from paca */	\
236 	std	rb,TASKTHREADPPR(ra);					\
237 END_FTR_SECTION_NESTED(CPU_FTR_HAS_PPR,CPU_FTR_HAS_PPR,940)
238 
239 #define RESTORE_PPR_PACA(area, ra)					\
240 BEGIN_FTR_SECTION_NESTED(941)						\
241 	ld	ra,area+EX_PPR(r13);					\
242 	mtspr	SPRN_PPR,ra;						\
243 END_FTR_SECTION_NESTED(CPU_FTR_HAS_PPR,CPU_FTR_HAS_PPR,941)
244 
245 /*
246  * Get an SPR into a register if the CPU has the given feature
247  */
248 #define OPT_GET_SPR(ra, spr, ftr)					\
249 BEGIN_FTR_SECTION_NESTED(943)						\
250 	mfspr	ra,spr;							\
251 END_FTR_SECTION_NESTED(ftr,ftr,943)
252 
253 /*
254  * Set an SPR from a register if the CPU has the given feature
255  */
256 #define OPT_SET_SPR(ra, spr, ftr)					\
257 BEGIN_FTR_SECTION_NESTED(943)						\
258 	mtspr	spr,ra;							\
259 END_FTR_SECTION_NESTED(ftr,ftr,943)
260 
261 /*
262  * Save a register to the PACA if the CPU has the given feature
263  */
264 #define OPT_SAVE_REG_TO_PACA(offset, ra, ftr)				\
265 BEGIN_FTR_SECTION_NESTED(943)						\
266 	std	ra,offset(r13);						\
267 END_FTR_SECTION_NESTED(ftr,ftr,943)
268 
269 #define EXCEPTION_PROLOG_0(area)					\
270 	GET_PACA(r13);							\
271 	std	r9,area+EX_R9(r13);	/* save r9 */			\
272 	OPT_GET_SPR(r9, SPRN_PPR, CPU_FTR_HAS_PPR);			\
273 	HMT_MEDIUM;							\
274 	std	r10,area+EX_R10(r13);	/* save r10 - r12 */		\
275 	OPT_GET_SPR(r10, SPRN_CFAR, CPU_FTR_CFAR)
276 
277 #define __EXCEPTION_PROLOG_1(area, extra, vec)				\
278 	OPT_SAVE_REG_TO_PACA(area+EX_PPR, r9, CPU_FTR_HAS_PPR);		\
279 	OPT_SAVE_REG_TO_PACA(area+EX_CFAR, r10, CPU_FTR_CFAR);		\
280 	INTERRUPT_TO_KERNEL;						\
281 	SAVE_CTR(r10, area);						\
282 	mfcr	r9;							\
283 	extra(vec);							\
284 	std	r11,area+EX_R11(r13);					\
285 	std	r12,area+EX_R12(r13);					\
286 	GET_SCRATCH0(r10);						\
287 	std	r10,area+EX_R13(r13)
288 #define EXCEPTION_PROLOG_1(area, extra, vec)				\
289 	__EXCEPTION_PROLOG_1(area, extra, vec)
290 
291 #define __EXCEPTION_PROLOG_PSERIES_1(label, h)				\
292 	ld	r10,PACAKMSR(r13);	/* get MSR value for kernel */	\
293 	mfspr	r11,SPRN_##h##SRR0;	/* save SRR0 */			\
294 	LOAD_HANDLER(r12,label)						\
295 	mtspr	SPRN_##h##SRR0,r12;					\
296 	mfspr	r12,SPRN_##h##SRR1;	/* and SRR1 */			\
297 	mtspr	SPRN_##h##SRR1,r10;					\
298 	h##RFI_TO_KERNEL;						\
299 	b	.	/* prevent speculative execution */
300 #define EXCEPTION_PROLOG_PSERIES_1(label, h)				\
301 	__EXCEPTION_PROLOG_PSERIES_1(label, h)
302 
303 /* _NORI variant keeps MSR_RI clear */
304 #define __EXCEPTION_PROLOG_PSERIES_1_NORI(label, h)			\
305 	ld	r10,PACAKMSR(r13);	/* get MSR value for kernel */	\
306 	xori	r10,r10,MSR_RI;		/* Clear MSR_RI */		\
307 	mfspr	r11,SPRN_##h##SRR0;	/* save SRR0 */			\
308 	LOAD_HANDLER(r12,label)						\
309 	mtspr	SPRN_##h##SRR0,r12;					\
310 	mfspr	r12,SPRN_##h##SRR1;	/* and SRR1 */			\
311 	mtspr	SPRN_##h##SRR1,r10;					\
312 	h##RFI_TO_KERNEL;						\
313 	b	.	/* prevent speculative execution */
314 
315 #define EXCEPTION_PROLOG_PSERIES_1_NORI(label, h)			\
316 	__EXCEPTION_PROLOG_PSERIES_1_NORI(label, h)
317 
318 #define EXCEPTION_PROLOG_PSERIES(area, label, h, extra, vec)		\
319 	EXCEPTION_PROLOG_0(area);					\
320 	EXCEPTION_PROLOG_1(area, extra, vec);				\
321 	EXCEPTION_PROLOG_PSERIES_1(label, h);
322 
323 #define __KVMTEST(h, n)							\
324 	lbz	r10,HSTATE_IN_GUEST(r13);				\
325 	cmpwi	r10,0;							\
326 	bne	do_kvm_##h##n
327 
328 #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
329 /*
330  * If hv is possible, interrupts come into to the hv version
331  * of the kvmppc_interrupt code, which then jumps to the PR handler,
332  * kvmppc_interrupt_pr, if the guest is a PR guest.
333  */
334 #define kvmppc_interrupt kvmppc_interrupt_hv
335 #else
336 #define kvmppc_interrupt kvmppc_interrupt_pr
337 #endif
338 
339 /*
340  * Branch to label using its 0xC000 address. This results in instruction
341  * address suitable for MSR[IR]=0 or 1, which allows relocation to be turned
342  * on using mtmsr rather than rfid.
343  *
344  * This could set the 0xc bits for !RELOCATABLE as an immediate, rather than
345  * load KBASE for a slight optimisation.
346  */
347 #define BRANCH_TO_C000(reg, label)					\
348 	__LOAD_HANDLER(reg, label);					\
349 	mtctr	reg;							\
350 	bctr
351 
352 #ifdef CONFIG_RELOCATABLE
353 #define BRANCH_TO_COMMON(reg, label)					\
354 	__LOAD_HANDLER(reg, label);					\
355 	mtctr	reg;							\
356 	bctr
357 
358 #define BRANCH_LINK_TO_FAR(label)					\
359 	__LOAD_FAR_HANDLER(r12, label);					\
360 	mtctr	r12;							\
361 	bctrl
362 
363 /*
364  * KVM requires __LOAD_FAR_HANDLER.
365  *
366  * __BRANCH_TO_KVM_EXIT branches are also a special case because they
367  * explicitly use r9 then reload it from PACA before branching. Hence
368  * the double-underscore.
369  */
370 #define __BRANCH_TO_KVM_EXIT(area, label)				\
371 	mfctr	r9;							\
372 	std	r9,HSTATE_SCRATCH1(r13);				\
373 	__LOAD_FAR_HANDLER(r9, label);					\
374 	mtctr	r9;							\
375 	ld	r9,area+EX_R9(r13);					\
376 	bctr
377 
378 #else
379 #define BRANCH_TO_COMMON(reg, label)					\
380 	b	label
381 
382 #define BRANCH_LINK_TO_FAR(label)					\
383 	bl	label
384 
385 #define __BRANCH_TO_KVM_EXIT(area, label)				\
386 	ld	r9,area+EX_R9(r13);					\
387 	b	label
388 
389 #endif
390 
391 /* Do not enable RI */
392 #define EXCEPTION_PROLOG_PSERIES_NORI(area, label, h, extra, vec)	\
393 	EXCEPTION_PROLOG_0(area);					\
394 	EXCEPTION_PROLOG_1(area, extra, vec);				\
395 	EXCEPTION_PROLOG_PSERIES_1_NORI(label, h);
396 
397 
398 #define __KVM_HANDLER(area, h, n)					\
399 	BEGIN_FTR_SECTION_NESTED(947)					\
400 	ld	r10,area+EX_CFAR(r13);					\
401 	std	r10,HSTATE_CFAR(r13);					\
402 	END_FTR_SECTION_NESTED(CPU_FTR_CFAR,CPU_FTR_CFAR,947);		\
403 	BEGIN_FTR_SECTION_NESTED(948)					\
404 	ld	r10,area+EX_PPR(r13);					\
405 	std	r10,HSTATE_PPR(r13);					\
406 	END_FTR_SECTION_NESTED(CPU_FTR_HAS_PPR,CPU_FTR_HAS_PPR,948);	\
407 	ld	r10,area+EX_R10(r13);					\
408 	std	r12,HSTATE_SCRATCH0(r13);				\
409 	sldi	r12,r9,32;						\
410 	ori	r12,r12,(n);						\
411 	/* This reloads r9 before branching to kvmppc_interrupt */	\
412 	__BRANCH_TO_KVM_EXIT(area, kvmppc_interrupt)
413 
414 #define __KVM_HANDLER_SKIP(area, h, n)					\
415 	cmpwi	r10,KVM_GUEST_MODE_SKIP;				\
416 	beq	89f;							\
417 	BEGIN_FTR_SECTION_NESTED(948)					\
418 	ld	r10,area+EX_PPR(r13);					\
419 	std	r10,HSTATE_PPR(r13);					\
420 	END_FTR_SECTION_NESTED(CPU_FTR_HAS_PPR,CPU_FTR_HAS_PPR,948);	\
421 	ld	r10,area+EX_R10(r13);					\
422 	std	r12,HSTATE_SCRATCH0(r13);				\
423 	sldi	r12,r9,32;						\
424 	ori	r12,r12,(n);						\
425 	/* This reloads r9 before branching to kvmppc_interrupt */	\
426 	__BRANCH_TO_KVM_EXIT(area, kvmppc_interrupt);			\
427 89:	mtocrf	0x80,r9;						\
428 	ld	r9,area+EX_R9(r13);					\
429 	ld	r10,area+EX_R10(r13);					\
430 	b	kvmppc_skip_##h##interrupt
431 
432 #ifdef CONFIG_KVM_BOOK3S_64_HANDLER
433 #define KVMTEST(h, n)			__KVMTEST(h, n)
434 #define KVM_HANDLER(area, h, n)		__KVM_HANDLER(area, h, n)
435 #define KVM_HANDLER_SKIP(area, h, n)	__KVM_HANDLER_SKIP(area, h, n)
436 
437 #else
438 #define KVMTEST(h, n)
439 #define KVM_HANDLER(area, h, n)
440 #define KVM_HANDLER_SKIP(area, h, n)
441 #endif
442 
443 #define NOTEST(n)
444 
445 #define EXCEPTION_PROLOG_COMMON_1()					   \
446 	std	r9,_CCR(r1);		/* save CR in stackframe	*/ \
447 	std	r11,_NIP(r1);		/* save SRR0 in stackframe	*/ \
448 	std	r12,_MSR(r1);		/* save SRR1 in stackframe	*/ \
449 	std	r10,0(r1);		/* make stack chain pointer	*/ \
450 	std	r0,GPR0(r1);		/* save r0 in stackframe	*/ \
451 	std	r10,GPR1(r1);		/* save r1 in stackframe	*/ \
452 
453 
454 /*
455  * The common exception prolog is used for all except a few exceptions
456  * such as a segment miss on a kernel address.  We have to be prepared
457  * to take another exception from the point where we first touch the
458  * kernel stack onwards.
459  *
460  * On entry r13 points to the paca, r9-r13 are saved in the paca,
461  * r9 contains the saved CR, r11 and r12 contain the saved SRR0 and
462  * SRR1, and relocation is on.
463  */
464 #define EXCEPTION_PROLOG_COMMON(n, area)				   \
465 	andi.	r10,r12,MSR_PR;		/* See if coming from user	*/ \
466 	mr	r10,r1;			/* Save r1			*/ \
467 	subi	r1,r1,INT_FRAME_SIZE;	/* alloc frame on kernel stack	*/ \
468 	beq-	1f;							   \
469 	ld	r1,PACAKSAVE(r13);	/* kernel stack to use		*/ \
470 1:	cmpdi	cr1,r1,-INT_FRAME_SIZE;	/* check if r1 is in userspace	*/ \
471 	blt+	cr1,3f;			/* abort if it is		*/ \
472 	li	r1,(n);			/* will be reloaded later	*/ \
473 	sth	r1,PACA_TRAP_SAVE(r13);					   \
474 	std	r3,area+EX_R3(r13);					   \
475 	addi	r3,r13,area;		/* r3 -> where regs are saved*/	   \
476 	RESTORE_CTR(r1, area);						   \
477 	b	bad_stack;						   \
478 3:	EXCEPTION_PROLOG_COMMON_1();					   \
479 	beq	4f;			/* if from kernel mode		*/ \
480 	ACCOUNT_CPU_USER_ENTRY(r13, r9, r10);				   \
481 	SAVE_PPR(area, r9, r10);					   \
482 4:	EXCEPTION_PROLOG_COMMON_2(area)					   \
483 	EXCEPTION_PROLOG_COMMON_3(n)					   \
484 	ACCOUNT_STOLEN_TIME
485 
486 /* Save original regs values from save area to stack frame. */
487 #define EXCEPTION_PROLOG_COMMON_2(area)					   \
488 	ld	r9,area+EX_R9(r13);	/* move r9, r10 to stackframe	*/ \
489 	ld	r10,area+EX_R10(r13);					   \
490 	std	r9,GPR9(r1);						   \
491 	std	r10,GPR10(r1);						   \
492 	ld	r9,area+EX_R11(r13);	/* move r11 - r13 to stackframe	*/ \
493 	ld	r10,area+EX_R12(r13);					   \
494 	ld	r11,area+EX_R13(r13);					   \
495 	std	r9,GPR11(r1);						   \
496 	std	r10,GPR12(r1);						   \
497 	std	r11,GPR13(r1);						   \
498 	BEGIN_FTR_SECTION_NESTED(66);					   \
499 	ld	r10,area+EX_CFAR(r13);					   \
500 	std	r10,ORIG_GPR3(r1);					   \
501 	END_FTR_SECTION_NESTED(CPU_FTR_CFAR, CPU_FTR_CFAR, 66);		   \
502 	GET_CTR(r10, area);						   \
503 	std	r10,_CTR(r1);
504 
505 #define EXCEPTION_PROLOG_COMMON_3(n)					   \
506 	std	r2,GPR2(r1);		/* save r2 in stackframe	*/ \
507 	SAVE_4GPRS(3, r1);		/* save r3 - r6 in stackframe   */ \
508 	SAVE_2GPRS(7, r1);		/* save r7, r8 in stackframe	*/ \
509 	mflr	r9;			/* Get LR, later save to stack	*/ \
510 	ld	r2,PACATOC(r13);	/* get kernel TOC into r2	*/ \
511 	std	r9,_LINK(r1);						   \
512 	lbz	r10,PACASOFTIRQEN(r13);				   \
513 	mfspr	r11,SPRN_XER;		/* save XER in stackframe	*/ \
514 	std	r10,SOFTE(r1);						   \
515 	std	r11,_XER(r1);						   \
516 	li	r9,(n)+1;						   \
517 	std	r9,_TRAP(r1);		/* set trap number		*/ \
518 	li	r10,0;							   \
519 	ld	r11,exception_marker@toc(r2);				   \
520 	std	r10,RESULT(r1);		/* clear regs->result		*/ \
521 	std	r11,STACK_FRAME_OVERHEAD-16(r1); /* mark the frame	*/
522 
523 /*
524  * Exception vectors.
525  */
526 #define STD_EXCEPTION_PSERIES(vec, label)			\
527 	SET_SCRATCH0(r13);		/* save r13 */		\
528 	EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, label,		\
529 				 EXC_STD, KVMTEST_PR, vec);	\
530 
531 /* Version of above for when we have to branch out-of-line */
532 #define __OOL_EXCEPTION(vec, label, hdlr)			\
533 	SET_SCRATCH0(r13)					\
534 	EXCEPTION_PROLOG_0(PACA_EXGEN)				\
535 	b hdlr;
536 
537 #define STD_EXCEPTION_PSERIES_OOL(vec, label)			\
538 	EXCEPTION_PROLOG_1(PACA_EXGEN, KVMTEST_PR, vec);	\
539 	EXCEPTION_PROLOG_PSERIES_1(label, EXC_STD)
540 
541 #define STD_EXCEPTION_HV(loc, vec, label)			\
542 	SET_SCRATCH0(r13);	/* save r13 */			\
543 	EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, label,		\
544 				 EXC_HV, KVMTEST_HV, vec);
545 
546 #define STD_EXCEPTION_HV_OOL(vec, label)			\
547 	EXCEPTION_PROLOG_1(PACA_EXGEN, KVMTEST_HV, vec);	\
548 	EXCEPTION_PROLOG_PSERIES_1(label, EXC_HV)
549 
550 #define STD_RELON_EXCEPTION_PSERIES(loc, vec, label)	\
551 	/* No guest interrupts come through here */	\
552 	SET_SCRATCH0(r13);		/* save r13 */	\
553 	EXCEPTION_RELON_PROLOG_PSERIES(PACA_EXGEN, label, EXC_STD, NOTEST, vec);
554 
555 #define STD_RELON_EXCEPTION_PSERIES_OOL(vec, label)		\
556 	EXCEPTION_PROLOG_1(PACA_EXGEN, NOTEST, vec);		\
557 	EXCEPTION_RELON_PROLOG_PSERIES_1(label, EXC_STD)
558 
559 #define STD_RELON_EXCEPTION_HV(loc, vec, label)		\
560 	SET_SCRATCH0(r13);	/* save r13 */		\
561 	EXCEPTION_RELON_PROLOG_PSERIES(PACA_EXGEN, label,	\
562 				       EXC_HV, KVMTEST_HV, vec);
563 
564 #define STD_RELON_EXCEPTION_HV_OOL(vec, label)			\
565 	EXCEPTION_PROLOG_1(PACA_EXGEN, KVMTEST_HV, vec);	\
566 	EXCEPTION_RELON_PROLOG_PSERIES_1(label, EXC_HV)
567 
568 /* This associate vector numbers with bits in paca->irq_happened */
569 #define SOFTEN_VALUE_0x500	PACA_IRQ_EE
570 #define SOFTEN_VALUE_0x900	PACA_IRQ_DEC
571 #define SOFTEN_VALUE_0x980	PACA_IRQ_DEC
572 #define SOFTEN_VALUE_0xa00	PACA_IRQ_DBELL
573 #define SOFTEN_VALUE_0xe80	PACA_IRQ_DBELL
574 #define SOFTEN_VALUE_0xe60	PACA_IRQ_HMI
575 #define SOFTEN_VALUE_0xea0	PACA_IRQ_EE
576 
577 #define __SOFTEN_TEST(h, vec)						\
578 	lbz	r10,PACASOFTIRQEN(r13);					\
579 	cmpwi	r10,0;							\
580 	li	r10,SOFTEN_VALUE_##vec;					\
581 	beq	masked_##h##interrupt
582 
583 #define _SOFTEN_TEST(h, vec)	__SOFTEN_TEST(h, vec)
584 
585 #define SOFTEN_TEST_PR(vec)						\
586 	KVMTEST(EXC_STD, vec);						\
587 	_SOFTEN_TEST(EXC_STD, vec)
588 
589 #define SOFTEN_TEST_HV(vec)						\
590 	KVMTEST(EXC_HV, vec);						\
591 	_SOFTEN_TEST(EXC_HV, vec)
592 
593 #define KVMTEST_PR(vec)							\
594 	KVMTEST(EXC_STD, vec)
595 
596 #define KVMTEST_HV(vec)							\
597 	KVMTEST(EXC_HV, vec)
598 
599 #define SOFTEN_NOTEST_PR(vec)		_SOFTEN_TEST(EXC_STD, vec)
600 #define SOFTEN_NOTEST_HV(vec)		_SOFTEN_TEST(EXC_HV, vec)
601 
602 #define __MASKABLE_EXCEPTION_PSERIES(vec, label, h, extra)		\
603 	SET_SCRATCH0(r13);    /* save r13 */				\
604 	EXCEPTION_PROLOG_0(PACA_EXGEN);					\
605 	__EXCEPTION_PROLOG_1(PACA_EXGEN, extra, vec);			\
606 	EXCEPTION_PROLOG_PSERIES_1(label, h);
607 
608 #define _MASKABLE_EXCEPTION_PSERIES(vec, label, h, extra)		\
609 	__MASKABLE_EXCEPTION_PSERIES(vec, label, h, extra)
610 
611 #define MASKABLE_EXCEPTION_PSERIES(loc, vec, label)			\
612 	_MASKABLE_EXCEPTION_PSERIES(vec, label,				\
613 				    EXC_STD, SOFTEN_TEST_PR)
614 
615 #define MASKABLE_EXCEPTION_PSERIES_OOL(vec, label)			\
616 	EXCEPTION_PROLOG_1(PACA_EXGEN, SOFTEN_TEST_PR, vec);		\
617 	EXCEPTION_PROLOG_PSERIES_1(label, EXC_STD)
618 
619 #define MASKABLE_EXCEPTION_HV(loc, vec, label)				\
620 	_MASKABLE_EXCEPTION_PSERIES(vec, label,				\
621 				    EXC_HV, SOFTEN_TEST_HV)
622 
623 #define MASKABLE_EXCEPTION_HV_OOL(vec, label)				\
624 	EXCEPTION_PROLOG_1(PACA_EXGEN, SOFTEN_TEST_HV, vec);		\
625 	EXCEPTION_PROLOG_PSERIES_1(label, EXC_HV)
626 
627 #define __MASKABLE_RELON_EXCEPTION_PSERIES(vec, label, h, extra)	\
628 	SET_SCRATCH0(r13);    /* save r13 */				\
629 	EXCEPTION_PROLOG_0(PACA_EXGEN);					\
630 	__EXCEPTION_PROLOG_1(PACA_EXGEN, extra, vec);			\
631 	EXCEPTION_RELON_PROLOG_PSERIES_1(label, h)
632 
633 #define _MASKABLE_RELON_EXCEPTION_PSERIES(vec, label, h, extra)		\
634 	__MASKABLE_RELON_EXCEPTION_PSERIES(vec, label, h, extra)
635 
636 #define MASKABLE_RELON_EXCEPTION_PSERIES(loc, vec, label)		\
637 	_MASKABLE_RELON_EXCEPTION_PSERIES(vec, label,			\
638 					  EXC_STD, SOFTEN_NOTEST_PR)
639 
640 #define MASKABLE_RELON_EXCEPTION_HV(loc, vec, label)			\
641 	_MASKABLE_RELON_EXCEPTION_PSERIES(vec, label,			\
642 					  EXC_HV, SOFTEN_TEST_HV)
643 
644 #define MASKABLE_RELON_EXCEPTION_HV_OOL(vec, label)			\
645 	EXCEPTION_PROLOG_1(PACA_EXGEN, SOFTEN_TEST_HV, vec);		\
646 	EXCEPTION_RELON_PROLOG_PSERIES_1(label, EXC_HV)
647 
648 /*
649  * Our exception common code can be passed various "additions"
650  * to specify the behaviour of interrupts, whether to kick the
651  * runlatch, etc...
652  */
653 
654 /*
655  * This addition reconciles our actual IRQ state with the various software
656  * flags that track it. This may call C code.
657  */
658 #define ADD_RECONCILE	RECONCILE_IRQ_STATE(r10,r11)
659 
660 #define ADD_NVGPRS				\
661 	bl	save_nvgprs
662 
663 #define RUNLATCH_ON				\
664 BEGIN_FTR_SECTION				\
665 	CURRENT_THREAD_INFO(r3, r1);		\
666 	ld	r4,TI_LOCAL_FLAGS(r3);		\
667 	andi.	r0,r4,_TLF_RUNLATCH;		\
668 	beql	ppc64_runlatch_on_trampoline;	\
669 END_FTR_SECTION_IFSET(CPU_FTR_CTRL)
670 
671 #define EXCEPTION_COMMON(area, trap, label, hdlr, ret, additions) \
672 	EXCEPTION_PROLOG_COMMON(trap, area);			\
673 	/* Volatile regs are potentially clobbered here */	\
674 	additions;						\
675 	addi	r3,r1,STACK_FRAME_OVERHEAD;			\
676 	bl	hdlr;						\
677 	b	ret
678 
679 /*
680  * Exception where stack is already set in r1, r1 is saved in r10, and it
681  * continues rather than returns.
682  */
683 #define EXCEPTION_COMMON_NORET_STACK(area, trap, label, hdlr, additions) \
684 	EXCEPTION_PROLOG_COMMON_1();				\
685 	EXCEPTION_PROLOG_COMMON_2(area);			\
686 	EXCEPTION_PROLOG_COMMON_3(trap);			\
687 	/* Volatile regs are potentially clobbered here */	\
688 	additions;						\
689 	addi	r3,r1,STACK_FRAME_OVERHEAD;			\
690 	bl	hdlr
691 
692 #define STD_EXCEPTION_COMMON(trap, label, hdlr)			\
693 	EXCEPTION_COMMON(PACA_EXGEN, trap, label, hdlr,		\
694 		ret_from_except, ADD_NVGPRS;ADD_RECONCILE)
695 
696 /*
697  * Like STD_EXCEPTION_COMMON, but for exceptions that can occur
698  * in the idle task and therefore need the special idle handling
699  * (finish nap and runlatch)
700  */
701 #define STD_EXCEPTION_COMMON_ASYNC(trap, label, hdlr)		\
702 	EXCEPTION_COMMON(PACA_EXGEN, trap, label, hdlr,		\
703 		ret_from_except_lite, FINISH_NAP;ADD_RECONCILE;RUNLATCH_ON)
704 
705 /*
706  * When the idle code in power4_idle puts the CPU into NAP mode,
707  * it has to do so in a loop, and relies on the external interrupt
708  * and decrementer interrupt entry code to get it out of the loop.
709  * It sets the _TLF_NAPPING bit in current_thread_info()->local_flags
710  * to signal that it is in the loop and needs help to get out.
711  */
712 #ifdef CONFIG_PPC_970_NAP
713 #define FINISH_NAP				\
714 BEGIN_FTR_SECTION				\
715 	CURRENT_THREAD_INFO(r11, r1);		\
716 	ld	r9,TI_LOCAL_FLAGS(r11);		\
717 	andi.	r10,r9,_TLF_NAPPING;		\
718 	bnel	power4_fixup_nap;		\
719 END_FTR_SECTION_IFSET(CPU_FTR_CAN_NAP)
720 #else
721 #define FINISH_NAP
722 #endif
723 
724 #endif	/* _ASM_POWERPC_EXCEPTION_H */
725