• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1/*
2 *  linux/arch/arm/kernel/entry-armv.S
3 *
4 *  Copyright (C) 1996,1997,1998 Russell King.
5 *  ARM700 fix by Matthew Godbolt (linux-user@willothewisp.demon.co.uk)
6 *  nommu support by Hyok S. Choi (hyok.choi@samsung.com)
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License version 2 as
10 * published by the Free Software Foundation.
11 *
12 *  Low-level vector interface routines
13 *
14 *  Note:  there is a StrongARM bug in the STMIA rn, {regs}^ instruction
15 *  that causes it to save wrong values...  Be aware!
16 */
17
18#include <asm/assembler.h>
19#include <asm/memory.h>
20#include <asm/glue-df.h>
21#include <asm/glue-pf.h>
22#include <asm/vfpmacros.h>
23#ifndef CONFIG_MULTI_IRQ_HANDLER
24#include <mach/entry-macro.S>
25#endif
26#include <asm/thread_notify.h>
27#include <asm/unwind.h>
28#include <asm/unistd.h>
29#include <asm/tls.h>
30#include <asm/system_info.h>
31
32#include "entry-header.S"
33#include <asm/entry-macro-multi.S>
34
35/*
36 * Interrupt handling.
37 */
38	.macro	irq_handler
39#ifdef CONFIG_MULTI_IRQ_HANDLER
40	ldr	r1, =handle_arch_irq
41	mov	r0, sp
42	adr	lr, BSYM(9997f)
43	ldr	pc, [r1]
44#else
45	arch_irq_handler_default
46#endif
479997:
48	.endm
49
50	.macro	pabt_helper
51	@ PABORT handler takes pt_regs in r2, fault address in r4 and psr in r5
52#ifdef MULTI_PABORT
53	ldr	ip, .LCprocfns
54	mov	lr, pc
55	ldr	pc, [ip, #PROCESSOR_PABT_FUNC]
56#else
57	bl	CPU_PABORT_HANDLER
58#endif
59	.endm
60
61	.macro	dabt_helper
62
63	@
64	@ Call the processor-specific abort handler:
65	@
66	@  r2 - pt_regs
67	@  r4 - aborted context pc
68	@  r5 - aborted context psr
69	@
70	@ The abort handler must return the aborted address in r0, and
71	@ the fault status register in r1.  r9 must be preserved.
72	@
73#ifdef MULTI_DABORT
74	ldr	ip, .LCprocfns
75	mov	lr, pc
76	ldr	pc, [ip, #PROCESSOR_DABT_FUNC]
77#else
78	bl	CPU_DABORT_HANDLER
79#endif
80	.endm
81
82#ifdef CONFIG_KPROBES
83	.section	.kprobes.text,"ax",%progbits
84#else
85	.text
86#endif
87
88/*
89 * Invalid mode handlers
90 */
91	.macro	inv_entry, reason
92	sub	sp, sp, #S_FRAME_SIZE
93 ARM(	stmib	sp, {r1 - lr}		)
94 THUMB(	stmia	sp, {r0 - r12}		)
95 THUMB(	str	sp, [sp, #S_SP]		)
96 THUMB(	str	lr, [sp, #S_LR]		)
97	mov	r1, #\reason
98	.endm
99
100__pabt_invalid:
101	inv_entry BAD_PREFETCH
102	b	common_invalid
103ENDPROC(__pabt_invalid)
104
105__dabt_invalid:
106	inv_entry BAD_DATA
107	b	common_invalid
108ENDPROC(__dabt_invalid)
109
110__irq_invalid:
111	inv_entry BAD_IRQ
112	b	common_invalid
113ENDPROC(__irq_invalid)
114
115__und_invalid:
116	inv_entry BAD_UNDEFINSTR
117
118	@
119	@ XXX fall through to common_invalid
120	@
121
122@
123@ common_invalid - generic code for failed exception (re-entrant version of handlers)
124@
125common_invalid:
126	zero_fp
127
128	ldmia	r0, {r4 - r6}
129	add	r0, sp, #S_PC		@ here for interlock avoidance
130	mov	r7, #-1			@  ""   ""    ""        ""
131	str	r4, [sp]		@ save preserved r0
132	stmia	r0, {r5 - r7}		@ lr_<exception>,
133					@ cpsr_<exception>, "old_r0"
134
135	mov	r0, sp
136	b	bad_mode
137ENDPROC(__und_invalid)
138
139/*
140 * SVC mode handlers
141 */
142
143#if defined(CONFIG_AEABI) && (__LINUX_ARM_ARCH__ >= 5)
144#define SPFIX(code...) code
145#else
146#define SPFIX(code...)
147#endif
148
149	.macro	svc_entry, stack_hole=0
150 UNWIND(.fnstart		)
151 UNWIND(.save {r0 - pc}		)
152	sub	sp, sp, #(S_FRAME_SIZE + \stack_hole - 4)
153#ifdef CONFIG_THUMB2_KERNEL
154 SPFIX(	str	r0, [sp]	)	@ temporarily saved
155 SPFIX(	mov	r0, sp		)
156 SPFIX(	tst	r0, #4		)	@ test original stack alignment
157 SPFIX(	ldr	r0, [sp]	)	@ restored
158#else
159 SPFIX(	tst	sp, #4		)
160#endif
161 SPFIX(	subeq	sp, sp, #4	)
162	stmia	sp, {r1 - r12}
163
164	ldmia	r0, {r3 - r5}
165	add	r7, sp, #S_SP - 4	@ here for interlock avoidance
166	mov	r6, #-1			@  ""  ""      ""       ""
167	add	r2, sp, #(S_FRAME_SIZE + \stack_hole - 4)
168 SPFIX(	addeq	r2, r2, #4	)
169	str	r3, [sp, #-4]!		@ save the "real" r0 copied
170					@ from the exception stack
171
172	mov	r3, lr
173
174	@
175	@ We are now ready to fill in the remaining blanks on the stack:
176	@
177	@  r2 - sp_svc
178	@  r3 - lr_svc
179	@  r4 - lr_<exception>, already fixed up for correct return/restart
180	@  r5 - spsr_<exception>
181	@  r6 - orig_r0 (see pt_regs definition in ptrace.h)
182	@
183	stmia	r7, {r2 - r6}
184
185#ifdef CONFIG_TRACE_IRQFLAGS
186	bl	trace_hardirqs_off
187#endif
188	.endm
189
190	.align	5
191__dabt_svc:
192	svc_entry
193	mov	r2, sp
194	dabt_helper
195
196	@
197	@ IRQs off again before pulling preserved data off the stack
198	@
199	disable_irq_notrace
200
201#ifdef CONFIG_TRACE_IRQFLAGS
202	tst	r5, #PSR_I_BIT
203	bleq	trace_hardirqs_on
204	tst	r5, #PSR_I_BIT
205	blne	trace_hardirqs_off
206#endif
207	svc_exit r5				@ return from exception
208 UNWIND(.fnend		)
209ENDPROC(__dabt_svc)
210
211	.align	5
212__irq_svc:
213	svc_entry
214	irq_handler
215
216#ifdef CONFIG_PREEMPT
217	get_thread_info tsk
218	ldr	r8, [tsk, #TI_PREEMPT]		@ get preempt count
219	ldr	r0, [tsk, #TI_FLAGS]		@ get flags
220	teq	r8, #0				@ if preempt count != 0
221	movne	r0, #0				@ force flags to 0
222	tst	r0, #_TIF_NEED_RESCHED
223	blne	svc_preempt
224#endif
225
226#ifdef CONFIG_TRACE_IRQFLAGS
227	@ The parent context IRQs must have been enabled to get here in
228	@ the first place, so there's no point checking the PSR I bit.
229	bl	trace_hardirqs_on
230#endif
231	svc_exit r5				@ return from exception
232 UNWIND(.fnend		)
233ENDPROC(__irq_svc)
234
235	.ltorg
236
237#ifdef CONFIG_PREEMPT
238svc_preempt:
239	mov	r8, lr
2401:	bl	preempt_schedule_irq		@ irq en/disable is done inside
241	ldr	r0, [tsk, #TI_FLAGS]		@ get new tasks TI_FLAGS
242	tst	r0, #_TIF_NEED_RESCHED
243	moveq	pc, r8				@ go again
244	b	1b
245#endif
246
247__und_fault:
248	@ Correct the PC such that it is pointing at the instruction
249	@ which caused the fault.  If the faulting instruction was ARM
250	@ the PC will be pointing at the next instruction, and have to
251	@ subtract 4.  Otherwise, it is Thumb, and the PC will be
252	@ pointing at the second half of the Thumb instruction.  We
253	@ have to subtract 2.
254	ldr	r2, [r0, #S_PC]
255	sub	r2, r2, r1
256	str	r2, [r0, #S_PC]
257	b	do_undefinstr
258ENDPROC(__und_fault)
259
260	.align	5
261__und_svc:
262#ifdef CONFIG_KPROBES
263	@ If a kprobe is about to simulate a "stmdb sp..." instruction,
264	@ it obviously needs free stack space which then will belong to
265	@ the saved context.
266	svc_entry 64
267#else
268	svc_entry
269#endif
270	@
271	@ call emulation code, which returns using r9 if it has emulated
272	@ the instruction, or the more conventional lr if we are to treat
273	@ this as a real undefined instruction
274	@
275	@  r0 - instruction
276	@
277#ifndef CONFIG_THUMB2_KERNEL
278	ldr	r0, [r4, #-4]
279#else
280	mov	r1, #2
281	ldrh	r0, [r4, #-2]			@ Thumb instruction at LR - 2
282	cmp	r0, #0xe800			@ 32-bit instruction if xx >= 0
283	blo	__und_svc_fault
284	ldrh	r9, [r4]			@ bottom 16 bits
285	add	r4, r4, #2
286	str	r4, [sp, #S_PC]
287	orr	r0, r9, r0, lsl #16
288#endif
289	adr	r9, BSYM(__und_svc_finish)
290	mov	r2, r4
291	bl	call_fpe
292
293	mov	r1, #4				@ PC correction to apply
294__und_svc_fault:
295	mov	r0, sp				@ struct pt_regs *regs
296	bl	__und_fault
297
298	@
299	@ IRQs off again before pulling preserved data off the stack
300	@
301__und_svc_finish:
302	disable_irq_notrace
303
304	@
305	@ restore SPSR and restart the instruction
306	@
307	ldr	r5, [sp, #S_PSR]		@ Get SVC cpsr
308#ifdef CONFIG_TRACE_IRQFLAGS
309	tst	r5, #PSR_I_BIT
310	bleq	trace_hardirqs_on
311	tst	r5, #PSR_I_BIT
312	blne	trace_hardirqs_off
313#endif
314	svc_exit r5				@ return from exception
315 UNWIND(.fnend		)
316ENDPROC(__und_svc)
317
318	.align	5
319__pabt_svc:
320	svc_entry
321	mov	r2, sp				@ regs
322	pabt_helper
323
324	@
325	@ IRQs off again before pulling preserved data off the stack
326	@
327	disable_irq_notrace
328
329#ifdef CONFIG_TRACE_IRQFLAGS
330	tst	r5, #PSR_I_BIT
331	bleq	trace_hardirqs_on
332	tst	r5, #PSR_I_BIT
333	blne	trace_hardirqs_off
334#endif
335	svc_exit r5				@ return from exception
336 UNWIND(.fnend		)
337ENDPROC(__pabt_svc)
338
339	.align	5
340.LCcralign:
341	.word	cr_alignment
342#ifdef MULTI_DABORT
343.LCprocfns:
344	.word	processor
345#endif
346.LCfp:
347	.word	fp_enter
348
349/*
350 * User mode handlers
351 *
352 * EABI note: sp_svc is always 64-bit aligned here, so should S_FRAME_SIZE
353 */
354
355#if defined(CONFIG_AEABI) && (__LINUX_ARM_ARCH__ >= 5) && (S_FRAME_SIZE & 7)
356#error "sizeof(struct pt_regs) must be a multiple of 8"
357#endif
358
359	.macro	usr_entry
360 UNWIND(.fnstart	)
361 UNWIND(.cantunwind	)	@ don't unwind the user space
362	sub	sp, sp, #S_FRAME_SIZE
363 ARM(	stmib	sp, {r1 - r12}	)
364 THUMB(	stmia	sp, {r0 - r12}	)
365
366	ldmia	r0, {r3 - r5}
367	add	r0, sp, #S_PC		@ here for interlock avoidance
368	mov	r6, #-1			@  ""  ""     ""        ""
369
370	str	r3, [sp]		@ save the "real" r0 copied
371					@ from the exception stack
372
373	@
374	@ We are now ready to fill in the remaining blanks on the stack:
375	@
376	@  r4 - lr_<exception>, already fixed up for correct return/restart
377	@  r5 - spsr_<exception>
378	@  r6 - orig_r0 (see pt_regs definition in ptrace.h)
379	@
380	@ Also, separately save sp_usr and lr_usr
381	@
382	stmia	r0, {r4 - r6}
383 ARM(	stmdb	r0, {sp, lr}^			)
384 THUMB(	store_user_sp_lr r0, r1, S_SP - S_PC	)
385
386	@
387	@ Enable the alignment trap while in kernel mode
388	@
389	alignment_trap r0
390
391	@
392	@ Clear FP to mark the first stack frame
393	@
394	zero_fp
395
396#ifdef CONFIG_IRQSOFF_TRACER
397	bl	trace_hardirqs_off
398#endif
399	.endm
400
401	.macro	kuser_cmpxchg_check
402#if !defined(CONFIG_CPU_32v6K) && !defined(CONFIG_NEEDS_SYSCALL_FOR_CMPXCHG)
403#ifndef CONFIG_MMU
404#warning "NPTL on non MMU needs fixing"
405#else
406	@ Make sure our user space atomic helper is restarted
407	@ if it was interrupted in a critical region.  Here we
408	@ perform a quick test inline since it should be false
409	@ 99.9999% of the time.  The rest is done out of line.
410	cmp	r4, #TASK_SIZE
411	blhs	kuser_cmpxchg64_fixup
412#endif
413#endif
414	.endm
415
416	.align	5
417__dabt_usr:
418	usr_entry
419	kuser_cmpxchg_check
420	mov	r2, sp
421	dabt_helper
422	b	ret_from_exception
423 UNWIND(.fnend		)
424ENDPROC(__dabt_usr)
425
426	.align	5
427__irq_usr:
428	usr_entry
429	kuser_cmpxchg_check
430	irq_handler
431	get_thread_info tsk
432	mov	why, #0
433	b	ret_to_user_from_irq
434 UNWIND(.fnend		)
435ENDPROC(__irq_usr)
436
437	.ltorg
438
439	.align	5
440__und_usr:
441	usr_entry
442
443	mov	r2, r4
444	mov	r3, r5
445
446	@ r2 = regs->ARM_pc, which is either 2 or 4 bytes ahead of the
447	@      faulting instruction depending on Thumb mode.
448	@ r3 = regs->ARM_cpsr
449	@
450	@ The emulation code returns using r9 if it has emulated the
451	@ instruction, or the more conventional lr if we are to treat
452	@ this as a real undefined instruction
453	@
454	adr	r9, BSYM(ret_from_exception)
455
456	tst	r3, #PSR_T_BIT			@ Thumb mode?
457	bne	__und_usr_thumb
458	sub	r4, r2, #4			@ ARM instr at LR - 4
4591:	ldrt	r0, [r4]
460#ifdef CONFIG_CPU_ENDIAN_BE8
461	rev	r0, r0				@ little endian instruction
462#endif
463	@ r0 = 32-bit ARM instruction which caused the exception
464	@ r2 = PC value for the following instruction (:= regs->ARM_pc)
465	@ r4 = PC value for the faulting instruction
466	@ lr = 32-bit undefined instruction function
467	adr	lr, BSYM(__und_usr_fault_32)
468	b	call_fpe
469
470__und_usr_thumb:
471	@ Thumb instruction
472	sub	r4, r2, #2			@ First half of thumb instr at LR - 2
473#if CONFIG_ARM_THUMB && __LINUX_ARM_ARCH__ >= 6 && CONFIG_CPU_V7
474/*
475 * Thumb-2 instruction handling.  Note that because pre-v6 and >= v6 platforms
476 * can never be supported in a single kernel, this code is not applicable at
477 * all when __LINUX_ARM_ARCH__ < 6.  This allows simplifying assumptions to be
478 * made about .arch directives.
479 */
480#if __LINUX_ARM_ARCH__ < 7
481/* If the target CPU may not be Thumb-2-capable, a run-time check is needed: */
482#define NEED_CPU_ARCHITECTURE
483	ldr	r5, .LCcpu_architecture
484	ldr	r5, [r5]
485	cmp	r5, #CPU_ARCH_ARMv7
486	blo	__und_usr_fault_16		@ 16bit undefined instruction
487/*
488 * The following code won't get run unless the running CPU really is v7, so
489 * coding round the lack of ldrht on older arches is pointless.  Temporarily
490 * override the assembler target arch with the minimum required instead:
491 */
492	.arch	armv6t2
493#endif
4942:	ldrht	r5, [r4]
495	cmp	r5, #0xe800			@ 32bit instruction if xx != 0
496	blo	__und_usr_fault_16		@ 16bit undefined instruction
4973:	ldrht	r0, [r2]
498	add	r2, r2, #2			@ r2 is PC + 2, make it PC + 4
499	str	r2, [sp, #S_PC]			@ it's a 2x16bit instr, update
500	orr	r0, r0, r5, lsl #16
501	adr	lr, BSYM(__und_usr_fault_32)
502	@ r0 = the two 16-bit Thumb instructions which caused the exception
503	@ r2 = PC value for the following Thumb instruction (:= regs->ARM_pc)
504	@ r4 = PC value for the first 16-bit Thumb instruction
505	@ lr = 32bit undefined instruction function
506
507#if __LINUX_ARM_ARCH__ < 7
508/* If the target arch was overridden, change it back: */
509#ifdef CONFIG_CPU_32v6K
510	.arch	armv6k
511#else
512	.arch	armv6
513#endif
514#endif /* __LINUX_ARM_ARCH__ < 7 */
515#else /* !(CONFIG_ARM_THUMB && __LINUX_ARM_ARCH__ >= 6 && CONFIG_CPU_V7) */
516	b	__und_usr_fault_16
517#endif
518 UNWIND(.fnend)
519ENDPROC(__und_usr)
520
521/*
522 * The out of line fixup for the ldrt instructions above.
523 */
524	.pushsection .fixup, "ax"
5254:	mov	pc, r9
526	.popsection
527	.pushsection __ex_table,"a"
528	.long	1b, 4b
529#if CONFIG_ARM_THUMB && __LINUX_ARM_ARCH__ >= 6 && CONFIG_CPU_V7
530	.long	2b, 4b
531	.long	3b, 4b
532#endif
533	.popsection
534
535/*
536 * Check whether the instruction is a co-processor instruction.
537 * If yes, we need to call the relevant co-processor handler.
538 *
539 * Note that we don't do a full check here for the co-processor
540 * instructions; all instructions with bit 27 set are well
541 * defined.  The only instructions that should fault are the
542 * co-processor instructions.  However, we have to watch out
543 * for the ARM6/ARM7 SWI bug.
544 *
545 * NEON is a special case that has to be handled here. Not all
546 * NEON instructions are co-processor instructions, so we have
547 * to make a special case of checking for them. Plus, there's
548 * five groups of them, so we have a table of mask/opcode pairs
549 * to check against, and if any match then we branch off into the
550 * NEON handler code.
551 *
552 * Emulators may wish to make use of the following registers:
553 *  r0  = instruction opcode (32-bit ARM or two 16-bit Thumb)
554 *  r2  = PC value to resume execution after successful emulation
555 *  r9  = normal "successful" return address
556 *  r10 = this threads thread_info structure
557 *  lr  = unrecognised instruction return address
558 * IRQs disabled, FIQs enabled.
559 */
560	@
561	@ Fall-through from Thumb-2 __und_usr
562	@
563#ifdef CONFIG_NEON
564	adr	r6, .LCneon_thumb_opcodes
565	b	2f
566#endif
567call_fpe:
568#ifdef CONFIG_NEON
569	adr	r6, .LCneon_arm_opcodes
5702:
571	ldr	r7, [r6], #4			@ mask value
572	cmp	r7, #0				@ end mask?
573	beq	1f
574	and	r8, r0, r7
575	ldr	r7, [r6], #4			@ opcode bits matching in mask
576	cmp	r8, r7				@ NEON instruction?
577	bne	2b
578	get_thread_info r10
579	mov	r7, #1
580	strb	r7, [r10, #TI_USED_CP + 10]	@ mark CP#10 as used
581	strb	r7, [r10, #TI_USED_CP + 11]	@ mark CP#11 as used
582	b	do_vfp				@ let VFP handler handle this
5831:
584#endif
585	tst	r0, #0x08000000			@ only CDP/CPRT/LDC/STC have bit 27
586	tstne	r0, #0x04000000			@ bit 26 set on both ARM and Thumb-2
587#if defined(CONFIG_CPU_ARM610) || defined(CONFIG_CPU_ARM710)
588	and	r8, r0, #0x0f000000		@ mask out op-code bits
589	teqne	r8, #0x0f000000			@ SWI (ARM6/7 bug)?
590#endif
591	moveq	pc, lr
592	get_thread_info r10			@ get current thread
593	and	r8, r0, #0x00000f00		@ mask out CP number
594 THUMB(	lsr	r8, r8, #8		)
595	mov	r7, #1
596	add	r6, r10, #TI_USED_CP
597 ARM(	strb	r7, [r6, r8, lsr #8]	)	@ set appropriate used_cp[]
598 THUMB(	strb	r7, [r6, r8]		)	@ set appropriate used_cp[]
599#ifdef CONFIG_IWMMXT
600	@ Test if we need to give access to iWMMXt coprocessors
601	ldr	r5, [r10, #TI_FLAGS]
602	rsbs	r7, r8, #(1 << 8)		@ CP 0 or 1 only
603	movcss	r7, r5, lsr #(TIF_USING_IWMMXT + 1)
604	bcs	iwmmxt_task_enable
605#endif
606 ARM(	add	pc, pc, r8, lsr #6	)
607 THUMB(	lsl	r8, r8, #2		)
608 THUMB(	add	pc, r8			)
609	nop
610
611	movw_pc	lr				@ CP#0
612	W(b)	do_fpe				@ CP#1 (FPE)
613	W(b)	do_fpe				@ CP#2 (FPE)
614	movw_pc	lr				@ CP#3
615#ifdef CONFIG_CRUNCH
616	b	crunch_task_enable		@ CP#4 (MaverickCrunch)
617	b	crunch_task_enable		@ CP#5 (MaverickCrunch)
618	b	crunch_task_enable		@ CP#6 (MaverickCrunch)
619#else
620	movw_pc	lr				@ CP#4
621	movw_pc	lr				@ CP#5
622	movw_pc	lr				@ CP#6
623#endif
624	movw_pc	lr				@ CP#7
625	movw_pc	lr				@ CP#8
626	movw_pc	lr				@ CP#9
627#ifdef CONFIG_VFP
628	W(b)	do_vfp				@ CP#10 (VFP)
629	W(b)	do_vfp				@ CP#11 (VFP)
630#else
631	movw_pc	lr				@ CP#10 (VFP)
632	movw_pc	lr				@ CP#11 (VFP)
633#endif
634	movw_pc	lr				@ CP#12
635	movw_pc	lr				@ CP#13
636	movw_pc	lr				@ CP#14 (Debug)
637	movw_pc	lr				@ CP#15 (Control)
638
639#ifdef NEED_CPU_ARCHITECTURE
640	.align	2
641.LCcpu_architecture:
642	.word	__cpu_architecture
643#endif
644
645#ifdef CONFIG_NEON
646	.align	6
647
648.LCneon_arm_opcodes:
649	.word	0xfe000000			@ mask
650	.word	0xf2000000			@ opcode
651
652	.word	0xff100000			@ mask
653	.word	0xf4000000			@ opcode
654
655	.word	0x00000000			@ mask
656	.word	0x00000000			@ opcode
657
658.LCneon_thumb_opcodes:
659	.word	0xef000000			@ mask
660	.word	0xef000000			@ opcode
661
662	.word	0xff100000			@ mask
663	.word	0xf9000000			@ opcode
664
665	.word	0x00000000			@ mask
666	.word	0x00000000			@ opcode
667#endif
668
669do_fpe:
670	enable_irq
671	ldr	r4, .LCfp
672	add	r10, r10, #TI_FPSTATE		@ r10 = workspace
673	ldr	pc, [r4]			@ Call FP module USR entry point
674
675/*
676 * The FP module is called with these registers set:
677 *  r0  = instruction
678 *  r2  = PC+4
679 *  r9  = normal "successful" return address
680 *  r10 = FP workspace
681 *  lr  = unrecognised FP instruction return address
682 */
683
684	.pushsection .data
685ENTRY(fp_enter)
686	.word	no_fp
687	.popsection
688
689ENTRY(no_fp)
690	mov	pc, lr
691ENDPROC(no_fp)
692
693__und_usr_fault_32:
694	mov	r1, #4
695	b	1f
696__und_usr_fault_16:
697	mov	r1, #2
6981:	enable_irq
699	mov	r0, sp
700	adr	lr, BSYM(ret_from_exception)
701	b	__und_fault
702ENDPROC(__und_usr_fault_32)
703ENDPROC(__und_usr_fault_16)
704
705	.align	5
706__pabt_usr:
707	usr_entry
708	mov	r2, sp				@ regs
709	pabt_helper
710 UNWIND(.fnend		)
711	/* fall through */
712/*
713 * This is the return code to user mode for abort handlers
714 */
715ENTRY(ret_from_exception)
716 UNWIND(.fnstart	)
717 UNWIND(.cantunwind	)
718	get_thread_info tsk
719	mov	why, #0
720	b	ret_to_user
721 UNWIND(.fnend		)
722ENDPROC(__pabt_usr)
723ENDPROC(ret_from_exception)
724
725/*
726 * Register switch for ARMv3 and ARMv4 processors
727 * r0 = previous task_struct, r1 = previous thread_info, r2 = next thread_info
728 * previous and next are guaranteed not to be the same.
729 */
730ENTRY(__switch_to)
731 UNWIND(.fnstart	)
732 UNWIND(.cantunwind	)
733	add	ip, r1, #TI_CPU_SAVE
734	ldr	r3, [r2, #TI_TP_VALUE]
735 ARM(	stmia	ip!, {r4 - sl, fp, sp, lr} )	@ Store most regs on stack
736 THUMB(	stmia	ip!, {r4 - sl, fp}	   )	@ Store most regs on stack
737 THUMB(	str	sp, [ip], #4		   )
738 THUMB(	str	lr, [ip], #4		   )
739#ifdef CONFIG_CPU_USE_DOMAINS
740	ldr	r6, [r2, #TI_CPU_DOMAIN]
741#endif
742	set_tls	r3, r4, r5
743#if defined(CONFIG_CC_STACKPROTECTOR) && !defined(CONFIG_SMP)
744	ldr	r7, [r2, #TI_TASK]
745	ldr	r8, =__stack_chk_guard
746	ldr	r7, [r7, #TSK_STACK_CANARY]
747#endif
748#ifdef CONFIG_CPU_USE_DOMAINS
749	mcr	p15, 0, r6, c3, c0, 0		@ Set domain register
750#endif
751	mov	r5, r0
752	add	r4, r2, #TI_CPU_SAVE
753	ldr	r0, =thread_notify_head
754	mov	r1, #THREAD_NOTIFY_SWITCH
755	bl	atomic_notifier_call_chain
756#if defined(CONFIG_CC_STACKPROTECTOR) && !defined(CONFIG_SMP)
757	str	r7, [r8]
758#endif
759 THUMB(	mov	ip, r4			   )
760#ifdef CONFIG_QEMU_TRACE
761/*
762	mcr	p15, 0, r0, c15, c0, 0		@ signal context switch
763*/
764#endif
765	mov	r0, r5
766 ARM(	ldmia	r4, {r4 - sl, fp, sp, pc}  )	@ Load all regs saved previously
767 THUMB(	ldmia	ip!, {r4 - sl, fp}	   )	@ Load all regs saved previously
768 THUMB(	ldr	sp, [ip], #4		   )
769 THUMB(	ldr	pc, [ip]		   )
770 UNWIND(.fnend		)
771ENDPROC(__switch_to)
772
773	__INIT
774
775/*
776 * User helpers.
777 *
778 * Each segment is 32-byte aligned and will be moved to the top of the high
779 * vector page.  New segments (if ever needed) must be added in front of
780 * existing ones.  This mechanism should be used only for things that are
781 * really small and justified, and not be abused freely.
782 *
783 * See Documentation/arm/kernel_user_helpers.txt for formal definitions.
784 */
785 THUMB(	.arm	)
786
787	.macro	usr_ret, reg
788#ifdef CONFIG_ARM_THUMB
789	bx	\reg
790#else
791	mov	pc, \reg
792#endif
793	.endm
794
795	.macro	kuser_pad, sym, size
796	.if	(. - \sym) & 3
797	.rept	4 - (. - \sym) & 3
798	.byte	0
799	.endr
800	.endif
801	.rept	(\size - (. - \sym)) / 4
802	.word	0xe7fddef1
803	.endr
804	.endm
805
806#ifdef CONFIG_KUSER_HELPERS
807	.align	5
808	.globl	__kuser_helper_start
809__kuser_helper_start:
810
811/*
812 * Due to the length of some sequences, __kuser_cmpxchg64 spans 2 regular
813 * kuser "slots", therefore 0xffff0f80 is not used as a valid entry point.
814 */
815
816__kuser_cmpxchg64:				@ 0xffff0f60
817
818#if defined(CONFIG_NEEDS_SYSCALL_FOR_CMPXCHG)
819
820	/*
821	 * Poor you.  No fast solution possible...
822	 * The kernel itself must perform the operation.
823	 * A special ghost syscall is used for that (see traps.c).
824	 */
825	stmfd	sp!, {r7, lr}
826	ldr	r7, 1f			@ it's 20 bits
827	swi	__ARM_NR_cmpxchg64
828	ldmfd	sp!, {r7, pc}
8291:	.word	__ARM_NR_cmpxchg64
830
831#elif defined(CONFIG_CPU_32v6K)
832
833	stmfd	sp!, {r4, r5, r6, r7}
834	ldrd	r4, r5, [r0]			@ load old val
835	ldrd	r6, r7, [r1]			@ load new val
836	smp_dmb	arm
8371:	ldrexd	r0, r1, [r2]			@ load current val
838	eors	r3, r0, r4			@ compare with oldval (1)
839	eoreqs	r3, r1, r5			@ compare with oldval (2)
840	strexdeq r3, r6, r7, [r2]		@ store newval if eq
841	teqeq	r3, #1				@ success?
842	beq	1b				@ if no then retry
843	smp_dmb	arm
844	rsbs	r0, r3, #0			@ set returned val and C flag
845	ldmfd	sp!, {r4, r5, r6, r7}
846	usr_ret	lr
847
848#elif !defined(CONFIG_SMP)
849
850#ifdef CONFIG_MMU
851
852	/*
853	 * The only thing that can break atomicity in this cmpxchg64
854	 * implementation is either an IRQ or a data abort exception
855	 * causing another process/thread to be scheduled in the middle of
856	 * the critical sequence.  The same strategy as for cmpxchg is used.
857	 */
858	stmfd	sp!, {r4, r5, r6, lr}
859	ldmia	r0, {r4, r5}			@ load old val
860	ldmia	r1, {r6, lr}			@ load new val
8611:	ldmia	r2, {r0, r1}			@ load current val
862	eors	r3, r0, r4			@ compare with oldval (1)
863	eoreqs	r3, r1, r5			@ compare with oldval (2)
8642:	stmeqia	r2, {r6, lr}			@ store newval if eq
865	rsbs	r0, r3, #0			@ set return val and C flag
866	ldmfd	sp!, {r4, r5, r6, pc}
867
868	.text
869kuser_cmpxchg64_fixup:
870	@ Called from kuser_cmpxchg_fixup.
871	@ r4 = address of interrupted insn (must be preserved).
872	@ sp = saved regs. r7 and r8 are clobbered.
873	@ 1b = first critical insn, 2b = last critical insn.
874	@ If r4 >= 1b and r4 <= 2b then saved pc_usr is set to 1b.
875	mov	r7, #0xffff0fff
876	sub	r7, r7, #(0xffff0fff - (0xffff0f60 + (1b - __kuser_cmpxchg64)))
877	subs	r8, r4, r7
878	rsbcss	r8, r8, #(2b - 1b)
879	strcs	r7, [sp, #S_PC]
880#if __LINUX_ARM_ARCH__ < 6
881	bcc	kuser_cmpxchg32_fixup
882#endif
883	mov	pc, lr
884	.previous
885
886#else
887#warning "NPTL on non MMU needs fixing"
888	mov	r0, #-1
889	adds	r0, r0, #0
890	usr_ret	lr
891#endif
892
893#else
894#error "incoherent kernel configuration"
895#endif
896
897	kuser_pad __kuser_cmpxchg64, 64
898
899__kuser_memory_barrier:				@ 0xffff0fa0
900	smp_dmb	arm
901	usr_ret	lr
902
903	kuser_pad __kuser_memory_barrier, 32
904
905__kuser_cmpxchg:				@ 0xffff0fc0
906
907#if defined(CONFIG_NEEDS_SYSCALL_FOR_CMPXCHG)
908
909	/*
910	 * Poor you.  No fast solution possible...
911	 * The kernel itself must perform the operation.
912	 * A special ghost syscall is used for that (see traps.c).
913	 */
914	stmfd	sp!, {r7, lr}
915	ldr	r7, 1f			@ it's 20 bits
916	swi	__ARM_NR_cmpxchg
917	ldmfd	sp!, {r7, pc}
9181:	.word	__ARM_NR_cmpxchg
919
920#elif __LINUX_ARM_ARCH__ < 6
921
922#ifdef CONFIG_MMU
923
924	/*
925	 * The only thing that can break atomicity in this cmpxchg
926	 * implementation is either an IRQ or a data abort exception
927	 * causing another process/thread to be scheduled in the middle
928	 * of the critical sequence.  To prevent this, code is added to
929	 * the IRQ and data abort exception handlers to set the pc back
930	 * to the beginning of the critical section if it is found to be
931	 * within that critical section (see kuser_cmpxchg_fixup).
932	 */
9331:	ldr	r3, [r2]			@ load current val
934	subs	r3, r3, r0			@ compare with oldval
9352:	streq	r1, [r2]			@ store newval if eq
936	rsbs	r0, r3, #0			@ set return val and C flag
937	usr_ret	lr
938
939	.text
940kuser_cmpxchg32_fixup:
941	@ Called from kuser_cmpxchg_check macro.
942	@ r4 = address of interrupted insn (must be preserved).
943	@ sp = saved regs. r7 and r8 are clobbered.
944	@ 1b = first critical insn, 2b = last critical insn.
945	@ If r4 >= 1b and r4 <= 2b then saved pc_usr is set to 1b.
946	mov	r7, #0xffff0fff
947	sub	r7, r7, #(0xffff0fff - (0xffff0fc0 + (1b - __kuser_cmpxchg)))
948	subs	r8, r4, r7
949	rsbcss	r8, r8, #(2b - 1b)
950	strcs	r7, [sp, #S_PC]
951	mov	pc, lr
952	.previous
953
954#else
955#warning "NPTL on non MMU needs fixing"
956	mov	r0, #-1
957	adds	r0, r0, #0
958	usr_ret	lr
959#endif
960
961#else
962
963	smp_dmb	arm
9641:	ldrex	r3, [r2]
965	subs	r3, r3, r0
966	strexeq	r3, r1, [r2]
967	teqeq	r3, #1
968	beq	1b
969	rsbs	r0, r3, #0
970	/* beware -- each __kuser slot must be 8 instructions max */
971	ALT_SMP(b	__kuser_memory_barrier)
972	ALT_UP(usr_ret	lr)
973
974#endif
975
976	kuser_pad __kuser_cmpxchg, 32
977
978__kuser_get_tls:				@ 0xffff0fe0
979	ldr	r0, [pc, #(16 - 8)]	@ read TLS, set in kuser_get_tls_init
980	usr_ret	lr
981	mrc	p15, 0, r0, c13, c0, 3	@ 0xffff0fe8 hardware TLS code
982	kuser_pad __kuser_get_tls, 16
983	.rep	3
984	.word	0			@ 0xffff0ff0 software TLS value, then
985	.endr				@ pad up to __kuser_helper_version
986
987__kuser_helper_version:				@ 0xffff0ffc
988	.word	((__kuser_helper_end - __kuser_helper_start) >> 5)
989
990	.globl	__kuser_helper_end
991__kuser_helper_end:
992
993#endif
994
995 THUMB(	.thumb	)
996
997/*
998 * Vector stubs.
999 *
1000 * This code is copied to 0xffff1000 so we can use branches in the
1001 * vectors, rather than ldr's.  Note that this code must not exceed
1002 * a page size.
1003 *
1004 * Common stub entry macro:
1005 *   Enter in IRQ mode, spsr = SVC/USR CPSR, lr = SVC/USR PC
1006 *
1007 * SP points to a minimal amount of processor-private memory, the address
1008 * of which is copied into r0 for the mode specific abort handler.
1009 */
1010	.macro	vector_stub, name, mode, correction=0
1011	.align	5
1012
1013vector_\name:
1014	.if \correction
1015	sub	lr, lr, #\correction
1016	.endif
1017
1018	@
1019	@ Save r0, lr_<exception> (parent PC) and spsr_<exception>
1020	@ (parent CPSR)
1021	@
1022	stmia	sp, {r0, lr}		@ save r0, lr
1023	mrs	lr, spsr
1024	str	lr, [sp, #8]		@ save spsr
1025
1026	@
1027	@ Prepare for SVC32 mode.  IRQs remain disabled.
1028	@
1029	mrs	r0, cpsr
1030	eor	r0, r0, #(\mode ^ SVC_MODE | PSR_ISETSTATE)
1031	msr	spsr_cxsf, r0
1032
1033	@
1034	@ the branch table must immediately follow this code
1035	@
1036	and	lr, lr, #0x0f
1037 THUMB(	adr	r0, 1f			)
1038 THUMB(	ldr	lr, [r0, lr, lsl #2]	)
1039	mov	r0, sp
1040 ARM(	ldr	lr, [pc, lr, lsl #2]	)
1041	movs	pc, lr			@ branch to handler in SVC mode
1042ENDPROC(vector_\name)
1043
1044	.align	2
1045	@ handler addresses follow this label
10461:
1047	.endm
1048
1049	.section .stubs, "ax", %progbits
1050__stubs_start:
1051	@ This must be the first word
1052	.word	vector_swi
1053
1054vector_rst:
1055 ARM(	swi	SYS_ERROR0	)
1056 THUMB(	svc	#0		)
1057 THUMB(	nop			)
1058	b	vector_und
1059
1060/*
1061 * Interrupt dispatcher
1062 */
1063	vector_stub	irq, IRQ_MODE, 4
1064
1065	.long	__irq_usr			@  0  (USR_26 / USR_32)
1066	.long	__irq_invalid			@  1  (FIQ_26 / FIQ_32)
1067	.long	__irq_invalid			@  2  (IRQ_26 / IRQ_32)
1068	.long	__irq_svc			@  3  (SVC_26 / SVC_32)
1069	.long	__irq_invalid			@  4
1070	.long	__irq_invalid			@  5
1071	.long	__irq_invalid			@  6
1072	.long	__irq_invalid			@  7
1073	.long	__irq_invalid			@  8
1074	.long	__irq_invalid			@  9
1075	.long	__irq_invalid			@  a
1076	.long	__irq_invalid			@  b
1077	.long	__irq_invalid			@  c
1078	.long	__irq_invalid			@  d
1079	.long	__irq_invalid			@  e
1080	.long	__irq_invalid			@  f
1081
1082/*
1083 * Data abort dispatcher
1084 * Enter in ABT mode, spsr = USR CPSR, lr = USR PC
1085 */
1086	vector_stub	dabt, ABT_MODE, 8
1087
1088	.long	__dabt_usr			@  0  (USR_26 / USR_32)
1089	.long	__dabt_invalid			@  1  (FIQ_26 / FIQ_32)
1090	.long	__dabt_invalid			@  2  (IRQ_26 / IRQ_32)
1091	.long	__dabt_svc			@  3  (SVC_26 / SVC_32)
1092	.long	__dabt_invalid			@  4
1093	.long	__dabt_invalid			@  5
1094	.long	__dabt_invalid			@  6
1095	.long	__dabt_invalid			@  7
1096	.long	__dabt_invalid			@  8
1097	.long	__dabt_invalid			@  9
1098	.long	__dabt_invalid			@  a
1099	.long	__dabt_invalid			@  b
1100	.long	__dabt_invalid			@  c
1101	.long	__dabt_invalid			@  d
1102	.long	__dabt_invalid			@  e
1103	.long	__dabt_invalid			@  f
1104
1105/*
1106 * Prefetch abort dispatcher
1107 * Enter in ABT mode, spsr = USR CPSR, lr = USR PC
1108 */
1109	vector_stub	pabt, ABT_MODE, 4
1110
1111	.long	__pabt_usr			@  0 (USR_26 / USR_32)
1112	.long	__pabt_invalid			@  1 (FIQ_26 / FIQ_32)
1113	.long	__pabt_invalid			@  2 (IRQ_26 / IRQ_32)
1114	.long	__pabt_svc			@  3 (SVC_26 / SVC_32)
1115	.long	__pabt_invalid			@  4
1116	.long	__pabt_invalid			@  5
1117	.long	__pabt_invalid			@  6
1118	.long	__pabt_invalid			@  7
1119	.long	__pabt_invalid			@  8
1120	.long	__pabt_invalid			@  9
1121	.long	__pabt_invalid			@  a
1122	.long	__pabt_invalid			@  b
1123	.long	__pabt_invalid			@  c
1124	.long	__pabt_invalid			@  d
1125	.long	__pabt_invalid			@  e
1126	.long	__pabt_invalid			@  f
1127
1128/*
1129 * Undef instr entry dispatcher
1130 * Enter in UND mode, spsr = SVC/USR CPSR, lr = SVC/USR PC
1131 */
1132	vector_stub	und, UND_MODE
1133
1134	.long	__und_usr			@  0 (USR_26 / USR_32)
1135	.long	__und_invalid			@  1 (FIQ_26 / FIQ_32)
1136	.long	__und_invalid			@  2 (IRQ_26 / IRQ_32)
1137	.long	__und_svc			@  3 (SVC_26 / SVC_32)
1138	.long	__und_invalid			@  4
1139	.long	__und_invalid			@  5
1140	.long	__und_invalid			@  6
1141	.long	__und_invalid			@  7
1142	.long	__und_invalid			@  8
1143	.long	__und_invalid			@  9
1144	.long	__und_invalid			@  a
1145	.long	__und_invalid			@  b
1146	.long	__und_invalid			@  c
1147	.long	__und_invalid			@  d
1148	.long	__und_invalid			@  e
1149	.long	__und_invalid			@  f
1150
1151	.align	5
1152
1153/*=============================================================================
1154 * Address exception handler
1155 *-----------------------------------------------------------------------------
1156 * These aren't too critical.
1157 * (they're not supposed to happen, and won't happen in 32-bit data mode).
1158 */
1159
1160vector_addrexcptn:
1161	b	vector_addrexcptn
1162
1163/*=============================================================================
1164 * Undefined FIQs
1165 *-----------------------------------------------------------------------------
1166 * Enter in FIQ mode, spsr = ANY CPSR, lr = ANY PC
1167 * MUST PRESERVE SVC SPSR, but need to switch to SVC mode to show our msg.
1168 * Basically to switch modes, we *HAVE* to clobber one register...  brain
1169 * damage alert!  I don't think that we can execute any code in here in any
1170 * other mode than FIQ...  Ok you can switch to another mode, but you can't
1171 * get out of that mode without clobbering one register.
1172 */
1173vector_fiq:
1174	subs	pc, lr, #4
1175
1176	.globl	vector_fiq_offset
1177	.equ	vector_fiq_offset, vector_fiq
1178
1179	.section .vectors, "ax", %progbits
1180__vectors_start:
1181	W(b)	vector_rst
1182	W(b)	vector_und
1183	W(ldr)	pc, __vectors_start + 0x1000
1184	W(b)	vector_pabt
1185	W(b)	vector_dabt
1186	W(b)	vector_addrexcptn
1187	W(b)	vector_irq
1188	W(b)	vector_fiq
1189
1190	.data
1191
1192	.globl	cr_alignment
1193	.globl	cr_no_alignment
1194cr_alignment:
1195	.space	4
1196cr_no_alignment:
1197	.space	4
1198
1199#ifdef CONFIG_MULTI_IRQ_HANDLER
1200	.globl	handle_arch_irq
1201handle_arch_irq:
1202	.space	4
1203#endif
1204