• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Exception handling code
4  *
5  * Copyright (C) 2019 ARM Ltd.
6  */
7 
8 #include <linux/context_tracking.h>
9 #include <linux/linkage.h>
10 #include <linux/lockdep.h>
11 #include <linux/ptrace.h>
12 #include <linux/sched.h>
13 #include <linux/sched/debug.h>
14 #include <linux/thread_info.h>
15 
16 #include <asm/cpufeature.h>
17 #include <asm/daifflags.h>
18 #include <asm/esr.h>
19 #include <asm/exception.h>
20 #include <asm/kprobes.h>
21 #include <asm/mmu.h>
22 #include <asm/processor.h>
23 #include <asm/sdei.h>
24 #include <asm/stacktrace.h>
25 #include <asm/sysreg.h>
26 #include <asm/system_misc.h>
27 
28 #include <trace/hooks/traps.h>
29 
30 /*
31  * Handle IRQ/context state management when entering from kernel mode.
32  * Before this function is called it is not safe to call regular kernel code,
33  * intrumentable code, or any code which may trigger an exception.
34  *
35  * This is intended to match the logic in irqentry_enter(), handling the kernel
36  * mode transitions only.
37  */
__enter_from_kernel_mode(struct pt_regs * regs)38 static __always_inline void __enter_from_kernel_mode(struct pt_regs *regs)
39 {
40 	regs->exit_rcu = false;
41 
42 	if (!IS_ENABLED(CONFIG_TINY_RCU) && is_idle_task(current)) {
43 		lockdep_hardirqs_off(CALLER_ADDR0);
44 		rcu_irq_enter();
45 		trace_hardirqs_off_finish();
46 
47 		regs->exit_rcu = true;
48 		return;
49 	}
50 
51 	lockdep_hardirqs_off(CALLER_ADDR0);
52 	rcu_irq_enter_check_tick();
53 	trace_hardirqs_off_finish();
54 }
55 
enter_from_kernel_mode(struct pt_regs * regs)56 static void noinstr enter_from_kernel_mode(struct pt_regs *regs)
57 {
58 	__enter_from_kernel_mode(regs);
59 	mte_check_tfsr_entry();
60 }
61 
62 /*
63  * Handle IRQ/context state management when exiting to kernel mode.
64  * After this function returns it is not safe to call regular kernel code,
65  * intrumentable code, or any code which may trigger an exception.
66  *
67  * This is intended to match the logic in irqentry_exit(), handling the kernel
68  * mode transitions only, and with preemption handled elsewhere.
69  */
__exit_to_kernel_mode(struct pt_regs * regs)70 static __always_inline void __exit_to_kernel_mode(struct pt_regs *regs)
71 {
72 	lockdep_assert_irqs_disabled();
73 
74 	if (interrupts_enabled(regs)) {
75 		if (regs->exit_rcu) {
76 			trace_hardirqs_on_prepare();
77 			lockdep_hardirqs_on_prepare();
78 			rcu_irq_exit();
79 			lockdep_hardirqs_on(CALLER_ADDR0);
80 			return;
81 		}
82 
83 		trace_hardirqs_on();
84 	} else {
85 		if (regs->exit_rcu)
86 			rcu_irq_exit();
87 	}
88 }
89 
exit_to_kernel_mode(struct pt_regs * regs)90 static void noinstr exit_to_kernel_mode(struct pt_regs *regs)
91 {
92 	mte_check_tfsr_exit();
93 	__exit_to_kernel_mode(regs);
94 }
95 
96 /*
97  * Handle IRQ/context state management when entering from user mode.
98  * Before this function is called it is not safe to call regular kernel code,
99  * intrumentable code, or any code which may trigger an exception.
100  */
__enter_from_user_mode(void)101 static __always_inline void __enter_from_user_mode(void)
102 {
103 	lockdep_hardirqs_off(CALLER_ADDR0);
104 	CT_WARN_ON(ct_state() != CONTEXT_USER);
105 	user_exit_irqoff();
106 	trace_hardirqs_off_finish();
107 }
108 
enter_from_user_mode(struct pt_regs * regs)109 static __always_inline void enter_from_user_mode(struct pt_regs *regs)
110 {
111 	__enter_from_user_mode();
112 }
113 
114 /*
115  * Handle IRQ/context state management when exiting to user mode.
116  * After this function returns it is not safe to call regular kernel code,
117  * intrumentable code, or any code which may trigger an exception.
118  */
__exit_to_user_mode(void)119 static __always_inline void __exit_to_user_mode(void)
120 {
121 	trace_hardirqs_on_prepare();
122 	lockdep_hardirqs_on_prepare();
123 	user_enter_irqoff();
124 	lockdep_hardirqs_on(CALLER_ADDR0);
125 }
126 
prepare_exit_to_user_mode(struct pt_regs * regs)127 static __always_inline void prepare_exit_to_user_mode(struct pt_regs *regs)
128 {
129 	unsigned long flags;
130 
131 	local_daif_mask();
132 
133 	flags = READ_ONCE(current_thread_info()->flags);
134 	if (unlikely(flags & _TIF_WORK_MASK))
135 		do_notify_resume(regs, flags);
136 }
137 
exit_to_user_mode(struct pt_regs * regs)138 static __always_inline void exit_to_user_mode(struct pt_regs *regs)
139 {
140 	prepare_exit_to_user_mode(regs);
141 	mte_check_tfsr_exit();
142 	__exit_to_user_mode();
143 }
144 
asm_exit_to_user_mode(struct pt_regs * regs)145 asmlinkage void noinstr asm_exit_to_user_mode(struct pt_regs *regs)
146 {
147 	exit_to_user_mode(regs);
148 }
149 
150 /*
151  * Handle IRQ/context state management when entering an NMI from user/kernel
152  * mode. Before this function is called it is not safe to call regular kernel
153  * code, intrumentable code, or any code which may trigger an exception.
154  */
arm64_enter_nmi(struct pt_regs * regs)155 static void noinstr arm64_enter_nmi(struct pt_regs *regs)
156 {
157 	regs->lockdep_hardirqs = lockdep_hardirqs_enabled();
158 
159 	__nmi_enter();
160 	lockdep_hardirqs_off(CALLER_ADDR0);
161 	lockdep_hardirq_enter();
162 	rcu_nmi_enter();
163 
164 	trace_hardirqs_off_finish();
165 	ftrace_nmi_enter();
166 }
167 
168 /*
169  * Handle IRQ/context state management when exiting an NMI from user/kernel
170  * mode. After this function returns it is not safe to call regular kernel
171  * code, intrumentable code, or any code which may trigger an exception.
172  */
arm64_exit_nmi(struct pt_regs * regs)173 static void noinstr arm64_exit_nmi(struct pt_regs *regs)
174 {
175 	bool restore = regs->lockdep_hardirqs;
176 
177 	ftrace_nmi_exit();
178 	if (restore) {
179 		trace_hardirqs_on_prepare();
180 		lockdep_hardirqs_on_prepare();
181 	}
182 
183 	rcu_nmi_exit();
184 	lockdep_hardirq_exit();
185 	if (restore)
186 		lockdep_hardirqs_on(CALLER_ADDR0);
187 	__nmi_exit();
188 }
189 
190 /*
191  * Handle IRQ/context state management when entering a debug exception from
192  * kernel mode. Before this function is called it is not safe to call regular
193  * kernel code, intrumentable code, or any code which may trigger an exception.
194  */
arm64_enter_el1_dbg(struct pt_regs * regs)195 static void noinstr arm64_enter_el1_dbg(struct pt_regs *regs)
196 {
197 	regs->lockdep_hardirqs = lockdep_hardirqs_enabled();
198 
199 	lockdep_hardirqs_off(CALLER_ADDR0);
200 	rcu_nmi_enter();
201 
202 	trace_hardirqs_off_finish();
203 }
204 
205 /*
206  * Handle IRQ/context state management when exiting a debug exception from
207  * kernel mode. After this function returns it is not safe to call regular
208  * kernel code, intrumentable code, or any code which may trigger an exception.
209  */
arm64_exit_el1_dbg(struct pt_regs * regs)210 static void noinstr arm64_exit_el1_dbg(struct pt_regs *regs)
211 {
212 	bool restore = regs->lockdep_hardirqs;
213 
214 	if (restore) {
215 		trace_hardirqs_on_prepare();
216 		lockdep_hardirqs_on_prepare();
217 	}
218 
219 	rcu_nmi_exit();
220 	if (restore)
221 		lockdep_hardirqs_on(CALLER_ADDR0);
222 }
223 
enter_el1_irq_or_nmi(struct pt_regs * regs)224 static void noinstr enter_el1_irq_or_nmi(struct pt_regs *regs)
225 {
226 	if (IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && !interrupts_enabled(regs))
227 		arm64_enter_nmi(regs);
228 	else
229 		enter_from_kernel_mode(regs);
230 }
231 
exit_el1_irq_or_nmi(struct pt_regs * regs)232 static void noinstr exit_el1_irq_or_nmi(struct pt_regs *regs)
233 {
234 	if (IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && !interrupts_enabled(regs))
235 		arm64_exit_nmi(regs);
236 	else
237 		exit_to_kernel_mode(regs);
238 }
239 
arm64_preempt_schedule_irq(void)240 static void __sched arm64_preempt_schedule_irq(void)
241 {
242 	lockdep_assert_irqs_disabled();
243 
244 	/*
245 	 * DAIF.DA are cleared at the start of IRQ/FIQ handling, and when GIC
246 	 * priority masking is used the GIC irqchip driver will clear DAIF.IF
247 	 * using gic_arch_enable_irqs() for normal IRQs. If anything is set in
248 	 * DAIF we must have handled an NMI, so skip preemption.
249 	 */
250 	if (system_uses_irq_prio_masking() && read_sysreg(daif))
251 		return;
252 
253 	/*
254 	 * Preempting a task from an IRQ means we leave copies of PSTATE
255 	 * on the stack. cpufeature's enable calls may modify PSTATE, but
256 	 * resuming one of these preempted tasks would undo those changes.
257 	 *
258 	 * Only allow a task to be preempted once cpufeatures have been
259 	 * enabled.
260 	 */
261 	if (system_capabilities_finalized())
262 		preempt_schedule_irq();
263 }
264 
do_interrupt_handler(struct pt_regs * regs,void (* handler)(struct pt_regs *))265 static void do_interrupt_handler(struct pt_regs *regs,
266 				 void (*handler)(struct pt_regs *))
267 {
268 	if (on_thread_stack())
269 		call_on_irq_stack(regs, handler);
270 	else
271 		handler(regs);
272 }
273 
274 extern void (*handle_arch_irq)(struct pt_regs *);
275 extern void (*handle_arch_fiq)(struct pt_regs *);
276 
__panic_unhandled(struct pt_regs * regs,const char * vector,unsigned int esr)277 static void noinstr __panic_unhandled(struct pt_regs *regs, const char *vector,
278 				      unsigned int esr)
279 {
280 	arm64_enter_nmi(regs);
281 
282 	console_verbose();
283 
284 	pr_crit("Unhandled %s exception on CPU%d, ESR 0x%08x -- %s\n",
285 		vector, smp_processor_id(), esr,
286 		esr_get_class_string(esr));
287 
288 	trace_android_rvh_panic_unhandled(regs, vector, esr);
289 	__show_regs(regs);
290 	panic("Unhandled exception");
291 }
292 
293 #define UNHANDLED(el, regsize, vector)							\
294 asmlinkage void noinstr el##_##regsize##_##vector##_handler(struct pt_regs *regs)	\
295 {											\
296 	const char *desc = #regsize "-bit " #el " " #vector;				\
297 	__panic_unhandled(regs, desc, read_sysreg(esr_el1));				\
298 }
299 
300 #ifdef CONFIG_ARM64_ERRATUM_1463225
301 static DEFINE_PER_CPU(int, __in_cortex_a76_erratum_1463225_wa);
302 
cortex_a76_erratum_1463225_svc_handler(void)303 static void cortex_a76_erratum_1463225_svc_handler(void)
304 {
305 	u32 reg, val;
306 
307 	if (!unlikely(test_thread_flag(TIF_SINGLESTEP)))
308 		return;
309 
310 	if (!unlikely(this_cpu_has_cap(ARM64_WORKAROUND_1463225)))
311 		return;
312 
313 	__this_cpu_write(__in_cortex_a76_erratum_1463225_wa, 1);
314 	reg = read_sysreg(mdscr_el1);
315 	val = reg | DBG_MDSCR_SS | DBG_MDSCR_KDE;
316 	write_sysreg(val, mdscr_el1);
317 	asm volatile("msr daifclr, #8");
318 	isb();
319 
320 	/* We will have taken a single-step exception by this point */
321 
322 	write_sysreg(reg, mdscr_el1);
323 	__this_cpu_write(__in_cortex_a76_erratum_1463225_wa, 0);
324 }
325 
326 static __always_inline bool
cortex_a76_erratum_1463225_debug_handler(struct pt_regs * regs)327 cortex_a76_erratum_1463225_debug_handler(struct pt_regs *regs)
328 {
329 	if (!__this_cpu_read(__in_cortex_a76_erratum_1463225_wa))
330 		return false;
331 
332 	/*
333 	 * We've taken a dummy step exception from the kernel to ensure
334 	 * that interrupts are re-enabled on the syscall path. Return back
335 	 * to cortex_a76_erratum_1463225_svc_handler() with debug exceptions
336 	 * masked so that we can safely restore the mdscr and get on with
337 	 * handling the syscall.
338 	 */
339 	regs->pstate |= PSR_D_BIT;
340 	return true;
341 }
342 #else /* CONFIG_ARM64_ERRATUM_1463225 */
cortex_a76_erratum_1463225_svc_handler(void)343 static void cortex_a76_erratum_1463225_svc_handler(void) { }
cortex_a76_erratum_1463225_debug_handler(struct pt_regs * regs)344 static bool cortex_a76_erratum_1463225_debug_handler(struct pt_regs *regs)
345 {
346 	return false;
347 }
348 #endif /* CONFIG_ARM64_ERRATUM_1463225 */
349 
350 UNHANDLED(el1t, 64, sync)
351 UNHANDLED(el1t, 64, irq)
352 UNHANDLED(el1t, 64, fiq)
353 UNHANDLED(el1t, 64, error)
354 
el1_abort(struct pt_regs * regs,unsigned long esr)355 static void noinstr el1_abort(struct pt_regs *regs, unsigned long esr)
356 {
357 	unsigned long far = read_sysreg(far_el1);
358 
359 	enter_from_kernel_mode(regs);
360 	local_daif_inherit(regs);
361 	do_mem_abort(far, esr, regs);
362 	local_daif_mask();
363 	exit_to_kernel_mode(regs);
364 }
365 
el1_pc(struct pt_regs * regs,unsigned long esr)366 static void noinstr el1_pc(struct pt_regs *regs, unsigned long esr)
367 {
368 	unsigned long far = read_sysreg(far_el1);
369 
370 	enter_from_kernel_mode(regs);
371 	local_daif_inherit(regs);
372 	do_sp_pc_abort(far, esr, regs);
373 	local_daif_mask();
374 	exit_to_kernel_mode(regs);
375 }
376 
el1_undef(struct pt_regs * regs,unsigned long esr)377 static void noinstr el1_undef(struct pt_regs *regs, unsigned long esr)
378 {
379 	enter_from_kernel_mode(regs);
380 	local_daif_inherit(regs);
381 	do_el1_undef(regs, esr);
382 	local_daif_mask();
383 	exit_to_kernel_mode(regs);
384 }
385 
el1_bti(struct pt_regs * regs,unsigned long esr)386 static void noinstr el1_bti(struct pt_regs *regs, unsigned long esr)
387 {
388 	enter_from_kernel_mode(regs);
389 	local_daif_inherit(regs);
390 	do_el1_bti(regs, esr);
391 	local_daif_mask();
392 	exit_to_kernel_mode(regs);
393 }
394 
el1_dbg(struct pt_regs * regs,unsigned long esr)395 static void noinstr el1_dbg(struct pt_regs *regs, unsigned long esr)
396 {
397 	unsigned long far = read_sysreg(far_el1);
398 
399 	arm64_enter_el1_dbg(regs);
400 	if (!cortex_a76_erratum_1463225_debug_handler(regs))
401 		do_debug_exception(far, esr, regs);
402 	arm64_exit_el1_dbg(regs);
403 }
404 
el1_fpac(struct pt_regs * regs,unsigned long esr)405 static void noinstr el1_fpac(struct pt_regs *regs, unsigned long esr)
406 {
407 	enter_from_kernel_mode(regs);
408 	local_daif_inherit(regs);
409 	do_el1_fpac(regs, esr);
410 	local_daif_mask();
411 	exit_to_kernel_mode(regs);
412 }
413 
el1h_64_sync_handler(struct pt_regs * regs)414 asmlinkage void noinstr el1h_64_sync_handler(struct pt_regs *regs)
415 {
416 	unsigned long esr = read_sysreg(esr_el1);
417 
418 	switch (ESR_ELx_EC(esr)) {
419 	case ESR_ELx_EC_DABT_CUR:
420 	case ESR_ELx_EC_IABT_CUR:
421 		el1_abort(regs, esr);
422 		break;
423 	/*
424 	 * We don't handle ESR_ELx_EC_SP_ALIGN, since we will have hit a
425 	 * recursive exception when trying to push the initial pt_regs.
426 	 */
427 	case ESR_ELx_EC_PC_ALIGN:
428 		el1_pc(regs, esr);
429 		break;
430 	case ESR_ELx_EC_SYS64:
431 	case ESR_ELx_EC_UNKNOWN:
432 		el1_undef(regs, esr);
433 		break;
434 	case ESR_ELx_EC_BTI:
435 		el1_bti(regs, esr);
436 		break;
437 	case ESR_ELx_EC_BREAKPT_CUR:
438 	case ESR_ELx_EC_SOFTSTP_CUR:
439 	case ESR_ELx_EC_WATCHPT_CUR:
440 	case ESR_ELx_EC_BRK64:
441 		el1_dbg(regs, esr);
442 		break;
443 	case ESR_ELx_EC_FPAC:
444 		el1_fpac(regs, esr);
445 		break;
446 	default:
447 		__panic_unhandled(regs, "64-bit el1h sync", esr);
448 	}
449 }
450 
el1_interrupt(struct pt_regs * regs,void (* handler)(struct pt_regs *))451 static void noinstr el1_interrupt(struct pt_regs *regs,
452 				  void (*handler)(struct pt_regs *))
453 {
454 	write_sysreg(DAIF_PROCCTX_NOIRQ, daif);
455 
456 	enter_el1_irq_or_nmi(regs);
457 	do_interrupt_handler(regs, handler);
458 
459 	/*
460 	 * Note: thread_info::preempt_count includes both thread_info::count
461 	 * and thread_info::need_resched, and is not equivalent to
462 	 * preempt_count().
463 	 */
464 	if (IS_ENABLED(CONFIG_PREEMPTION) &&
465 	    READ_ONCE(current_thread_info()->preempt_count) == 0)
466 		arm64_preempt_schedule_irq();
467 
468 	exit_el1_irq_or_nmi(regs);
469 }
470 
el1h_64_irq_handler(struct pt_regs * regs)471 asmlinkage void noinstr el1h_64_irq_handler(struct pt_regs *regs)
472 {
473 	el1_interrupt(regs, handle_arch_irq);
474 }
475 
el1h_64_fiq_handler(struct pt_regs * regs)476 asmlinkage void noinstr el1h_64_fiq_handler(struct pt_regs *regs)
477 {
478 	el1_interrupt(regs, handle_arch_fiq);
479 }
480 
el1h_64_error_handler(struct pt_regs * regs)481 asmlinkage void noinstr el1h_64_error_handler(struct pt_regs *regs)
482 {
483 	unsigned long esr = read_sysreg(esr_el1);
484 
485 	local_daif_restore(DAIF_ERRCTX);
486 	arm64_enter_nmi(regs);
487 	do_serror(regs, esr);
488 	arm64_exit_nmi(regs);
489 }
490 
el0_da(struct pt_regs * regs,unsigned long esr)491 static void noinstr el0_da(struct pt_regs *regs, unsigned long esr)
492 {
493 	unsigned long far = read_sysreg(far_el1);
494 
495 	enter_from_user_mode(regs);
496 	local_daif_restore(DAIF_PROCCTX);
497 	do_mem_abort(far, esr, regs);
498 	exit_to_user_mode(regs);
499 }
500 
el0_ia(struct pt_regs * regs,unsigned long esr)501 static void noinstr el0_ia(struct pt_regs *regs, unsigned long esr)
502 {
503 	unsigned long far = read_sysreg(far_el1);
504 
505 	/*
506 	 * We've taken an instruction abort from userspace and not yet
507 	 * re-enabled IRQs. If the address is a kernel address, apply
508 	 * BP hardening prior to enabling IRQs and pre-emption.
509 	 */
510 	if (!is_ttbr0_addr(far))
511 		arm64_apply_bp_hardening();
512 
513 	enter_from_user_mode(regs);
514 	local_daif_restore(DAIF_PROCCTX);
515 	do_mem_abort(far, esr, regs);
516 	exit_to_user_mode(regs);
517 }
518 
el0_fpsimd_acc(struct pt_regs * regs,unsigned long esr)519 static void noinstr el0_fpsimd_acc(struct pt_regs *regs, unsigned long esr)
520 {
521 	enter_from_user_mode(regs);
522 	local_daif_restore(DAIF_PROCCTX);
523 	do_fpsimd_acc(esr, regs);
524 	exit_to_user_mode(regs);
525 }
526 
el0_sve_acc(struct pt_regs * regs,unsigned long esr)527 static void noinstr el0_sve_acc(struct pt_regs *regs, unsigned long esr)
528 {
529 	enter_from_user_mode(regs);
530 	local_daif_restore(DAIF_PROCCTX);
531 	do_sve_acc(esr, regs);
532 	exit_to_user_mode(regs);
533 }
534 
el0_fpsimd_exc(struct pt_regs * regs,unsigned long esr)535 static void noinstr el0_fpsimd_exc(struct pt_regs *regs, unsigned long esr)
536 {
537 	enter_from_user_mode(regs);
538 	local_daif_restore(DAIF_PROCCTX);
539 	do_fpsimd_exc(esr, regs);
540 	exit_to_user_mode(regs);
541 }
542 
el0_sys(struct pt_regs * regs,unsigned long esr)543 static void noinstr el0_sys(struct pt_regs *regs, unsigned long esr)
544 {
545 	enter_from_user_mode(regs);
546 	local_daif_restore(DAIF_PROCCTX);
547 	do_el0_sys(esr, regs);
548 	exit_to_user_mode(regs);
549 }
550 
el0_pc(struct pt_regs * regs,unsigned long esr)551 static void noinstr el0_pc(struct pt_regs *regs, unsigned long esr)
552 {
553 	unsigned long far = read_sysreg(far_el1);
554 
555 	if (!is_ttbr0_addr(instruction_pointer(regs)))
556 		arm64_apply_bp_hardening();
557 
558 	enter_from_user_mode(regs);
559 	local_daif_restore(DAIF_PROCCTX);
560 	do_sp_pc_abort(far, esr, regs);
561 	exit_to_user_mode(regs);
562 }
563 
el0_sp(struct pt_regs * regs,unsigned long esr)564 static void noinstr el0_sp(struct pt_regs *regs, unsigned long esr)
565 {
566 	enter_from_user_mode(regs);
567 	local_daif_restore(DAIF_PROCCTX);
568 	do_sp_pc_abort(regs->sp, esr, regs);
569 	exit_to_user_mode(regs);
570 }
571 
el0_undef(struct pt_regs * regs,unsigned long esr)572 static void noinstr el0_undef(struct pt_regs *regs, unsigned long esr)
573 {
574 	enter_from_user_mode(regs);
575 	local_daif_restore(DAIF_PROCCTX);
576 	do_el0_undef(regs, esr);
577 	exit_to_user_mode(regs);
578 }
579 
el0_bti(struct pt_regs * regs)580 static void noinstr el0_bti(struct pt_regs *regs)
581 {
582 	enter_from_user_mode(regs);
583 	local_daif_restore(DAIF_PROCCTX);
584 	do_el0_bti(regs);
585 	exit_to_user_mode(regs);
586 }
587 
el0_inv(struct pt_regs * regs,unsigned long esr)588 static void noinstr el0_inv(struct pt_regs *regs, unsigned long esr)
589 {
590 	enter_from_user_mode(regs);
591 	local_daif_restore(DAIF_PROCCTX);
592 	bad_el0_sync(regs, 0, esr);
593 	exit_to_user_mode(regs);
594 }
595 
el0_dbg(struct pt_regs * regs,unsigned long esr)596 static void noinstr el0_dbg(struct pt_regs *regs, unsigned long esr)
597 {
598 	/* Only watchpoints write FAR_EL1, otherwise its UNKNOWN */
599 	unsigned long far = read_sysreg(far_el1);
600 
601 	enter_from_user_mode(regs);
602 	do_debug_exception(far, esr, regs);
603 	local_daif_restore(DAIF_PROCCTX);
604 	exit_to_user_mode(regs);
605 }
606 
el0_svc(struct pt_regs * regs)607 static void noinstr el0_svc(struct pt_regs *regs)
608 {
609 	enter_from_user_mode(regs);
610 	cortex_a76_erratum_1463225_svc_handler();
611 	do_el0_svc(regs);
612 	exit_to_user_mode(regs);
613 }
614 
el0_fpac(struct pt_regs * regs,unsigned long esr)615 static void noinstr el0_fpac(struct pt_regs *regs, unsigned long esr)
616 {
617 	enter_from_user_mode(regs);
618 	local_daif_restore(DAIF_PROCCTX);
619 	do_el0_fpac(regs, esr);
620 	exit_to_user_mode(regs);
621 }
622 
el0t_64_sync_handler(struct pt_regs * regs)623 asmlinkage void noinstr el0t_64_sync_handler(struct pt_regs *regs)
624 {
625 	unsigned long esr = read_sysreg(esr_el1);
626 
627 	switch (ESR_ELx_EC(esr)) {
628 	case ESR_ELx_EC_SVC64:
629 		el0_svc(regs);
630 		break;
631 	case ESR_ELx_EC_DABT_LOW:
632 		el0_da(regs, esr);
633 		break;
634 	case ESR_ELx_EC_IABT_LOW:
635 		el0_ia(regs, esr);
636 		break;
637 	case ESR_ELx_EC_FP_ASIMD:
638 		el0_fpsimd_acc(regs, esr);
639 		break;
640 	case ESR_ELx_EC_SVE:
641 		el0_sve_acc(regs, esr);
642 		break;
643 	case ESR_ELx_EC_FP_EXC64:
644 		el0_fpsimd_exc(regs, esr);
645 		break;
646 	case ESR_ELx_EC_SYS64:
647 	case ESR_ELx_EC_WFx:
648 		el0_sys(regs, esr);
649 		break;
650 	case ESR_ELx_EC_SP_ALIGN:
651 		el0_sp(regs, esr);
652 		break;
653 	case ESR_ELx_EC_PC_ALIGN:
654 		el0_pc(regs, esr);
655 		break;
656 	case ESR_ELx_EC_UNKNOWN:
657 		el0_undef(regs, esr);
658 		break;
659 	case ESR_ELx_EC_BTI:
660 		el0_bti(regs);
661 		break;
662 	case ESR_ELx_EC_BREAKPT_LOW:
663 	case ESR_ELx_EC_SOFTSTP_LOW:
664 	case ESR_ELx_EC_WATCHPT_LOW:
665 	case ESR_ELx_EC_BRK64:
666 		el0_dbg(regs, esr);
667 		break;
668 	case ESR_ELx_EC_FPAC:
669 		el0_fpac(regs, esr);
670 		break;
671 	default:
672 		el0_inv(regs, esr);
673 	}
674 }
675 
el0_interrupt(struct pt_regs * regs,void (* handler)(struct pt_regs *))676 static void noinstr el0_interrupt(struct pt_regs *regs,
677 				  void (*handler)(struct pt_regs *))
678 {
679 	enter_from_user_mode(regs);
680 
681 	write_sysreg(DAIF_PROCCTX_NOIRQ, daif);
682 
683 	if (regs->pc & BIT(55))
684 		arm64_apply_bp_hardening();
685 
686 	do_interrupt_handler(regs, handler);
687 
688 	exit_to_user_mode(regs);
689 }
690 
__el0_irq_handler_common(struct pt_regs * regs)691 static void noinstr __el0_irq_handler_common(struct pt_regs *regs)
692 {
693 	el0_interrupt(regs, handle_arch_irq);
694 }
695 
el0t_64_irq_handler(struct pt_regs * regs)696 asmlinkage void noinstr el0t_64_irq_handler(struct pt_regs *regs)
697 {
698 	__el0_irq_handler_common(regs);
699 }
700 
__el0_fiq_handler_common(struct pt_regs * regs)701 static void noinstr __el0_fiq_handler_common(struct pt_regs *regs)
702 {
703 	el0_interrupt(regs, handle_arch_fiq);
704 }
705 
el0t_64_fiq_handler(struct pt_regs * regs)706 asmlinkage void noinstr el0t_64_fiq_handler(struct pt_regs *regs)
707 {
708 	__el0_fiq_handler_common(regs);
709 }
710 
__el0_error_handler_common(struct pt_regs * regs)711 static void noinstr __el0_error_handler_common(struct pt_regs *regs)
712 {
713 	unsigned long esr = read_sysreg(esr_el1);
714 
715 	enter_from_user_mode(regs);
716 	local_daif_restore(DAIF_ERRCTX);
717 	arm64_enter_nmi(regs);
718 	do_serror(regs, esr);
719 	arm64_exit_nmi(regs);
720 	local_daif_restore(DAIF_PROCCTX);
721 	exit_to_user_mode(regs);
722 }
723 
el0t_64_error_handler(struct pt_regs * regs)724 asmlinkage void noinstr el0t_64_error_handler(struct pt_regs *regs)
725 {
726 	__el0_error_handler_common(regs);
727 }
728 
729 #ifdef CONFIG_COMPAT
el0_cp15(struct pt_regs * regs,unsigned long esr)730 static void noinstr el0_cp15(struct pt_regs *regs, unsigned long esr)
731 {
732 	enter_from_user_mode(regs);
733 	local_daif_restore(DAIF_PROCCTX);
734 	do_el0_cp15(esr, regs);
735 	exit_to_user_mode(regs);
736 }
737 
el0_svc_compat(struct pt_regs * regs)738 static void noinstr el0_svc_compat(struct pt_regs *regs)
739 {
740 	enter_from_user_mode(regs);
741 	cortex_a76_erratum_1463225_svc_handler();
742 	do_el0_svc_compat(regs);
743 	exit_to_user_mode(regs);
744 }
745 
el0t_32_sync_handler(struct pt_regs * regs)746 asmlinkage void noinstr el0t_32_sync_handler(struct pt_regs *regs)
747 {
748 	unsigned long esr = read_sysreg(esr_el1);
749 
750 	switch (ESR_ELx_EC(esr)) {
751 	case ESR_ELx_EC_SVC32:
752 		el0_svc_compat(regs);
753 		break;
754 	case ESR_ELx_EC_DABT_LOW:
755 		el0_da(regs, esr);
756 		break;
757 	case ESR_ELx_EC_IABT_LOW:
758 		el0_ia(regs, esr);
759 		break;
760 	case ESR_ELx_EC_FP_ASIMD:
761 		el0_fpsimd_acc(regs, esr);
762 		break;
763 	case ESR_ELx_EC_FP_EXC32:
764 		el0_fpsimd_exc(regs, esr);
765 		break;
766 	case ESR_ELx_EC_PC_ALIGN:
767 		el0_pc(regs, esr);
768 		break;
769 	case ESR_ELx_EC_UNKNOWN:
770 	case ESR_ELx_EC_CP14_MR:
771 	case ESR_ELx_EC_CP14_LS:
772 	case ESR_ELx_EC_CP14_64:
773 		el0_undef(regs, esr);
774 		break;
775 	case ESR_ELx_EC_CP15_32:
776 	case ESR_ELx_EC_CP15_64:
777 		el0_cp15(regs, esr);
778 		break;
779 	case ESR_ELx_EC_BREAKPT_LOW:
780 	case ESR_ELx_EC_SOFTSTP_LOW:
781 	case ESR_ELx_EC_WATCHPT_LOW:
782 	case ESR_ELx_EC_BKPT32:
783 		el0_dbg(regs, esr);
784 		break;
785 	default:
786 		el0_inv(regs, esr);
787 	}
788 }
789 
el0t_32_irq_handler(struct pt_regs * regs)790 asmlinkage void noinstr el0t_32_irq_handler(struct pt_regs *regs)
791 {
792 	__el0_irq_handler_common(regs);
793 }
794 
el0t_32_fiq_handler(struct pt_regs * regs)795 asmlinkage void noinstr el0t_32_fiq_handler(struct pt_regs *regs)
796 {
797 	__el0_fiq_handler_common(regs);
798 }
799 
el0t_32_error_handler(struct pt_regs * regs)800 asmlinkage void noinstr el0t_32_error_handler(struct pt_regs *regs)
801 {
802 	__el0_error_handler_common(regs);
803 }
804 #else /* CONFIG_COMPAT */
805 UNHANDLED(el0t, 32, sync)
806 UNHANDLED(el0t, 32, irq)
807 UNHANDLED(el0t, 32, fiq)
808 UNHANDLED(el0t, 32, error)
809 #endif /* CONFIG_COMPAT */
810 
811 #ifdef CONFIG_VMAP_STACK
handle_bad_stack(struct pt_regs * regs)812 asmlinkage void noinstr handle_bad_stack(struct pt_regs *regs)
813 {
814 	unsigned int esr = read_sysreg(esr_el1);
815 	unsigned long far = read_sysreg(far_el1);
816 
817 	arm64_enter_nmi(regs);
818 	panic_bad_stack(regs, esr, far);
819 }
820 #endif /* CONFIG_VMAP_STACK */
821 
822 #ifdef CONFIG_ARM_SDE_INTERFACE
823 asmlinkage noinstr unsigned long
__sdei_handler(struct pt_regs * regs,struct sdei_registered_event * arg)824 __sdei_handler(struct pt_regs *regs, struct sdei_registered_event *arg)
825 {
826 	unsigned long ret;
827 
828 	/*
829 	 * We didn't take an exception to get here, so the HW hasn't
830 	 * set/cleared bits in PSTATE that we may rely on.
831 	 *
832 	 * The original SDEI spec (ARM DEN 0054A) can be read ambiguously as to
833 	 * whether PSTATE bits are inherited unchanged or generated from
834 	 * scratch, and the TF-A implementation always clears PAN and always
835 	 * clears UAO. There are no other known implementations.
836 	 *
837 	 * Subsequent revisions (ARM DEN 0054B) follow the usual rules for how
838 	 * PSTATE is modified upon architectural exceptions, and so PAN is
839 	 * either inherited or set per SCTLR_ELx.SPAN, and UAO is always
840 	 * cleared.
841 	 *
842 	 * We must explicitly reset PAN to the expected state, including
843 	 * clearing it when the host isn't using it, in case a VM had it set.
844 	 */
845 	if (system_uses_hw_pan())
846 		set_pstate_pan(1);
847 	else if (cpu_has_pan())
848 		set_pstate_pan(0);
849 
850 	arm64_enter_nmi(regs);
851 	ret = do_sdei_event(regs, arg);
852 	arm64_exit_nmi(regs);
853 
854 	return ret;
855 }
856 #endif /* CONFIG_ARM_SDE_INTERFACE */
857