1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Based on arch/arm/kernel/traps.c
4 *
5 * Copyright (C) 1995-2009 Russell King
6 * Copyright (C) 2012 ARM Ltd.
7 */
8
9 #include <linux/bug.h>
10 #include <linux/context_tracking.h>
11 #include <linux/signal.h>
12 #include <linux/personality.h>
13 #include <linux/kallsyms.h>
14 #include <linux/kprobes.h>
15 #include <linux/spinlock.h>
16 #include <linux/uaccess.h>
17 #include <linux/hardirq.h>
18 #include <linux/kdebug.h>
19 #include <linux/module.h>
20 #include <linux/kexec.h>
21 #include <linux/delay.h>
22 #include <linux/init.h>
23 #include <linux/sched/signal.h>
24 #include <linux/sched/debug.h>
25 #include <linux/sched/task_stack.h>
26 #include <linux/sizes.h>
27 #include <linux/syscalls.h>
28 #include <linux/mm_types.h>
29 #include <linux/kasan.h>
30
31 #include <asm/atomic.h>
32 #include <asm/bug.h>
33 #include <asm/cpufeature.h>
34 #include <asm/daifflags.h>
35 #include <asm/debug-monitors.h>
36 #include <asm/esr.h>
37 #include <asm/exception.h>
38 #include <asm/extable.h>
39 #include <asm/insn.h>
40 #include <asm/kprobes.h>
41 #include <asm/traps.h>
42 #include <asm/smp.h>
43 #include <asm/stack_pointer.h>
44 #include <asm/stacktrace.h>
45 #include <asm/exception.h>
46 #include <asm/system_misc.h>
47 #include <asm/sysreg.h>
48
49 #include <trace/hooks/traps.h>
50
51 static const char *handler[]= {
52 "Synchronous Abort",
53 "IRQ",
54 "FIQ",
55 "Error"
56 };
57
58 int show_unhandled_signals = 0;
59
dump_kernel_instr(const char * lvl,struct pt_regs * regs)60 static void dump_kernel_instr(const char *lvl, struct pt_regs *regs)
61 {
62 unsigned long addr = instruction_pointer(regs);
63 char str[sizeof("00000000 ") * 5 + 2 + 1], *p = str;
64 int i;
65
66 if (user_mode(regs))
67 return;
68
69 for (i = -4; i < 1; i++) {
70 unsigned int val, bad;
71
72 bad = aarch64_insn_read(&((u32 *)addr)[i], &val);
73
74 if (!bad)
75 p += sprintf(p, i == 0 ? "(%08x) " : "%08x ", val);
76 else {
77 p += sprintf(p, "bad PC value");
78 break;
79 }
80 }
81
82 printk("%sCode: %s\n", lvl, str);
83 }
84
85 #ifdef CONFIG_PREEMPT
86 #define S_PREEMPT " PREEMPT"
87 #elif defined(CONFIG_PREEMPT_RT)
88 #define S_PREEMPT " PREEMPT_RT"
89 #else
90 #define S_PREEMPT ""
91 #endif
92
93 #define S_SMP " SMP"
94
__die(const char * str,int err,struct pt_regs * regs)95 static int __die(const char *str, int err, struct pt_regs *regs)
96 {
97 static int die_counter;
98 int ret;
99
100 pr_emerg("Internal error: %s: %x [#%d]" S_PREEMPT S_SMP "\n",
101 str, err, ++die_counter);
102
103 /* trap and error numbers are mostly meaningless on ARM */
104 ret = notify_die(DIE_OOPS, str, regs, err, 0, SIGSEGV);
105 if (ret == NOTIFY_STOP)
106 return ret;
107
108 print_modules();
109 show_regs(regs);
110
111 dump_kernel_instr(KERN_EMERG, regs);
112
113 return ret;
114 }
115
116 static DEFINE_RAW_SPINLOCK(die_lock);
117
118 /*
119 * This function is protected against re-entrancy.
120 */
die(const char * str,struct pt_regs * regs,int err)121 void die(const char *str, struct pt_regs *regs, int err)
122 {
123 int ret;
124 unsigned long flags;
125
126 raw_spin_lock_irqsave(&die_lock, flags);
127
128 oops_enter();
129
130 console_verbose();
131 bust_spinlocks(1);
132 ret = __die(str, err, regs);
133
134 if (regs && kexec_should_crash(current))
135 crash_kexec(regs);
136
137 bust_spinlocks(0);
138 add_taint(TAINT_DIE, LOCKDEP_NOW_UNRELIABLE);
139 oops_exit();
140
141 if (in_interrupt())
142 panic("%s: Fatal exception in interrupt", str);
143 if (panic_on_oops)
144 panic("%s: Fatal exception", str);
145
146 raw_spin_unlock_irqrestore(&die_lock, flags);
147
148 if (ret != NOTIFY_STOP)
149 make_task_dead(SIGSEGV);
150 }
151
arm64_show_signal(int signo,const char * str)152 static void arm64_show_signal(int signo, const char *str)
153 {
154 static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL,
155 DEFAULT_RATELIMIT_BURST);
156 struct task_struct *tsk = current;
157 unsigned int esr = tsk->thread.fault_code;
158 struct pt_regs *regs = task_pt_regs(tsk);
159
160 /* Leave if the signal won't be shown */
161 if (!show_unhandled_signals ||
162 !unhandled_signal(tsk, signo) ||
163 !__ratelimit(&rs))
164 return;
165
166 pr_info("%s[%d]: unhandled exception: ", tsk->comm, task_pid_nr(tsk));
167 if (esr)
168 pr_cont("%s, ESR 0x%08x, ", esr_get_class_string(esr), esr);
169
170 pr_cont("%s", str);
171 print_vma_addr(KERN_CONT " in ", regs->pc);
172 pr_cont("\n");
173 __show_regs(regs);
174 }
175
arm64_force_sig_fault(int signo,int code,unsigned long far,const char * str)176 void arm64_force_sig_fault(int signo, int code, unsigned long far,
177 const char *str)
178 {
179 arm64_show_signal(signo, str);
180 if (signo == SIGKILL)
181 force_sig(SIGKILL);
182 else
183 force_sig_fault(signo, code, (void __user *)far);
184 }
185
arm64_force_sig_mceerr(int code,unsigned long far,short lsb,const char * str)186 void arm64_force_sig_mceerr(int code, unsigned long far, short lsb,
187 const char *str)
188 {
189 arm64_show_signal(SIGBUS, str);
190 force_sig_mceerr(code, (void __user *)far, lsb);
191 }
192
arm64_force_sig_ptrace_errno_trap(int errno,unsigned long far,const char * str)193 void arm64_force_sig_ptrace_errno_trap(int errno, unsigned long far,
194 const char *str)
195 {
196 arm64_show_signal(SIGTRAP, str);
197 force_sig_ptrace_errno_trap(errno, (void __user *)far);
198 }
199
arm64_notify_die(const char * str,struct pt_regs * regs,int signo,int sicode,unsigned long far,int err)200 void arm64_notify_die(const char *str, struct pt_regs *regs,
201 int signo, int sicode, unsigned long far,
202 int err)
203 {
204 if (user_mode(regs)) {
205 WARN_ON(regs != current_pt_regs());
206 current->thread.fault_address = 0;
207 current->thread.fault_code = err;
208
209 arm64_force_sig_fault(signo, sicode, far, str);
210 } else {
211 die(str, regs, err);
212 }
213 }
214
215 #ifdef CONFIG_COMPAT
216 #define PSTATE_IT_1_0_SHIFT 25
217 #define PSTATE_IT_1_0_MASK (0x3 << PSTATE_IT_1_0_SHIFT)
218 #define PSTATE_IT_7_2_SHIFT 10
219 #define PSTATE_IT_7_2_MASK (0x3f << PSTATE_IT_7_2_SHIFT)
220
compat_get_it_state(struct pt_regs * regs)221 static u32 compat_get_it_state(struct pt_regs *regs)
222 {
223 u32 it, pstate = regs->pstate;
224
225 it = (pstate & PSTATE_IT_1_0_MASK) >> PSTATE_IT_1_0_SHIFT;
226 it |= ((pstate & PSTATE_IT_7_2_MASK) >> PSTATE_IT_7_2_SHIFT) << 2;
227
228 return it;
229 }
230
compat_set_it_state(struct pt_regs * regs,u32 it)231 static void compat_set_it_state(struct pt_regs *regs, u32 it)
232 {
233 u32 pstate_it;
234
235 pstate_it = (it << PSTATE_IT_1_0_SHIFT) & PSTATE_IT_1_0_MASK;
236 pstate_it |= ((it >> 2) << PSTATE_IT_7_2_SHIFT) & PSTATE_IT_7_2_MASK;
237
238 regs->pstate &= ~PSR_AA32_IT_MASK;
239 regs->pstate |= pstate_it;
240 }
241
advance_itstate(struct pt_regs * regs)242 static void advance_itstate(struct pt_regs *regs)
243 {
244 u32 it;
245
246 /* ARM mode */
247 if (!(regs->pstate & PSR_AA32_T_BIT) ||
248 !(regs->pstate & PSR_AA32_IT_MASK))
249 return;
250
251 it = compat_get_it_state(regs);
252
253 /*
254 * If this is the last instruction of the block, wipe the IT
255 * state. Otherwise advance it.
256 */
257 if (!(it & 7))
258 it = 0;
259 else
260 it = (it & 0xe0) | ((it << 1) & 0x1f);
261
262 compat_set_it_state(regs, it);
263 }
264 #else
advance_itstate(struct pt_regs * regs)265 static void advance_itstate(struct pt_regs *regs)
266 {
267 }
268 #endif
269
arm64_skip_faulting_instruction(struct pt_regs * regs,unsigned long size)270 void arm64_skip_faulting_instruction(struct pt_regs *regs, unsigned long size)
271 {
272 regs->pc += size;
273
274 /*
275 * If we were single stepping, we want to get the step exception after
276 * we return from the trap.
277 */
278 if (user_mode(regs))
279 user_fastforward_single_step(current);
280
281 if (compat_user_mode(regs))
282 advance_itstate(regs);
283 else
284 regs->pstate &= ~PSR_BTYPE_MASK;
285 }
286
287 static LIST_HEAD(undef_hook);
288 static DEFINE_RAW_SPINLOCK(undef_lock);
289
register_undef_hook(struct undef_hook * hook)290 void register_undef_hook(struct undef_hook *hook)
291 {
292 unsigned long flags;
293
294 raw_spin_lock_irqsave(&undef_lock, flags);
295 list_add(&hook->node, &undef_hook);
296 raw_spin_unlock_irqrestore(&undef_lock, flags);
297 }
298
unregister_undef_hook(struct undef_hook * hook)299 void unregister_undef_hook(struct undef_hook *hook)
300 {
301 unsigned long flags;
302
303 raw_spin_lock_irqsave(&undef_lock, flags);
304 list_del(&hook->node);
305 raw_spin_unlock_irqrestore(&undef_lock, flags);
306 }
307
call_undef_hook(struct pt_regs * regs)308 static int call_undef_hook(struct pt_regs *regs)
309 {
310 struct undef_hook *hook;
311 unsigned long flags;
312 u32 instr;
313 int (*fn)(struct pt_regs *regs, u32 instr) = NULL;
314 void __user *pc = (void __user *)instruction_pointer(regs);
315
316 if (!user_mode(regs)) {
317 __le32 instr_le;
318 if (get_kernel_nofault(instr_le, (__force __le32 *)pc))
319 goto exit;
320 instr = le32_to_cpu(instr_le);
321 } else if (compat_thumb_mode(regs)) {
322 /* 16-bit Thumb instruction */
323 __le16 instr_le;
324 if (get_user(instr_le, (__le16 __user *)pc))
325 goto exit;
326 instr = le16_to_cpu(instr_le);
327 if (aarch32_insn_is_wide(instr)) {
328 u32 instr2;
329
330 if (get_user(instr_le, (__le16 __user *)(pc + 2)))
331 goto exit;
332 instr2 = le16_to_cpu(instr_le);
333 instr = (instr << 16) | instr2;
334 }
335 } else {
336 /* 32-bit ARM instruction */
337 __le32 instr_le;
338 if (get_user(instr_le, (__le32 __user *)pc))
339 goto exit;
340 instr = le32_to_cpu(instr_le);
341 }
342
343 raw_spin_lock_irqsave(&undef_lock, flags);
344 list_for_each_entry(hook, &undef_hook, node)
345 if ((instr & hook->instr_mask) == hook->instr_val &&
346 (regs->pstate & hook->pstate_mask) == hook->pstate_val)
347 fn = hook->fn;
348
349 raw_spin_unlock_irqrestore(&undef_lock, flags);
350 exit:
351 return fn ? fn(regs, instr) : 1;
352 }
353
force_signal_inject(int signal,int code,unsigned long address,unsigned int err)354 void force_signal_inject(int signal, int code, unsigned long address, unsigned int err)
355 {
356 const char *desc;
357 struct pt_regs *regs = current_pt_regs();
358
359 if (WARN_ON(!user_mode(regs)))
360 return;
361
362 switch (signal) {
363 case SIGILL:
364 desc = "undefined instruction";
365 break;
366 case SIGSEGV:
367 desc = "illegal memory access";
368 break;
369 default:
370 desc = "unknown or unrecoverable error";
371 break;
372 }
373
374 /* Force signals we don't understand to SIGKILL */
375 if (WARN_ON(signal != SIGKILL &&
376 siginfo_layout(signal, code) != SIL_FAULT)) {
377 signal = SIGKILL;
378 }
379
380 arm64_notify_die(desc, regs, signal, code, address, err);
381 }
382
383 /*
384 * Set up process info to signal segmentation fault - called on access error.
385 */
arm64_notify_segfault(unsigned long addr)386 void arm64_notify_segfault(unsigned long addr)
387 {
388 int code;
389
390 mmap_read_lock(current->mm);
391 if (find_vma(current->mm, untagged_addr(addr)) == NULL)
392 code = SEGV_MAPERR;
393 else
394 code = SEGV_ACCERR;
395 mmap_read_unlock(current->mm);
396
397 force_signal_inject(SIGSEGV, code, addr, 0);
398 }
399
do_undefinstr(struct pt_regs * regs)400 void do_undefinstr(struct pt_regs *regs)
401 {
402 /* check for AArch32 breakpoint instructions */
403 if (!aarch32_break_handler(regs))
404 return;
405
406 if (call_undef_hook(regs) == 0)
407 return;
408
409 trace_android_rvh_do_undefinstr(regs, user_mode(regs));
410 BUG_ON(!user_mode(regs));
411 force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0);
412 }
413 NOKPROBE_SYMBOL(do_undefinstr);
414
do_bti(struct pt_regs * regs)415 void do_bti(struct pt_regs *regs)
416 {
417 BUG_ON(!user_mode(regs));
418 force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0);
419 }
420 NOKPROBE_SYMBOL(do_bti);
421
do_ptrauth_fault(struct pt_regs * regs,unsigned int esr)422 void do_ptrauth_fault(struct pt_regs *regs, unsigned int esr)
423 {
424 /*
425 * Unexpected FPAC exception or pointer authentication failure in
426 * the kernel: kill the task before it does any more harm.
427 */
428 BUG_ON(!user_mode(regs));
429 force_signal_inject(SIGILL, ILL_ILLOPN, regs->pc, esr);
430 }
431 NOKPROBE_SYMBOL(do_ptrauth_fault);
432
433 #define __user_cache_maint(insn, address, res) \
434 if (address >= user_addr_max()) { \
435 res = -EFAULT; \
436 } else { \
437 uaccess_ttbr0_enable(); \
438 asm volatile ( \
439 "1: " insn ", %1\n" \
440 " mov %w0, #0\n" \
441 "2:\n" \
442 " .pushsection .fixup,\"ax\"\n" \
443 " .align 2\n" \
444 "3: mov %w0, %w2\n" \
445 " b 2b\n" \
446 " .popsection\n" \
447 _ASM_EXTABLE(1b, 3b) \
448 : "=r" (res) \
449 : "r" (address), "i" (-EFAULT)); \
450 uaccess_ttbr0_disable(); \
451 }
452
user_cache_maint_handler(unsigned int esr,struct pt_regs * regs)453 static void user_cache_maint_handler(unsigned int esr, struct pt_regs *regs)
454 {
455 unsigned long tagged_address, address;
456 int rt = ESR_ELx_SYS64_ISS_RT(esr);
457 int crm = (esr & ESR_ELx_SYS64_ISS_CRM_MASK) >> ESR_ELx_SYS64_ISS_CRM_SHIFT;
458 int ret = 0;
459
460 tagged_address = pt_regs_read_reg(regs, rt);
461 address = untagged_addr(tagged_address);
462
463 switch (crm) {
464 case ESR_ELx_SYS64_ISS_CRM_DC_CVAU: /* DC CVAU, gets promoted */
465 __user_cache_maint("dc civac", address, ret);
466 break;
467 case ESR_ELx_SYS64_ISS_CRM_DC_CVAC: /* DC CVAC, gets promoted */
468 __user_cache_maint("dc civac", address, ret);
469 break;
470 case ESR_ELx_SYS64_ISS_CRM_DC_CVADP: /* DC CVADP */
471 __user_cache_maint("sys 3, c7, c13, 1", address, ret);
472 break;
473 case ESR_ELx_SYS64_ISS_CRM_DC_CVAP: /* DC CVAP */
474 __user_cache_maint("sys 3, c7, c12, 1", address, ret);
475 break;
476 case ESR_ELx_SYS64_ISS_CRM_DC_CIVAC: /* DC CIVAC */
477 __user_cache_maint("dc civac", address, ret);
478 break;
479 case ESR_ELx_SYS64_ISS_CRM_IC_IVAU: /* IC IVAU */
480 __user_cache_maint("ic ivau", address, ret);
481 break;
482 default:
483 force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0);
484 return;
485 }
486
487 if (ret)
488 arm64_notify_segfault(tagged_address);
489 else
490 arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE);
491 }
492
ctr_read_handler(unsigned int esr,struct pt_regs * regs)493 static void ctr_read_handler(unsigned int esr, struct pt_regs *regs)
494 {
495 int rt = ESR_ELx_SYS64_ISS_RT(esr);
496 unsigned long val = arm64_ftr_reg_user_value(&arm64_ftr_reg_ctrel0);
497
498 if (cpus_have_const_cap(ARM64_WORKAROUND_1542419)) {
499 /* Hide DIC so that we can trap the unnecessary maintenance...*/
500 val &= ~BIT(CTR_DIC_SHIFT);
501
502 /* ... and fake IminLine to reduce the number of traps. */
503 val &= ~CTR_IMINLINE_MASK;
504 val |= (PAGE_SHIFT - 2) & CTR_IMINLINE_MASK;
505 }
506
507 pt_regs_write_reg(regs, rt, val);
508
509 arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE);
510 }
511
cntvct_read_handler(unsigned int esr,struct pt_regs * regs)512 static void cntvct_read_handler(unsigned int esr, struct pt_regs *regs)
513 {
514 int rt = ESR_ELx_SYS64_ISS_RT(esr);
515
516 pt_regs_write_reg(regs, rt, arch_timer_read_counter());
517 arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE);
518 }
519
cntfrq_read_handler(unsigned int esr,struct pt_regs * regs)520 static void cntfrq_read_handler(unsigned int esr, struct pt_regs *regs)
521 {
522 int rt = ESR_ELx_SYS64_ISS_RT(esr);
523
524 pt_regs_write_reg(regs, rt, arch_timer_get_rate());
525 arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE);
526 }
527
mrs_handler(unsigned int esr,struct pt_regs * regs)528 static void mrs_handler(unsigned int esr, struct pt_regs *regs)
529 {
530 u32 sysreg, rt;
531
532 rt = ESR_ELx_SYS64_ISS_RT(esr);
533 sysreg = esr_sys64_to_sysreg(esr);
534
535 if (do_emulate_mrs(regs, sysreg, rt) != 0)
536 force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0);
537 }
538
wfi_handler(unsigned int esr,struct pt_regs * regs)539 static void wfi_handler(unsigned int esr, struct pt_regs *regs)
540 {
541 arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE);
542 }
543
544 struct sys64_hook {
545 unsigned int esr_mask;
546 unsigned int esr_val;
547 void (*handler)(unsigned int esr, struct pt_regs *regs);
548 };
549
550 static const struct sys64_hook sys64_hooks[] = {
551 {
552 .esr_mask = ESR_ELx_SYS64_ISS_EL0_CACHE_OP_MASK,
553 .esr_val = ESR_ELx_SYS64_ISS_EL0_CACHE_OP_VAL,
554 .handler = user_cache_maint_handler,
555 },
556 {
557 /* Trap read access to CTR_EL0 */
558 .esr_mask = ESR_ELx_SYS64_ISS_SYS_OP_MASK,
559 .esr_val = ESR_ELx_SYS64_ISS_SYS_CTR_READ,
560 .handler = ctr_read_handler,
561 },
562 {
563 /* Trap read access to CNTVCT_EL0 */
564 .esr_mask = ESR_ELx_SYS64_ISS_SYS_OP_MASK,
565 .esr_val = ESR_ELx_SYS64_ISS_SYS_CNTVCT,
566 .handler = cntvct_read_handler,
567 },
568 {
569 /* Trap read access to CNTFRQ_EL0 */
570 .esr_mask = ESR_ELx_SYS64_ISS_SYS_OP_MASK,
571 .esr_val = ESR_ELx_SYS64_ISS_SYS_CNTFRQ,
572 .handler = cntfrq_read_handler,
573 },
574 {
575 /* Trap read access to CPUID registers */
576 .esr_mask = ESR_ELx_SYS64_ISS_SYS_MRS_OP_MASK,
577 .esr_val = ESR_ELx_SYS64_ISS_SYS_MRS_OP_VAL,
578 .handler = mrs_handler,
579 },
580 {
581 /* Trap WFI instructions executed in userspace */
582 .esr_mask = ESR_ELx_WFx_MASK,
583 .esr_val = ESR_ELx_WFx_WFI_VAL,
584 .handler = wfi_handler,
585 },
586 {},
587 };
588
589 #ifdef CONFIG_COMPAT
cp15_cond_valid(unsigned int esr,struct pt_regs * regs)590 static bool cp15_cond_valid(unsigned int esr, struct pt_regs *regs)
591 {
592 int cond;
593
594 /* Only a T32 instruction can trap without CV being set */
595 if (!(esr & ESR_ELx_CV)) {
596 u32 it;
597
598 it = compat_get_it_state(regs);
599 if (!it)
600 return true;
601
602 cond = it >> 4;
603 } else {
604 cond = (esr & ESR_ELx_COND_MASK) >> ESR_ELx_COND_SHIFT;
605 }
606
607 return aarch32_opcode_cond_checks[cond](regs->pstate);
608 }
609
compat_cntfrq_read_handler(unsigned int esr,struct pt_regs * regs)610 static void compat_cntfrq_read_handler(unsigned int esr, struct pt_regs *regs)
611 {
612 int reg = (esr & ESR_ELx_CP15_32_ISS_RT_MASK) >> ESR_ELx_CP15_32_ISS_RT_SHIFT;
613
614 pt_regs_write_reg(regs, reg, arch_timer_get_rate());
615 arm64_skip_faulting_instruction(regs, 4);
616 }
617
618 static const struct sys64_hook cp15_32_hooks[] = {
619 {
620 .esr_mask = ESR_ELx_CP15_32_ISS_SYS_MASK,
621 .esr_val = ESR_ELx_CP15_32_ISS_SYS_CNTFRQ,
622 .handler = compat_cntfrq_read_handler,
623 },
624 {},
625 };
626
compat_cntvct_read_handler(unsigned int esr,struct pt_regs * regs)627 static void compat_cntvct_read_handler(unsigned int esr, struct pt_regs *regs)
628 {
629 int rt = (esr & ESR_ELx_CP15_64_ISS_RT_MASK) >> ESR_ELx_CP15_64_ISS_RT_SHIFT;
630 int rt2 = (esr & ESR_ELx_CP15_64_ISS_RT2_MASK) >> ESR_ELx_CP15_64_ISS_RT2_SHIFT;
631 u64 val = arch_timer_read_counter();
632
633 pt_regs_write_reg(regs, rt, lower_32_bits(val));
634 pt_regs_write_reg(regs, rt2, upper_32_bits(val));
635 arm64_skip_faulting_instruction(regs, 4);
636 }
637
638 static const struct sys64_hook cp15_64_hooks[] = {
639 {
640 .esr_mask = ESR_ELx_CP15_64_ISS_SYS_MASK,
641 .esr_val = ESR_ELx_CP15_64_ISS_SYS_CNTVCT,
642 .handler = compat_cntvct_read_handler,
643 },
644 {},
645 };
646
do_cp15instr(unsigned int esr,struct pt_regs * regs)647 void do_cp15instr(unsigned int esr, struct pt_regs *regs)
648 {
649 const struct sys64_hook *hook, *hook_base;
650
651 if (!cp15_cond_valid(esr, regs)) {
652 /*
653 * There is no T16 variant of a CP access, so we
654 * always advance PC by 4 bytes.
655 */
656 arm64_skip_faulting_instruction(regs, 4);
657 return;
658 }
659
660 switch (ESR_ELx_EC(esr)) {
661 case ESR_ELx_EC_CP15_32:
662 hook_base = cp15_32_hooks;
663 break;
664 case ESR_ELx_EC_CP15_64:
665 hook_base = cp15_64_hooks;
666 break;
667 default:
668 do_undefinstr(regs);
669 return;
670 }
671
672 for (hook = hook_base; hook->handler; hook++)
673 if ((hook->esr_mask & esr) == hook->esr_val) {
674 hook->handler(esr, regs);
675 return;
676 }
677
678 /*
679 * New cp15 instructions may previously have been undefined at
680 * EL0. Fall back to our usual undefined instruction handler
681 * so that we handle these consistently.
682 */
683 do_undefinstr(regs);
684 }
685 NOKPROBE_SYMBOL(do_cp15instr);
686 #endif
687
do_sysinstr(unsigned int esr,struct pt_regs * regs)688 void do_sysinstr(unsigned int esr, struct pt_regs *regs)
689 {
690 const struct sys64_hook *hook;
691
692 for (hook = sys64_hooks; hook->handler; hook++)
693 if ((hook->esr_mask & esr) == hook->esr_val) {
694 hook->handler(esr, regs);
695 return;
696 }
697
698 /*
699 * New SYS instructions may previously have been undefined at EL0. Fall
700 * back to our usual undefined instruction handler so that we handle
701 * these consistently.
702 */
703 do_undefinstr(regs);
704 }
705 NOKPROBE_SYMBOL(do_sysinstr);
706
707 static const char *esr_class_str[] = {
708 [0 ... ESR_ELx_EC_MAX] = "UNRECOGNIZED EC",
709 [ESR_ELx_EC_UNKNOWN] = "Unknown/Uncategorized",
710 [ESR_ELx_EC_WFx] = "WFI/WFE",
711 [ESR_ELx_EC_CP15_32] = "CP15 MCR/MRC",
712 [ESR_ELx_EC_CP15_64] = "CP15 MCRR/MRRC",
713 [ESR_ELx_EC_CP14_MR] = "CP14 MCR/MRC",
714 [ESR_ELx_EC_CP14_LS] = "CP14 LDC/STC",
715 [ESR_ELx_EC_FP_ASIMD] = "ASIMD",
716 [ESR_ELx_EC_CP10_ID] = "CP10 MRC/VMRS",
717 [ESR_ELx_EC_PAC] = "PAC",
718 [ESR_ELx_EC_CP14_64] = "CP14 MCRR/MRRC",
719 [ESR_ELx_EC_BTI] = "BTI",
720 [ESR_ELx_EC_ILL] = "PSTATE.IL",
721 [ESR_ELx_EC_SVC32] = "SVC (AArch32)",
722 [ESR_ELx_EC_HVC32] = "HVC (AArch32)",
723 [ESR_ELx_EC_SMC32] = "SMC (AArch32)",
724 [ESR_ELx_EC_SVC64] = "SVC (AArch64)",
725 [ESR_ELx_EC_HVC64] = "HVC (AArch64)",
726 [ESR_ELx_EC_SMC64] = "SMC (AArch64)",
727 [ESR_ELx_EC_SYS64] = "MSR/MRS (AArch64)",
728 [ESR_ELx_EC_SVE] = "SVE",
729 [ESR_ELx_EC_ERET] = "ERET/ERETAA/ERETAB",
730 [ESR_ELx_EC_FPAC] = "FPAC",
731 [ESR_ELx_EC_IMP_DEF] = "EL3 IMP DEF",
732 [ESR_ELx_EC_IABT_LOW] = "IABT (lower EL)",
733 [ESR_ELx_EC_IABT_CUR] = "IABT (current EL)",
734 [ESR_ELx_EC_PC_ALIGN] = "PC Alignment",
735 [ESR_ELx_EC_DABT_LOW] = "DABT (lower EL)",
736 [ESR_ELx_EC_DABT_CUR] = "DABT (current EL)",
737 [ESR_ELx_EC_SP_ALIGN] = "SP Alignment",
738 [ESR_ELx_EC_FP_EXC32] = "FP (AArch32)",
739 [ESR_ELx_EC_FP_EXC64] = "FP (AArch64)",
740 [ESR_ELx_EC_SERROR] = "SError",
741 [ESR_ELx_EC_BREAKPT_LOW] = "Breakpoint (lower EL)",
742 [ESR_ELx_EC_BREAKPT_CUR] = "Breakpoint (current EL)",
743 [ESR_ELx_EC_SOFTSTP_LOW] = "Software Step (lower EL)",
744 [ESR_ELx_EC_SOFTSTP_CUR] = "Software Step (current EL)",
745 [ESR_ELx_EC_WATCHPT_LOW] = "Watchpoint (lower EL)",
746 [ESR_ELx_EC_WATCHPT_CUR] = "Watchpoint (current EL)",
747 [ESR_ELx_EC_BKPT32] = "BKPT (AArch32)",
748 [ESR_ELx_EC_VECTOR32] = "Vector catch (AArch32)",
749 [ESR_ELx_EC_BRK64] = "BRK (AArch64)",
750 };
751
esr_get_class_string(u32 esr)752 const char *esr_get_class_string(u32 esr)
753 {
754 return esr_class_str[ESR_ELx_EC(esr)];
755 }
756
757 /*
758 * bad_mode handles the impossible case in the exception vector. This is always
759 * fatal.
760 */
bad_mode(struct pt_regs * regs,int reason,unsigned int esr)761 asmlinkage void notrace bad_mode(struct pt_regs *regs, int reason, unsigned int esr)
762 {
763 arm64_enter_nmi(regs);
764
765 console_verbose();
766
767 pr_crit("Bad mode in %s handler detected on CPU%d, code 0x%08x -- %s\n",
768 handler[reason], smp_processor_id(), esr,
769 esr_get_class_string(esr));
770
771 trace_android_rvh_bad_mode(regs, esr, reason);
772 __show_regs(regs);
773 local_daif_mask();
774 panic("bad mode");
775 }
776
777 /*
778 * bad_el0_sync handles unexpected, but potentially recoverable synchronous
779 * exceptions taken from EL0. Unlike bad_mode, this returns.
780 */
bad_el0_sync(struct pt_regs * regs,int reason,unsigned int esr)781 void bad_el0_sync(struct pt_regs *regs, int reason, unsigned int esr)
782 {
783 unsigned long pc = instruction_pointer(regs);
784
785 current->thread.fault_address = 0;
786 current->thread.fault_code = esr;
787
788 arm64_force_sig_fault(SIGILL, ILL_ILLOPC, pc,
789 "Bad EL0 synchronous exception");
790 }
791
792 #ifdef CONFIG_VMAP_STACK
793
794 DEFINE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)], overflow_stack)
795 __aligned(16);
796
handle_bad_stack(struct pt_regs * regs)797 asmlinkage void noinstr handle_bad_stack(struct pt_regs *regs)
798 {
799 unsigned long tsk_stk = (unsigned long)current->stack;
800 unsigned long irq_stk = (unsigned long)this_cpu_read(irq_stack_ptr);
801 unsigned long ovf_stk = (unsigned long)this_cpu_ptr(overflow_stack);
802 unsigned int esr = read_sysreg(esr_el1);
803 unsigned long far = read_sysreg(far_el1);
804
805 arm64_enter_nmi(regs);
806
807 console_verbose();
808 pr_emerg("Insufficient stack space to handle exception!");
809
810 pr_emerg("ESR: 0x%08x -- %s\n", esr, esr_get_class_string(esr));
811 pr_emerg("FAR: 0x%016lx\n", far);
812
813 pr_emerg("Task stack: [0x%016lx..0x%016lx]\n",
814 tsk_stk, tsk_stk + THREAD_SIZE);
815 pr_emerg("IRQ stack: [0x%016lx..0x%016lx]\n",
816 irq_stk, irq_stk + IRQ_STACK_SIZE);
817 pr_emerg("Overflow stack: [0x%016lx..0x%016lx]\n",
818 ovf_stk, ovf_stk + OVERFLOW_STACK_SIZE);
819
820 __show_regs(regs);
821
822 /*
823 * We use nmi_panic to limit the potential for recusive overflows, and
824 * to get a better stack trace.
825 */
826 nmi_panic(NULL, "kernel stack overflow");
827 cpu_park_loop();
828 }
829 #endif
830
arm64_serror_panic(struct pt_regs * regs,u32 esr)831 void __noreturn arm64_serror_panic(struct pt_regs *regs, u32 esr)
832 {
833 console_verbose();
834
835 pr_crit("SError Interrupt on CPU%d, code 0x%08x -- %s\n",
836 smp_processor_id(), esr, esr_get_class_string(esr));
837
838 trace_android_rvh_arm64_serror_panic(regs, esr);
839 if (regs)
840 __show_regs(regs);
841
842 nmi_panic(regs, "Asynchronous SError Interrupt");
843
844 cpu_park_loop();
845 unreachable();
846 }
847
arm64_is_fatal_ras_serror(struct pt_regs * regs,unsigned int esr)848 bool arm64_is_fatal_ras_serror(struct pt_regs *regs, unsigned int esr)
849 {
850 u32 aet = arm64_ras_serror_get_severity(esr);
851
852 switch (aet) {
853 case ESR_ELx_AET_CE: /* corrected error */
854 case ESR_ELx_AET_UEO: /* restartable, not yet consumed */
855 /*
856 * The CPU can make progress. We may take UEO again as
857 * a more severe error.
858 */
859 return false;
860
861 case ESR_ELx_AET_UEU: /* Uncorrected Unrecoverable */
862 case ESR_ELx_AET_UER: /* Uncorrected Recoverable */
863 /*
864 * The CPU can't make progress. The exception may have
865 * been imprecise.
866 *
867 * Neoverse-N1 #1349291 means a non-KVM SError reported as
868 * Unrecoverable should be treated as Uncontainable. We
869 * call arm64_serror_panic() in both cases.
870 */
871 return true;
872
873 case ESR_ELx_AET_UC: /* Uncontainable or Uncategorized error */
874 default:
875 /* Error has been silently propagated */
876 arm64_serror_panic(regs, esr);
877 }
878 }
879
do_serror(struct pt_regs * regs,unsigned int esr)880 asmlinkage void noinstr do_serror(struct pt_regs *regs, unsigned int esr)
881 {
882 arm64_enter_nmi(regs);
883
884 /* non-RAS errors are not containable */
885 if (!arm64_is_ras_serror(esr) || arm64_is_fatal_ras_serror(regs, esr))
886 arm64_serror_panic(regs, esr);
887
888 arm64_exit_nmi(regs);
889 }
890
891 /* GENERIC_BUG traps */
892
is_valid_bugaddr(unsigned long addr)893 int is_valid_bugaddr(unsigned long addr)
894 {
895 /*
896 * bug_handler() only called for BRK #BUG_BRK_IMM.
897 * So the answer is trivial -- any spurious instances with no
898 * bug table entry will be rejected by report_bug() and passed
899 * back to the debug-monitors code and handled as a fatal
900 * unexpected debug exception.
901 */
902 return 1;
903 }
904
bug_handler(struct pt_regs * regs,unsigned int esr)905 static int bug_handler(struct pt_regs *regs, unsigned int esr)
906 {
907 switch (report_bug(regs->pc, regs)) {
908 case BUG_TRAP_TYPE_BUG:
909 die("Oops - BUG", regs, 0);
910 break;
911
912 case BUG_TRAP_TYPE_WARN:
913 break;
914
915 default:
916 /* unknown/unrecognised bug trap type */
917 return DBG_HOOK_ERROR;
918 }
919
920 /* If thread survives, skip over the BUG instruction and continue: */
921 arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE);
922 return DBG_HOOK_HANDLED;
923 }
924
925 static struct break_hook bug_break_hook = {
926 .fn = bug_handler,
927 .imm = BUG_BRK_IMM,
928 };
929
reserved_fault_handler(struct pt_regs * regs,unsigned int esr)930 static int reserved_fault_handler(struct pt_regs *regs, unsigned int esr)
931 {
932 pr_err("%s generated an invalid instruction at %pS!\n",
933 "Kernel text patching",
934 (void *)instruction_pointer(regs));
935
936 /* We cannot handle this */
937 return DBG_HOOK_ERROR;
938 }
939
940 static struct break_hook fault_break_hook = {
941 .fn = reserved_fault_handler,
942 .imm = FAULT_BRK_IMM,
943 };
944
945 #ifdef CONFIG_KASAN_SW_TAGS
946
947 #define KASAN_ESR_RECOVER 0x20
948 #define KASAN_ESR_WRITE 0x10
949 #define KASAN_ESR_SIZE_MASK 0x0f
950 #define KASAN_ESR_SIZE(esr) (1 << ((esr) & KASAN_ESR_SIZE_MASK))
951
kasan_handler(struct pt_regs * regs,unsigned int esr)952 static int kasan_handler(struct pt_regs *regs, unsigned int esr)
953 {
954 bool recover = esr & KASAN_ESR_RECOVER;
955 bool write = esr & KASAN_ESR_WRITE;
956 size_t size = KASAN_ESR_SIZE(esr);
957 u64 addr = regs->regs[0];
958 u64 pc = regs->pc;
959
960 kasan_report(addr, size, write, pc);
961
962 /*
963 * The instrumentation allows to control whether we can proceed after
964 * a crash was detected. This is done by passing the -recover flag to
965 * the compiler. Disabling recovery allows to generate more compact
966 * code.
967 *
968 * Unfortunately disabling recovery doesn't work for the kernel right
969 * now. KASAN reporting is disabled in some contexts (for example when
970 * the allocator accesses slab object metadata; this is controlled by
971 * current->kasan_depth). All these accesses are detected by the tool,
972 * even though the reports for them are not printed.
973 *
974 * This is something that might be fixed at some point in the future.
975 */
976 if (!recover)
977 die("Oops - KASAN", regs, 0);
978
979 /* If thread survives, skip over the brk instruction and continue: */
980 arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE);
981 return DBG_HOOK_HANDLED;
982 }
983
984 static struct break_hook kasan_break_hook = {
985 .fn = kasan_handler,
986 .imm = KASAN_BRK_IMM,
987 .mask = KASAN_BRK_MASK,
988 };
989 #endif
990
991 /*
992 * Initial handler for AArch64 BRK exceptions
993 * This handler only used until debug_traps_init().
994 */
early_brk64(unsigned long addr,unsigned int esr,struct pt_regs * regs)995 int __init early_brk64(unsigned long addr, unsigned int esr,
996 struct pt_regs *regs)
997 {
998 #ifdef CONFIG_KASAN_SW_TAGS
999 unsigned int comment = esr & ESR_ELx_BRK64_ISS_COMMENT_MASK;
1000
1001 if ((comment & ~KASAN_BRK_MASK) == KASAN_BRK_IMM)
1002 return kasan_handler(regs, esr) != DBG_HOOK_HANDLED;
1003 #endif
1004 return bug_handler(regs, esr) != DBG_HOOK_HANDLED;
1005 }
1006
trap_init(void)1007 void __init trap_init(void)
1008 {
1009 register_kernel_break_hook(&bug_break_hook);
1010 register_kernel_break_hook(&fault_break_hook);
1011 #ifdef CONFIG_KASAN_SW_TAGS
1012 register_kernel_break_hook(&kasan_break_hook);
1013 #endif
1014 debug_traps_init();
1015 }
1016