1 /*
2 * HW_breakpoint: a unified kernel/user-space hardware breakpoint facility,
3 * using the CPU's debug registers.
4 *
5 * Copyright (C) 2012 ARM Limited
6 * Author: Will Deacon <will.deacon@arm.com>
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License version 2 as
10 * published by the Free Software Foundation.
11 *
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with this program. If not, see <http://www.gnu.org/licenses/>.
19 */
20
21 #define pr_fmt(fmt) "hw-breakpoint: " fmt
22
23 #include <linux/compat.h>
24 #include <linux/cpu_pm.h>
25 #include <linux/errno.h>
26 #include <linux/hw_breakpoint.h>
27 #include <linux/kprobes.h>
28 #include <linux/perf_event.h>
29 #include <linux/ptrace.h>
30 #include <linux/smp.h>
31
32 #include <asm/compat.h>
33 #include <asm/current.h>
34 #include <asm/debug-monitors.h>
35 #include <asm/hw_breakpoint.h>
36 #include <asm/traps.h>
37 #include <asm/cputype.h>
38 #include <asm/system_misc.h>
39 #include <asm/uaccess.h>
40
41 /* Breakpoint currently in use for each BRP. */
42 static DEFINE_PER_CPU(struct perf_event *, bp_on_reg[ARM_MAX_BRP]);
43
44 /* Watchpoint currently in use for each WRP. */
45 static DEFINE_PER_CPU(struct perf_event *, wp_on_reg[ARM_MAX_WRP]);
46
47 /* Currently stepping a per-CPU kernel breakpoint. */
48 static DEFINE_PER_CPU(int, stepping_kernel_bp);
49
50 /* Number of BRP/WRP registers on this CPU. */
51 static int core_num_brps;
52 static int core_num_wrps;
53
hw_breakpoint_slots(int type)54 int hw_breakpoint_slots(int type)
55 {
56 /*
57 * We can be called early, so don't rely on
58 * our static variables being initialised.
59 */
60 switch (type) {
61 case TYPE_INST:
62 return get_num_brps();
63 case TYPE_DATA:
64 return get_num_wrps();
65 default:
66 pr_warning("unknown slot type: %d\n", type);
67 return 0;
68 }
69 }
70
71 #define READ_WB_REG_CASE(OFF, N, REG, VAL) \
72 case (OFF + N): \
73 AARCH64_DBG_READ(N, REG, VAL); \
74 break
75
76 #define WRITE_WB_REG_CASE(OFF, N, REG, VAL) \
77 case (OFF + N): \
78 AARCH64_DBG_WRITE(N, REG, VAL); \
79 break
80
81 #define GEN_READ_WB_REG_CASES(OFF, REG, VAL) \
82 READ_WB_REG_CASE(OFF, 0, REG, VAL); \
83 READ_WB_REG_CASE(OFF, 1, REG, VAL); \
84 READ_WB_REG_CASE(OFF, 2, REG, VAL); \
85 READ_WB_REG_CASE(OFF, 3, REG, VAL); \
86 READ_WB_REG_CASE(OFF, 4, REG, VAL); \
87 READ_WB_REG_CASE(OFF, 5, REG, VAL); \
88 READ_WB_REG_CASE(OFF, 6, REG, VAL); \
89 READ_WB_REG_CASE(OFF, 7, REG, VAL); \
90 READ_WB_REG_CASE(OFF, 8, REG, VAL); \
91 READ_WB_REG_CASE(OFF, 9, REG, VAL); \
92 READ_WB_REG_CASE(OFF, 10, REG, VAL); \
93 READ_WB_REG_CASE(OFF, 11, REG, VAL); \
94 READ_WB_REG_CASE(OFF, 12, REG, VAL); \
95 READ_WB_REG_CASE(OFF, 13, REG, VAL); \
96 READ_WB_REG_CASE(OFF, 14, REG, VAL); \
97 READ_WB_REG_CASE(OFF, 15, REG, VAL)
98
99 #define GEN_WRITE_WB_REG_CASES(OFF, REG, VAL) \
100 WRITE_WB_REG_CASE(OFF, 0, REG, VAL); \
101 WRITE_WB_REG_CASE(OFF, 1, REG, VAL); \
102 WRITE_WB_REG_CASE(OFF, 2, REG, VAL); \
103 WRITE_WB_REG_CASE(OFF, 3, REG, VAL); \
104 WRITE_WB_REG_CASE(OFF, 4, REG, VAL); \
105 WRITE_WB_REG_CASE(OFF, 5, REG, VAL); \
106 WRITE_WB_REG_CASE(OFF, 6, REG, VAL); \
107 WRITE_WB_REG_CASE(OFF, 7, REG, VAL); \
108 WRITE_WB_REG_CASE(OFF, 8, REG, VAL); \
109 WRITE_WB_REG_CASE(OFF, 9, REG, VAL); \
110 WRITE_WB_REG_CASE(OFF, 10, REG, VAL); \
111 WRITE_WB_REG_CASE(OFF, 11, REG, VAL); \
112 WRITE_WB_REG_CASE(OFF, 12, REG, VAL); \
113 WRITE_WB_REG_CASE(OFF, 13, REG, VAL); \
114 WRITE_WB_REG_CASE(OFF, 14, REG, VAL); \
115 WRITE_WB_REG_CASE(OFF, 15, REG, VAL)
116
read_wb_reg(int reg,int n)117 static u64 read_wb_reg(int reg, int n)
118 {
119 u64 val = 0;
120
121 switch (reg + n) {
122 GEN_READ_WB_REG_CASES(AARCH64_DBG_REG_BVR, AARCH64_DBG_REG_NAME_BVR, val);
123 GEN_READ_WB_REG_CASES(AARCH64_DBG_REG_BCR, AARCH64_DBG_REG_NAME_BCR, val);
124 GEN_READ_WB_REG_CASES(AARCH64_DBG_REG_WVR, AARCH64_DBG_REG_NAME_WVR, val);
125 GEN_READ_WB_REG_CASES(AARCH64_DBG_REG_WCR, AARCH64_DBG_REG_NAME_WCR, val);
126 default:
127 pr_warning("attempt to read from unknown breakpoint register %d\n", n);
128 }
129
130 return val;
131 }
132 NOKPROBE_SYMBOL(read_wb_reg);
133
write_wb_reg(int reg,int n,u64 val)134 static void write_wb_reg(int reg, int n, u64 val)
135 {
136 switch (reg + n) {
137 GEN_WRITE_WB_REG_CASES(AARCH64_DBG_REG_BVR, AARCH64_DBG_REG_NAME_BVR, val);
138 GEN_WRITE_WB_REG_CASES(AARCH64_DBG_REG_BCR, AARCH64_DBG_REG_NAME_BCR, val);
139 GEN_WRITE_WB_REG_CASES(AARCH64_DBG_REG_WVR, AARCH64_DBG_REG_NAME_WVR, val);
140 GEN_WRITE_WB_REG_CASES(AARCH64_DBG_REG_WCR, AARCH64_DBG_REG_NAME_WCR, val);
141 default:
142 pr_warning("attempt to write to unknown breakpoint register %d\n", n);
143 }
144 isb();
145 }
146 NOKPROBE_SYMBOL(write_wb_reg);
147
148 /*
149 * Convert a breakpoint privilege level to the corresponding exception
150 * level.
151 */
debug_exception_level(int privilege)152 static enum dbg_active_el debug_exception_level(int privilege)
153 {
154 switch (privilege) {
155 case AARCH64_BREAKPOINT_EL0:
156 return DBG_ACTIVE_EL0;
157 case AARCH64_BREAKPOINT_EL1:
158 return DBG_ACTIVE_EL1;
159 default:
160 pr_warning("invalid breakpoint privilege level %d\n", privilege);
161 return -EINVAL;
162 }
163 }
164 NOKPROBE_SYMBOL(debug_exception_level);
165
166 enum hw_breakpoint_ops {
167 HW_BREAKPOINT_INSTALL,
168 HW_BREAKPOINT_UNINSTALL,
169 HW_BREAKPOINT_RESTORE
170 };
171
is_compat_bp(struct perf_event * bp)172 static int is_compat_bp(struct perf_event *bp)
173 {
174 struct task_struct *tsk = bp->hw.target;
175
176 /*
177 * tsk can be NULL for per-cpu (non-ptrace) breakpoints.
178 * In this case, use the native interface, since we don't have
179 * the notion of a "compat CPU" and could end up relying on
180 * deprecated behaviour if we use unaligned watchpoints in
181 * AArch64 state.
182 */
183 return tsk && is_compat_thread(task_thread_info(tsk));
184 }
185
186 /**
187 * hw_breakpoint_slot_setup - Find and setup a perf slot according to
188 * operations
189 *
190 * @slots: pointer to array of slots
191 * @max_slots: max number of slots
192 * @bp: perf_event to setup
193 * @ops: operation to be carried out on the slot
194 *
195 * Return:
196 * slot index on success
197 * -ENOSPC if no slot is available/matches
198 * -EINVAL on wrong operations parameter
199 */
hw_breakpoint_slot_setup(struct perf_event ** slots,int max_slots,struct perf_event * bp,enum hw_breakpoint_ops ops)200 static int hw_breakpoint_slot_setup(struct perf_event **slots, int max_slots,
201 struct perf_event *bp,
202 enum hw_breakpoint_ops ops)
203 {
204 int i;
205 struct perf_event **slot;
206
207 for (i = 0; i < max_slots; ++i) {
208 slot = &slots[i];
209 switch (ops) {
210 case HW_BREAKPOINT_INSTALL:
211 if (!*slot) {
212 *slot = bp;
213 return i;
214 }
215 break;
216 case HW_BREAKPOINT_UNINSTALL:
217 if (*slot == bp) {
218 *slot = NULL;
219 return i;
220 }
221 break;
222 case HW_BREAKPOINT_RESTORE:
223 if (*slot == bp)
224 return i;
225 break;
226 default:
227 pr_warn_once("Unhandled hw breakpoint ops %d\n", ops);
228 return -EINVAL;
229 }
230 }
231 return -ENOSPC;
232 }
233
hw_breakpoint_control(struct perf_event * bp,enum hw_breakpoint_ops ops)234 static int hw_breakpoint_control(struct perf_event *bp,
235 enum hw_breakpoint_ops ops)
236 {
237 struct arch_hw_breakpoint *info = counter_arch_bp(bp);
238 struct perf_event **slots;
239 struct debug_info *debug_info = ¤t->thread.debug;
240 int i, max_slots, ctrl_reg, val_reg, reg_enable;
241 enum dbg_active_el dbg_el = debug_exception_level(info->ctrl.privilege);
242 u32 ctrl;
243
244 if (info->ctrl.type == ARM_BREAKPOINT_EXECUTE) {
245 /* Breakpoint */
246 ctrl_reg = AARCH64_DBG_REG_BCR;
247 val_reg = AARCH64_DBG_REG_BVR;
248 slots = this_cpu_ptr(bp_on_reg);
249 max_slots = core_num_brps;
250 reg_enable = !debug_info->bps_disabled;
251 } else {
252 /* Watchpoint */
253 ctrl_reg = AARCH64_DBG_REG_WCR;
254 val_reg = AARCH64_DBG_REG_WVR;
255 slots = this_cpu_ptr(wp_on_reg);
256 max_slots = core_num_wrps;
257 reg_enable = !debug_info->wps_disabled;
258 }
259
260 i = hw_breakpoint_slot_setup(slots, max_slots, bp, ops);
261
262 if (WARN_ONCE(i < 0, "Can't find any breakpoint slot"))
263 return i;
264
265 switch (ops) {
266 case HW_BREAKPOINT_INSTALL:
267 /*
268 * Ensure debug monitors are enabled at the correct exception
269 * level.
270 */
271 enable_debug_monitors(dbg_el);
272 /* Fall through */
273 case HW_BREAKPOINT_RESTORE:
274 /* Setup the address register. */
275 write_wb_reg(val_reg, i, info->address);
276
277 /* Setup the control register. */
278 ctrl = encode_ctrl_reg(info->ctrl);
279 write_wb_reg(ctrl_reg, i,
280 reg_enable ? ctrl | 0x1 : ctrl & ~0x1);
281 break;
282 case HW_BREAKPOINT_UNINSTALL:
283 /* Reset the control register. */
284 write_wb_reg(ctrl_reg, i, 0);
285
286 /*
287 * Release the debug monitors for the correct exception
288 * level.
289 */
290 disable_debug_monitors(dbg_el);
291 break;
292 }
293
294 return 0;
295 }
296
297 /*
298 * Install a perf counter breakpoint.
299 */
arch_install_hw_breakpoint(struct perf_event * bp)300 int arch_install_hw_breakpoint(struct perf_event *bp)
301 {
302 return hw_breakpoint_control(bp, HW_BREAKPOINT_INSTALL);
303 }
304
arch_uninstall_hw_breakpoint(struct perf_event * bp)305 void arch_uninstall_hw_breakpoint(struct perf_event *bp)
306 {
307 hw_breakpoint_control(bp, HW_BREAKPOINT_UNINSTALL);
308 }
309
get_hbp_len(u8 hbp_len)310 static int get_hbp_len(u8 hbp_len)
311 {
312 unsigned int len_in_bytes = 0;
313
314 switch (hbp_len) {
315 case ARM_BREAKPOINT_LEN_1:
316 len_in_bytes = 1;
317 break;
318 case ARM_BREAKPOINT_LEN_2:
319 len_in_bytes = 2;
320 break;
321 case ARM_BREAKPOINT_LEN_3:
322 len_in_bytes = 3;
323 break;
324 case ARM_BREAKPOINT_LEN_4:
325 len_in_bytes = 4;
326 break;
327 case ARM_BREAKPOINT_LEN_5:
328 len_in_bytes = 5;
329 break;
330 case ARM_BREAKPOINT_LEN_6:
331 len_in_bytes = 6;
332 break;
333 case ARM_BREAKPOINT_LEN_7:
334 len_in_bytes = 7;
335 break;
336 case ARM_BREAKPOINT_LEN_8:
337 len_in_bytes = 8;
338 break;
339 }
340
341 return len_in_bytes;
342 }
343
344 /*
345 * Check whether bp virtual address is in kernel space.
346 */
arch_check_bp_in_kernelspace(struct perf_event * bp)347 int arch_check_bp_in_kernelspace(struct perf_event *bp)
348 {
349 unsigned int len;
350 unsigned long va;
351 struct arch_hw_breakpoint *info = counter_arch_bp(bp);
352
353 va = info->address;
354 len = get_hbp_len(info->ctrl.len);
355
356 return (va >= TASK_SIZE) && ((va + len - 1) >= TASK_SIZE);
357 }
358
359 /*
360 * Extract generic type and length encodings from an arch_hw_breakpoint_ctrl.
361 * Hopefully this will disappear when ptrace can bypass the conversion
362 * to generic breakpoint descriptions.
363 */
arch_bp_generic_fields(struct arch_hw_breakpoint_ctrl ctrl,int * gen_len,int * gen_type,int * offset)364 int arch_bp_generic_fields(struct arch_hw_breakpoint_ctrl ctrl,
365 int *gen_len, int *gen_type, int *offset)
366 {
367 /* Type */
368 switch (ctrl.type) {
369 case ARM_BREAKPOINT_EXECUTE:
370 *gen_type = HW_BREAKPOINT_X;
371 break;
372 case ARM_BREAKPOINT_LOAD:
373 *gen_type = HW_BREAKPOINT_R;
374 break;
375 case ARM_BREAKPOINT_STORE:
376 *gen_type = HW_BREAKPOINT_W;
377 break;
378 case ARM_BREAKPOINT_LOAD | ARM_BREAKPOINT_STORE:
379 *gen_type = HW_BREAKPOINT_RW;
380 break;
381 default:
382 return -EINVAL;
383 }
384
385 if (!ctrl.len)
386 return -EINVAL;
387 *offset = __ffs(ctrl.len);
388
389 /* Len */
390 switch (ctrl.len >> *offset) {
391 case ARM_BREAKPOINT_LEN_1:
392 *gen_len = HW_BREAKPOINT_LEN_1;
393 break;
394 case ARM_BREAKPOINT_LEN_2:
395 *gen_len = HW_BREAKPOINT_LEN_2;
396 break;
397 case ARM_BREAKPOINT_LEN_3:
398 *gen_len = HW_BREAKPOINT_LEN_3;
399 break;
400 case ARM_BREAKPOINT_LEN_4:
401 *gen_len = HW_BREAKPOINT_LEN_4;
402 break;
403 case ARM_BREAKPOINT_LEN_5:
404 *gen_len = HW_BREAKPOINT_LEN_5;
405 break;
406 case ARM_BREAKPOINT_LEN_6:
407 *gen_len = HW_BREAKPOINT_LEN_6;
408 break;
409 case ARM_BREAKPOINT_LEN_7:
410 *gen_len = HW_BREAKPOINT_LEN_7;
411 break;
412 case ARM_BREAKPOINT_LEN_8:
413 *gen_len = HW_BREAKPOINT_LEN_8;
414 break;
415 default:
416 return -EINVAL;
417 }
418
419 return 0;
420 }
421
422 /*
423 * Construct an arch_hw_breakpoint from a perf_event.
424 */
arch_build_bp_info(struct perf_event * bp)425 static int arch_build_bp_info(struct perf_event *bp)
426 {
427 struct arch_hw_breakpoint *info = counter_arch_bp(bp);
428
429 /* Type */
430 switch (bp->attr.bp_type) {
431 case HW_BREAKPOINT_X:
432 info->ctrl.type = ARM_BREAKPOINT_EXECUTE;
433 break;
434 case HW_BREAKPOINT_R:
435 info->ctrl.type = ARM_BREAKPOINT_LOAD;
436 break;
437 case HW_BREAKPOINT_W:
438 info->ctrl.type = ARM_BREAKPOINT_STORE;
439 break;
440 case HW_BREAKPOINT_RW:
441 info->ctrl.type = ARM_BREAKPOINT_LOAD | ARM_BREAKPOINT_STORE;
442 break;
443 default:
444 return -EINVAL;
445 }
446
447 /* Len */
448 switch (bp->attr.bp_len) {
449 case HW_BREAKPOINT_LEN_1:
450 info->ctrl.len = ARM_BREAKPOINT_LEN_1;
451 break;
452 case HW_BREAKPOINT_LEN_2:
453 info->ctrl.len = ARM_BREAKPOINT_LEN_2;
454 break;
455 case HW_BREAKPOINT_LEN_3:
456 info->ctrl.len = ARM_BREAKPOINT_LEN_3;
457 break;
458 case HW_BREAKPOINT_LEN_4:
459 info->ctrl.len = ARM_BREAKPOINT_LEN_4;
460 break;
461 case HW_BREAKPOINT_LEN_5:
462 info->ctrl.len = ARM_BREAKPOINT_LEN_5;
463 break;
464 case HW_BREAKPOINT_LEN_6:
465 info->ctrl.len = ARM_BREAKPOINT_LEN_6;
466 break;
467 case HW_BREAKPOINT_LEN_7:
468 info->ctrl.len = ARM_BREAKPOINT_LEN_7;
469 break;
470 case HW_BREAKPOINT_LEN_8:
471 info->ctrl.len = ARM_BREAKPOINT_LEN_8;
472 break;
473 default:
474 return -EINVAL;
475 }
476
477 /*
478 * On AArch64, we only permit breakpoints of length 4, whereas
479 * AArch32 also requires breakpoints of length 2 for Thumb.
480 * Watchpoints can be of length 1, 2, 4 or 8 bytes.
481 */
482 if (info->ctrl.type == ARM_BREAKPOINT_EXECUTE) {
483 if (is_compat_bp(bp)) {
484 if (info->ctrl.len != ARM_BREAKPOINT_LEN_2 &&
485 info->ctrl.len != ARM_BREAKPOINT_LEN_4)
486 return -EINVAL;
487 } else if (info->ctrl.len != ARM_BREAKPOINT_LEN_4) {
488 /*
489 * FIXME: Some tools (I'm looking at you perf) assume
490 * that breakpoints should be sizeof(long). This
491 * is nonsense. For now, we fix up the parameter
492 * but we should probably return -EINVAL instead.
493 */
494 info->ctrl.len = ARM_BREAKPOINT_LEN_4;
495 }
496 }
497
498 /* Address */
499 info->address = bp->attr.bp_addr;
500
501 /*
502 * Privilege
503 * Note that we disallow combined EL0/EL1 breakpoints because
504 * that would complicate the stepping code.
505 */
506 if (arch_check_bp_in_kernelspace(bp))
507 info->ctrl.privilege = AARCH64_BREAKPOINT_EL1;
508 else
509 info->ctrl.privilege = AARCH64_BREAKPOINT_EL0;
510
511 /* Enabled? */
512 info->ctrl.enabled = !bp->attr.disabled;
513
514 return 0;
515 }
516
517 /*
518 * Validate the arch-specific HW Breakpoint register settings.
519 */
arch_validate_hwbkpt_settings(struct perf_event * bp)520 int arch_validate_hwbkpt_settings(struct perf_event *bp)
521 {
522 struct arch_hw_breakpoint *info = counter_arch_bp(bp);
523 int ret;
524 u64 alignment_mask, offset;
525
526 /* Build the arch_hw_breakpoint. */
527 ret = arch_build_bp_info(bp);
528 if (ret)
529 return ret;
530
531 /*
532 * Check address alignment.
533 * We don't do any clever alignment correction for watchpoints
534 * because using 64-bit unaligned addresses is deprecated for
535 * AArch64.
536 *
537 * AArch32 tasks expect some simple alignment fixups, so emulate
538 * that here.
539 */
540 if (is_compat_bp(bp)) {
541 if (info->ctrl.len == ARM_BREAKPOINT_LEN_8)
542 alignment_mask = 0x7;
543 else
544 alignment_mask = 0x3;
545 offset = info->address & alignment_mask;
546 switch (offset) {
547 case 0:
548 /* Aligned */
549 break;
550 case 1:
551 /* Allow single byte watchpoint. */
552 if (info->ctrl.len == ARM_BREAKPOINT_LEN_1)
553 break;
554 case 2:
555 /* Allow halfword watchpoints and breakpoints. */
556 if (info->ctrl.len == ARM_BREAKPOINT_LEN_2)
557 break;
558 default:
559 return -EINVAL;
560 }
561 } else {
562 if (info->ctrl.type == ARM_BREAKPOINT_EXECUTE)
563 alignment_mask = 0x3;
564 else
565 alignment_mask = 0x7;
566 offset = info->address & alignment_mask;
567 }
568
569 info->address &= ~alignment_mask;
570 info->ctrl.len <<= offset;
571
572 /*
573 * Disallow per-task kernel breakpoints since these would
574 * complicate the stepping code.
575 */
576 if (info->ctrl.privilege == AARCH64_BREAKPOINT_EL1 && bp->hw.target)
577 return -EINVAL;
578
579 return 0;
580 }
581
582 /*
583 * Enable/disable all of the breakpoints active at the specified
584 * exception level at the register level.
585 * This is used when single-stepping after a breakpoint exception.
586 */
toggle_bp_registers(int reg,enum dbg_active_el el,int enable)587 static void toggle_bp_registers(int reg, enum dbg_active_el el, int enable)
588 {
589 int i, max_slots, privilege;
590 u32 ctrl;
591 struct perf_event **slots;
592
593 switch (reg) {
594 case AARCH64_DBG_REG_BCR:
595 slots = this_cpu_ptr(bp_on_reg);
596 max_slots = core_num_brps;
597 break;
598 case AARCH64_DBG_REG_WCR:
599 slots = this_cpu_ptr(wp_on_reg);
600 max_slots = core_num_wrps;
601 break;
602 default:
603 return;
604 }
605
606 for (i = 0; i < max_slots; ++i) {
607 if (!slots[i])
608 continue;
609
610 privilege = counter_arch_bp(slots[i])->ctrl.privilege;
611 if (debug_exception_level(privilege) != el)
612 continue;
613
614 ctrl = read_wb_reg(reg, i);
615 if (enable)
616 ctrl |= 0x1;
617 else
618 ctrl &= ~0x1;
619 write_wb_reg(reg, i, ctrl);
620 }
621 }
622 NOKPROBE_SYMBOL(toggle_bp_registers);
623
624 /*
625 * Debug exception handlers.
626 */
breakpoint_handler(unsigned long unused,unsigned int esr,struct pt_regs * regs)627 static int breakpoint_handler(unsigned long unused, unsigned int esr,
628 struct pt_regs *regs)
629 {
630 int i, step = 0, *kernel_step;
631 u32 ctrl_reg;
632 u64 addr, val;
633 struct perf_event *bp, **slots;
634 struct debug_info *debug_info;
635 struct arch_hw_breakpoint_ctrl ctrl;
636
637 slots = this_cpu_ptr(bp_on_reg);
638 addr = instruction_pointer(regs);
639 debug_info = ¤t->thread.debug;
640
641 for (i = 0; i < core_num_brps; ++i) {
642 rcu_read_lock();
643
644 bp = slots[i];
645
646 if (bp == NULL)
647 goto unlock;
648
649 /* Check if the breakpoint value matches. */
650 val = read_wb_reg(AARCH64_DBG_REG_BVR, i);
651 if (val != (addr & ~0x3))
652 goto unlock;
653
654 /* Possible match, check the byte address select to confirm. */
655 ctrl_reg = read_wb_reg(AARCH64_DBG_REG_BCR, i);
656 decode_ctrl_reg(ctrl_reg, &ctrl);
657 if (!((1 << (addr & 0x3)) & ctrl.len))
658 goto unlock;
659
660 counter_arch_bp(bp)->trigger = addr;
661 perf_bp_event(bp, regs);
662
663 /* Do we need to handle the stepping? */
664 if (is_default_overflow_handler(bp))
665 step = 1;
666 unlock:
667 rcu_read_unlock();
668 }
669
670 if (!step)
671 return 0;
672
673 if (user_mode(regs)) {
674 debug_info->bps_disabled = 1;
675 toggle_bp_registers(AARCH64_DBG_REG_BCR, DBG_ACTIVE_EL0, 0);
676
677 /* If we're already stepping a watchpoint, just return. */
678 if (debug_info->wps_disabled)
679 return 0;
680
681 if (test_thread_flag(TIF_SINGLESTEP))
682 debug_info->suspended_step = 1;
683 else
684 user_enable_single_step(current);
685 } else {
686 toggle_bp_registers(AARCH64_DBG_REG_BCR, DBG_ACTIVE_EL1, 0);
687 kernel_step = this_cpu_ptr(&stepping_kernel_bp);
688
689 if (*kernel_step != ARM_KERNEL_STEP_NONE)
690 return 0;
691
692 if (kernel_active_single_step()) {
693 *kernel_step = ARM_KERNEL_STEP_SUSPEND;
694 } else {
695 *kernel_step = ARM_KERNEL_STEP_ACTIVE;
696 kernel_enable_single_step(regs);
697 }
698 }
699
700 return 0;
701 }
702 NOKPROBE_SYMBOL(breakpoint_handler);
703
704 /*
705 * Arm64 hardware does not always report a watchpoint hit address that matches
706 * one of the watchpoints set. It can also report an address "near" the
707 * watchpoint if a single instruction access both watched and unwatched
708 * addresses. There is no straight-forward way, short of disassembling the
709 * offending instruction, to map that address back to the watchpoint. This
710 * function computes the distance of the memory access from the watchpoint as a
711 * heuristic for the likelyhood that a given access triggered the watchpoint.
712 *
713 * See Section D2.10.5 "Determining the memory location that caused a Watchpoint
714 * exception" of ARMv8 Architecture Reference Manual for details.
715 *
716 * The function returns the distance of the address from the bytes watched by
717 * the watchpoint. In case of an exact match, it returns 0.
718 */
get_distance_from_watchpoint(unsigned long addr,u64 val,struct arch_hw_breakpoint_ctrl * ctrl)719 static u64 get_distance_from_watchpoint(unsigned long addr, u64 val,
720 struct arch_hw_breakpoint_ctrl *ctrl)
721 {
722 u64 wp_low, wp_high;
723 u32 lens, lene;
724
725 lens = __ffs(ctrl->len);
726 lene = __fls(ctrl->len);
727
728 wp_low = val + lens;
729 wp_high = val + lene;
730 if (addr < wp_low)
731 return wp_low - addr;
732 else if (addr > wp_high)
733 return addr - wp_high;
734 else
735 return 0;
736 }
737
watchpoint_handler(unsigned long addr,unsigned int esr,struct pt_regs * regs)738 static int watchpoint_handler(unsigned long addr, unsigned int esr,
739 struct pt_regs *regs)
740 {
741 int i, step = 0, *kernel_step, access, closest_match = 0;
742 u64 min_dist = -1, dist;
743 u32 ctrl_reg;
744 u64 val;
745 struct perf_event *wp, **slots;
746 struct debug_info *debug_info;
747 struct arch_hw_breakpoint *info;
748 struct arch_hw_breakpoint_ctrl ctrl;
749
750 slots = this_cpu_ptr(wp_on_reg);
751 debug_info = ¤t->thread.debug;
752
753 /*
754 * Find all watchpoints that match the reported address. If no exact
755 * match is found. Attribute the hit to the closest watchpoint.
756 */
757 rcu_read_lock();
758 for (i = 0; i < core_num_wrps; ++i) {
759 wp = slots[i];
760 if (wp == NULL)
761 continue;
762
763 /*
764 * Check that the access type matches.
765 * 0 => load, otherwise => store
766 */
767 access = (esr & AARCH64_ESR_ACCESS_MASK) ? HW_BREAKPOINT_W :
768 HW_BREAKPOINT_R;
769 if (!(access & hw_breakpoint_type(wp)))
770 continue;
771
772 /* Check if the watchpoint value and byte select match. */
773 val = read_wb_reg(AARCH64_DBG_REG_WVR, i);
774 ctrl_reg = read_wb_reg(AARCH64_DBG_REG_WCR, i);
775 decode_ctrl_reg(ctrl_reg, &ctrl);
776 dist = get_distance_from_watchpoint(addr, val, &ctrl);
777 if (dist < min_dist) {
778 min_dist = dist;
779 closest_match = i;
780 }
781 /* Is this an exact match? */
782 if (dist != 0)
783 continue;
784
785 info = counter_arch_bp(wp);
786 info->trigger = addr;
787 perf_bp_event(wp, regs);
788
789 /* Do we need to handle the stepping? */
790 if (is_default_overflow_handler(wp))
791 step = 1;
792 }
793 if (min_dist > 0 && min_dist != -1) {
794 /* No exact match found. */
795 wp = slots[closest_match];
796 info = counter_arch_bp(wp);
797 info->trigger = addr;
798 perf_bp_event(wp, regs);
799
800 /* Do we need to handle the stepping? */
801 if (is_default_overflow_handler(wp))
802 step = 1;
803 }
804 rcu_read_unlock();
805
806 if (!step)
807 return 0;
808
809 /*
810 * We always disable EL0 watchpoints because the kernel can
811 * cause these to fire via an unprivileged access.
812 */
813 toggle_bp_registers(AARCH64_DBG_REG_WCR, DBG_ACTIVE_EL0, 0);
814
815 if (user_mode(regs)) {
816 debug_info->wps_disabled = 1;
817
818 /* If we're already stepping a breakpoint, just return. */
819 if (debug_info->bps_disabled)
820 return 0;
821
822 if (test_thread_flag(TIF_SINGLESTEP))
823 debug_info->suspended_step = 1;
824 else
825 user_enable_single_step(current);
826 } else {
827 toggle_bp_registers(AARCH64_DBG_REG_WCR, DBG_ACTIVE_EL1, 0);
828 kernel_step = this_cpu_ptr(&stepping_kernel_bp);
829
830 if (*kernel_step != ARM_KERNEL_STEP_NONE)
831 return 0;
832
833 if (kernel_active_single_step()) {
834 *kernel_step = ARM_KERNEL_STEP_SUSPEND;
835 } else {
836 *kernel_step = ARM_KERNEL_STEP_ACTIVE;
837 kernel_enable_single_step(regs);
838 }
839 }
840
841 return 0;
842 }
843 NOKPROBE_SYMBOL(watchpoint_handler);
844
845 /*
846 * Handle single-step exception.
847 */
reinstall_suspended_bps(struct pt_regs * regs)848 int reinstall_suspended_bps(struct pt_regs *regs)
849 {
850 struct debug_info *debug_info = ¤t->thread.debug;
851 int handled_exception = 0, *kernel_step;
852
853 kernel_step = this_cpu_ptr(&stepping_kernel_bp);
854
855 /*
856 * Called from single-step exception handler.
857 * Return 0 if execution can resume, 1 if a SIGTRAP should be
858 * reported.
859 */
860 if (user_mode(regs)) {
861 if (debug_info->bps_disabled) {
862 debug_info->bps_disabled = 0;
863 toggle_bp_registers(AARCH64_DBG_REG_BCR, DBG_ACTIVE_EL0, 1);
864 handled_exception = 1;
865 }
866
867 if (debug_info->wps_disabled) {
868 debug_info->wps_disabled = 0;
869 toggle_bp_registers(AARCH64_DBG_REG_WCR, DBG_ACTIVE_EL0, 1);
870 handled_exception = 1;
871 }
872
873 if (handled_exception) {
874 if (debug_info->suspended_step) {
875 debug_info->suspended_step = 0;
876 /* Allow exception handling to fall-through. */
877 handled_exception = 0;
878 } else {
879 user_disable_single_step(current);
880 }
881 }
882 } else if (*kernel_step != ARM_KERNEL_STEP_NONE) {
883 toggle_bp_registers(AARCH64_DBG_REG_BCR, DBG_ACTIVE_EL1, 1);
884 toggle_bp_registers(AARCH64_DBG_REG_WCR, DBG_ACTIVE_EL1, 1);
885
886 if (!debug_info->wps_disabled)
887 toggle_bp_registers(AARCH64_DBG_REG_WCR, DBG_ACTIVE_EL0, 1);
888
889 if (*kernel_step != ARM_KERNEL_STEP_SUSPEND) {
890 kernel_disable_single_step();
891 handled_exception = 1;
892 } else {
893 handled_exception = 0;
894 }
895
896 *kernel_step = ARM_KERNEL_STEP_NONE;
897 }
898
899 return !handled_exception;
900 }
901 NOKPROBE_SYMBOL(reinstall_suspended_bps);
902
903 /*
904 * Context-switcher for restoring suspended breakpoints.
905 */
hw_breakpoint_thread_switch(struct task_struct * next)906 void hw_breakpoint_thread_switch(struct task_struct *next)
907 {
908 /*
909 * current next
910 * disabled: 0 0 => The usual case, NOTIFY_DONE
911 * 0 1 => Disable the registers
912 * 1 0 => Enable the registers
913 * 1 1 => NOTIFY_DONE. per-task bps will
914 * get taken care of by perf.
915 */
916
917 struct debug_info *current_debug_info, *next_debug_info;
918
919 current_debug_info = ¤t->thread.debug;
920 next_debug_info = &next->thread.debug;
921
922 /* Update breakpoints. */
923 if (current_debug_info->bps_disabled != next_debug_info->bps_disabled)
924 toggle_bp_registers(AARCH64_DBG_REG_BCR,
925 DBG_ACTIVE_EL0,
926 !next_debug_info->bps_disabled);
927
928 /* Update watchpoints. */
929 if (current_debug_info->wps_disabled != next_debug_info->wps_disabled)
930 toggle_bp_registers(AARCH64_DBG_REG_WCR,
931 DBG_ACTIVE_EL0,
932 !next_debug_info->wps_disabled);
933 }
934
935 /*
936 * CPU initialisation.
937 */
hw_breakpoint_reset(unsigned int cpu)938 static int hw_breakpoint_reset(unsigned int cpu)
939 {
940 int i;
941 struct perf_event **slots;
942 /*
943 * When a CPU goes through cold-boot, it does not have any installed
944 * slot, so it is safe to share the same function for restoring and
945 * resetting breakpoints; when a CPU is hotplugged in, it goes
946 * through the slots, which are all empty, hence it just resets control
947 * and value for debug registers.
948 * When this function is triggered on warm-boot through a CPU PM
949 * notifier some slots might be initialized; if so they are
950 * reprogrammed according to the debug slots content.
951 */
952 for (slots = this_cpu_ptr(bp_on_reg), i = 0; i < core_num_brps; ++i) {
953 if (slots[i]) {
954 hw_breakpoint_control(slots[i], HW_BREAKPOINT_RESTORE);
955 } else {
956 write_wb_reg(AARCH64_DBG_REG_BCR, i, 0UL);
957 write_wb_reg(AARCH64_DBG_REG_BVR, i, 0UL);
958 }
959 }
960
961 for (slots = this_cpu_ptr(wp_on_reg), i = 0; i < core_num_wrps; ++i) {
962 if (slots[i]) {
963 hw_breakpoint_control(slots[i], HW_BREAKPOINT_RESTORE);
964 } else {
965 write_wb_reg(AARCH64_DBG_REG_WCR, i, 0UL);
966 write_wb_reg(AARCH64_DBG_REG_WVR, i, 0UL);
967 }
968 }
969
970 return 0;
971 }
972
973 #ifdef CONFIG_CPU_PM
974 extern void cpu_suspend_set_dbg_restorer(int (*hw_bp_restore)(unsigned int));
975 #else
cpu_suspend_set_dbg_restorer(int (* hw_bp_restore)(unsigned int))976 static inline void cpu_suspend_set_dbg_restorer(int (*hw_bp_restore)(unsigned int))
977 {
978 }
979 #endif
980
981 /*
982 * One-time initialisation.
983 */
arch_hw_breakpoint_init(void)984 static int __init arch_hw_breakpoint_init(void)
985 {
986 int ret;
987
988 core_num_brps = get_num_brps();
989 core_num_wrps = get_num_wrps();
990
991 pr_info("found %d breakpoint and %d watchpoint registers.\n",
992 core_num_brps, core_num_wrps);
993
994 /* Register debug fault handlers. */
995 hook_debug_fault_code(DBG_ESR_EVT_HWBP, breakpoint_handler, SIGTRAP,
996 TRAP_HWBKPT, "hw-breakpoint handler");
997 hook_debug_fault_code(DBG_ESR_EVT_HWWP, watchpoint_handler, SIGTRAP,
998 TRAP_HWBKPT, "hw-watchpoint handler");
999
1000 /*
1001 * Reset the breakpoint resources. We assume that a halting
1002 * debugger will leave the world in a nice state for us.
1003 */
1004 ret = cpuhp_setup_state(CPUHP_AP_PERF_ARM_HW_BREAKPOINT_STARTING,
1005 "CPUHP_AP_PERF_ARM_HW_BREAKPOINT_STARTING",
1006 hw_breakpoint_reset, NULL);
1007 if (ret)
1008 pr_err("failed to register CPU hotplug notifier: %d\n", ret);
1009
1010 /* Register cpu_suspend hw breakpoint restore hook */
1011 cpu_suspend_set_dbg_restorer(hw_breakpoint_reset);
1012
1013 return ret;
1014 }
1015 arch_initcall(arch_hw_breakpoint_init);
1016
hw_breakpoint_pmu_read(struct perf_event * bp)1017 void hw_breakpoint_pmu_read(struct perf_event *bp)
1018 {
1019 }
1020
1021 /*
1022 * Dummy function to register with die_notifier.
1023 */
hw_breakpoint_exceptions_notify(struct notifier_block * unused,unsigned long val,void * data)1024 int hw_breakpoint_exceptions_notify(struct notifier_block *unused,
1025 unsigned long val, void *data)
1026 {
1027 return NOTIFY_DONE;
1028 }
1029