1 /*
2 * HW_breakpoint: a unified kernel/user-space hardware breakpoint facility,
3 * using the CPU's debug registers.
4 *
5 * Copyright (C) 2012 ARM Limited
6 * Author: Will Deacon <will.deacon@arm.com>
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License version 2 as
10 * published by the Free Software Foundation.
11 *
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with this program. If not, see <http://www.gnu.org/licenses/>.
19 */
20
21 #define pr_fmt(fmt) "hw-breakpoint: " fmt
22
23 #include <linux/compat.h>
24 #include <linux/cpu_pm.h>
25 #include <linux/errno.h>
26 #include <linux/hw_breakpoint.h>
27 #include <linux/perf_event.h>
28 #include <linux/ptrace.h>
29 #include <linux/smp.h>
30
31 #include <asm/current.h>
32 #include <asm/debug-monitors.h>
33 #include <asm/hw_breakpoint.h>
34 #include <asm/kdebug.h>
35 #include <asm/traps.h>
36 #include <asm/cpufeature.h>
37 #include <asm/cputype.h>
38 #include <asm/sysreg.h>
39 #include <asm/system_misc.h>
40 #include <asm/uaccess.h>
41
42 /* Breakpoint currently in use for each BRP. */
43 static DEFINE_PER_CPU(struct perf_event *, bp_on_reg[ARM_MAX_BRP]);
44
45 /* Watchpoint currently in use for each WRP. */
46 static DEFINE_PER_CPU(struct perf_event *, wp_on_reg[ARM_MAX_WRP]);
47
48 /* Currently stepping a per-CPU kernel breakpoint. */
49 static DEFINE_PER_CPU(int, stepping_kernel_bp);
50
51 /* Number of BRP/WRP registers on this CPU. */
52 static int core_num_brps;
53 static int core_num_wrps;
54
55 /* Determine number of BRP registers available. */
get_num_brps(void)56 static int get_num_brps(void)
57 {
58 return 1 +
59 cpuid_feature_extract_field(read_system_reg(SYS_ID_AA64DFR0_EL1),
60 ID_AA64DFR0_BRPS_SHIFT);
61 }
62
63 /* Determine number of WRP registers available. */
get_num_wrps(void)64 static int get_num_wrps(void)
65 {
66 return 1 +
67 cpuid_feature_extract_field(read_system_reg(SYS_ID_AA64DFR0_EL1),
68 ID_AA64DFR0_WRPS_SHIFT);
69 }
70
hw_breakpoint_slots(int type)71 int hw_breakpoint_slots(int type)
72 {
73 /*
74 * We can be called early, so don't rely on
75 * our static variables being initialised.
76 */
77 switch (type) {
78 case TYPE_INST:
79 return get_num_brps();
80 case TYPE_DATA:
81 return get_num_wrps();
82 default:
83 pr_warning("unknown slot type: %d\n", type);
84 return 0;
85 }
86 }
87
88 #define READ_WB_REG_CASE(OFF, N, REG, VAL) \
89 case (OFF + N): \
90 AARCH64_DBG_READ(N, REG, VAL); \
91 break
92
93 #define WRITE_WB_REG_CASE(OFF, N, REG, VAL) \
94 case (OFF + N): \
95 AARCH64_DBG_WRITE(N, REG, VAL); \
96 break
97
98 #define GEN_READ_WB_REG_CASES(OFF, REG, VAL) \
99 READ_WB_REG_CASE(OFF, 0, REG, VAL); \
100 READ_WB_REG_CASE(OFF, 1, REG, VAL); \
101 READ_WB_REG_CASE(OFF, 2, REG, VAL); \
102 READ_WB_REG_CASE(OFF, 3, REG, VAL); \
103 READ_WB_REG_CASE(OFF, 4, REG, VAL); \
104 READ_WB_REG_CASE(OFF, 5, REG, VAL); \
105 READ_WB_REG_CASE(OFF, 6, REG, VAL); \
106 READ_WB_REG_CASE(OFF, 7, REG, VAL); \
107 READ_WB_REG_CASE(OFF, 8, REG, VAL); \
108 READ_WB_REG_CASE(OFF, 9, REG, VAL); \
109 READ_WB_REG_CASE(OFF, 10, REG, VAL); \
110 READ_WB_REG_CASE(OFF, 11, REG, VAL); \
111 READ_WB_REG_CASE(OFF, 12, REG, VAL); \
112 READ_WB_REG_CASE(OFF, 13, REG, VAL); \
113 READ_WB_REG_CASE(OFF, 14, REG, VAL); \
114 READ_WB_REG_CASE(OFF, 15, REG, VAL)
115
116 #define GEN_WRITE_WB_REG_CASES(OFF, REG, VAL) \
117 WRITE_WB_REG_CASE(OFF, 0, REG, VAL); \
118 WRITE_WB_REG_CASE(OFF, 1, REG, VAL); \
119 WRITE_WB_REG_CASE(OFF, 2, REG, VAL); \
120 WRITE_WB_REG_CASE(OFF, 3, REG, VAL); \
121 WRITE_WB_REG_CASE(OFF, 4, REG, VAL); \
122 WRITE_WB_REG_CASE(OFF, 5, REG, VAL); \
123 WRITE_WB_REG_CASE(OFF, 6, REG, VAL); \
124 WRITE_WB_REG_CASE(OFF, 7, REG, VAL); \
125 WRITE_WB_REG_CASE(OFF, 8, REG, VAL); \
126 WRITE_WB_REG_CASE(OFF, 9, REG, VAL); \
127 WRITE_WB_REG_CASE(OFF, 10, REG, VAL); \
128 WRITE_WB_REG_CASE(OFF, 11, REG, VAL); \
129 WRITE_WB_REG_CASE(OFF, 12, REG, VAL); \
130 WRITE_WB_REG_CASE(OFF, 13, REG, VAL); \
131 WRITE_WB_REG_CASE(OFF, 14, REG, VAL); \
132 WRITE_WB_REG_CASE(OFF, 15, REG, VAL)
133
read_wb_reg(int reg,int n)134 static u64 read_wb_reg(int reg, int n)
135 {
136 u64 val = 0;
137
138 switch (reg + n) {
139 GEN_READ_WB_REG_CASES(AARCH64_DBG_REG_BVR, AARCH64_DBG_REG_NAME_BVR, val);
140 GEN_READ_WB_REG_CASES(AARCH64_DBG_REG_BCR, AARCH64_DBG_REG_NAME_BCR, val);
141 GEN_READ_WB_REG_CASES(AARCH64_DBG_REG_WVR, AARCH64_DBG_REG_NAME_WVR, val);
142 GEN_READ_WB_REG_CASES(AARCH64_DBG_REG_WCR, AARCH64_DBG_REG_NAME_WCR, val);
143 default:
144 pr_warning("attempt to read from unknown breakpoint register %d\n", n);
145 }
146
147 return val;
148 }
149
write_wb_reg(int reg,int n,u64 val)150 static void write_wb_reg(int reg, int n, u64 val)
151 {
152 switch (reg + n) {
153 GEN_WRITE_WB_REG_CASES(AARCH64_DBG_REG_BVR, AARCH64_DBG_REG_NAME_BVR, val);
154 GEN_WRITE_WB_REG_CASES(AARCH64_DBG_REG_BCR, AARCH64_DBG_REG_NAME_BCR, val);
155 GEN_WRITE_WB_REG_CASES(AARCH64_DBG_REG_WVR, AARCH64_DBG_REG_NAME_WVR, val);
156 GEN_WRITE_WB_REG_CASES(AARCH64_DBG_REG_WCR, AARCH64_DBG_REG_NAME_WCR, val);
157 default:
158 pr_warning("attempt to write to unknown breakpoint register %d\n", n);
159 }
160 isb();
161 }
162
163 /*
164 * Convert a breakpoint privilege level to the corresponding exception
165 * level.
166 */
debug_exception_level(int privilege)167 static enum debug_el debug_exception_level(int privilege)
168 {
169 switch (privilege) {
170 case AARCH64_BREAKPOINT_EL0:
171 return DBG_ACTIVE_EL0;
172 case AARCH64_BREAKPOINT_EL1:
173 return DBG_ACTIVE_EL1;
174 default:
175 pr_warning("invalid breakpoint privilege level %d\n", privilege);
176 return -EINVAL;
177 }
178 }
179
180 enum hw_breakpoint_ops {
181 HW_BREAKPOINT_INSTALL,
182 HW_BREAKPOINT_UNINSTALL,
183 HW_BREAKPOINT_RESTORE
184 };
185
186 /**
187 * hw_breakpoint_slot_setup - Find and setup a perf slot according to
188 * operations
189 *
190 * @slots: pointer to array of slots
191 * @max_slots: max number of slots
192 * @bp: perf_event to setup
193 * @ops: operation to be carried out on the slot
194 *
195 * Return:
196 * slot index on success
197 * -ENOSPC if no slot is available/matches
198 * -EINVAL on wrong operations parameter
199 */
hw_breakpoint_slot_setup(struct perf_event ** slots,int max_slots,struct perf_event * bp,enum hw_breakpoint_ops ops)200 static int hw_breakpoint_slot_setup(struct perf_event **slots, int max_slots,
201 struct perf_event *bp,
202 enum hw_breakpoint_ops ops)
203 {
204 int i;
205 struct perf_event **slot;
206
207 for (i = 0; i < max_slots; ++i) {
208 slot = &slots[i];
209 switch (ops) {
210 case HW_BREAKPOINT_INSTALL:
211 if (!*slot) {
212 *slot = bp;
213 return i;
214 }
215 break;
216 case HW_BREAKPOINT_UNINSTALL:
217 if (*slot == bp) {
218 *slot = NULL;
219 return i;
220 }
221 break;
222 case HW_BREAKPOINT_RESTORE:
223 if (*slot == bp)
224 return i;
225 break;
226 default:
227 pr_warn_once("Unhandled hw breakpoint ops %d\n", ops);
228 return -EINVAL;
229 }
230 }
231 return -ENOSPC;
232 }
233
hw_breakpoint_control(struct perf_event * bp,enum hw_breakpoint_ops ops)234 static int hw_breakpoint_control(struct perf_event *bp,
235 enum hw_breakpoint_ops ops)
236 {
237 struct arch_hw_breakpoint *info = counter_arch_bp(bp);
238 struct perf_event **slots;
239 struct debug_info *debug_info = ¤t->thread.debug;
240 int i, max_slots, ctrl_reg, val_reg, reg_enable;
241 enum debug_el dbg_el = debug_exception_level(info->ctrl.privilege);
242 u32 ctrl;
243
244 if (info->ctrl.type == ARM_BREAKPOINT_EXECUTE) {
245 /* Breakpoint */
246 ctrl_reg = AARCH64_DBG_REG_BCR;
247 val_reg = AARCH64_DBG_REG_BVR;
248 slots = this_cpu_ptr(bp_on_reg);
249 max_slots = core_num_brps;
250 reg_enable = !debug_info->bps_disabled;
251 } else {
252 /* Watchpoint */
253 ctrl_reg = AARCH64_DBG_REG_WCR;
254 val_reg = AARCH64_DBG_REG_WVR;
255 slots = this_cpu_ptr(wp_on_reg);
256 max_slots = core_num_wrps;
257 reg_enable = !debug_info->wps_disabled;
258 }
259
260 i = hw_breakpoint_slot_setup(slots, max_slots, bp, ops);
261
262 if (WARN_ONCE(i < 0, "Can't find any breakpoint slot"))
263 return i;
264
265 switch (ops) {
266 case HW_BREAKPOINT_INSTALL:
267 /*
268 * Ensure debug monitors are enabled at the correct exception
269 * level.
270 */
271 enable_debug_monitors(dbg_el);
272 /* Fall through */
273 case HW_BREAKPOINT_RESTORE:
274 /* Setup the address register. */
275 write_wb_reg(val_reg, i, info->address);
276
277 /* Setup the control register. */
278 ctrl = encode_ctrl_reg(info->ctrl);
279 write_wb_reg(ctrl_reg, i,
280 reg_enable ? ctrl | 0x1 : ctrl & ~0x1);
281 break;
282 case HW_BREAKPOINT_UNINSTALL:
283 /* Reset the control register. */
284 write_wb_reg(ctrl_reg, i, 0);
285
286 /*
287 * Release the debug monitors for the correct exception
288 * level.
289 */
290 disable_debug_monitors(dbg_el);
291 break;
292 }
293
294 return 0;
295 }
296
297 /*
298 * Install a perf counter breakpoint.
299 */
arch_install_hw_breakpoint(struct perf_event * bp)300 int arch_install_hw_breakpoint(struct perf_event *bp)
301 {
302 return hw_breakpoint_control(bp, HW_BREAKPOINT_INSTALL);
303 }
304
arch_uninstall_hw_breakpoint(struct perf_event * bp)305 void arch_uninstall_hw_breakpoint(struct perf_event *bp)
306 {
307 hw_breakpoint_control(bp, HW_BREAKPOINT_UNINSTALL);
308 }
309
get_hbp_len(u8 hbp_len)310 static int get_hbp_len(u8 hbp_len)
311 {
312 unsigned int len_in_bytes = 0;
313
314 switch (hbp_len) {
315 case ARM_BREAKPOINT_LEN_1:
316 len_in_bytes = 1;
317 break;
318 case ARM_BREAKPOINT_LEN_2:
319 len_in_bytes = 2;
320 break;
321 case ARM_BREAKPOINT_LEN_3:
322 len_in_bytes = 3;
323 break;
324 case ARM_BREAKPOINT_LEN_4:
325 len_in_bytes = 4;
326 break;
327 case ARM_BREAKPOINT_LEN_5:
328 len_in_bytes = 5;
329 break;
330 case ARM_BREAKPOINT_LEN_6:
331 len_in_bytes = 6;
332 break;
333 case ARM_BREAKPOINT_LEN_7:
334 len_in_bytes = 7;
335 break;
336 case ARM_BREAKPOINT_LEN_8:
337 len_in_bytes = 8;
338 break;
339 }
340
341 return len_in_bytes;
342 }
343
344 /*
345 * Check whether bp virtual address is in kernel space.
346 */
arch_check_bp_in_kernelspace(struct perf_event * bp)347 int arch_check_bp_in_kernelspace(struct perf_event *bp)
348 {
349 unsigned int len;
350 unsigned long va;
351 struct arch_hw_breakpoint *info = counter_arch_bp(bp);
352
353 va = info->address;
354 len = get_hbp_len(info->ctrl.len);
355
356 return (va >= TASK_SIZE) && ((va + len - 1) >= TASK_SIZE);
357 }
358
359 /*
360 * Extract generic type and length encodings from an arch_hw_breakpoint_ctrl.
361 * Hopefully this will disappear when ptrace can bypass the conversion
362 * to generic breakpoint descriptions.
363 */
arch_bp_generic_fields(struct arch_hw_breakpoint_ctrl ctrl,int * gen_len,int * gen_type,int * offset)364 int arch_bp_generic_fields(struct arch_hw_breakpoint_ctrl ctrl,
365 int *gen_len, int *gen_type, int *offset)
366 {
367 /* Type */
368 switch (ctrl.type) {
369 case ARM_BREAKPOINT_EXECUTE:
370 *gen_type = HW_BREAKPOINT_X;
371 break;
372 case ARM_BREAKPOINT_LOAD:
373 *gen_type = HW_BREAKPOINT_R;
374 break;
375 case ARM_BREAKPOINT_STORE:
376 *gen_type = HW_BREAKPOINT_W;
377 break;
378 case ARM_BREAKPOINT_LOAD | ARM_BREAKPOINT_STORE:
379 *gen_type = HW_BREAKPOINT_RW;
380 break;
381 default:
382 return -EINVAL;
383 }
384
385 if (!ctrl.len)
386 return -EINVAL;
387 *offset = __ffs(ctrl.len);
388
389 /* Len */
390 switch (ctrl.len >> *offset) {
391 case ARM_BREAKPOINT_LEN_1:
392 *gen_len = HW_BREAKPOINT_LEN_1;
393 break;
394 case ARM_BREAKPOINT_LEN_2:
395 *gen_len = HW_BREAKPOINT_LEN_2;
396 break;
397 case ARM_BREAKPOINT_LEN_3:
398 *gen_len = HW_BREAKPOINT_LEN_3;
399 break;
400 case ARM_BREAKPOINT_LEN_4:
401 *gen_len = HW_BREAKPOINT_LEN_4;
402 break;
403 case ARM_BREAKPOINT_LEN_5:
404 *gen_len = HW_BREAKPOINT_LEN_5;
405 break;
406 case ARM_BREAKPOINT_LEN_6:
407 *gen_len = HW_BREAKPOINT_LEN_6;
408 break;
409 case ARM_BREAKPOINT_LEN_7:
410 *gen_len = HW_BREAKPOINT_LEN_7;
411 break;
412 case ARM_BREAKPOINT_LEN_8:
413 *gen_len = HW_BREAKPOINT_LEN_8;
414 break;
415 default:
416 return -EINVAL;
417 }
418
419 return 0;
420 }
421
422 /*
423 * Construct an arch_hw_breakpoint from a perf_event.
424 */
arch_build_bp_info(struct perf_event * bp)425 static int arch_build_bp_info(struct perf_event *bp)
426 {
427 struct arch_hw_breakpoint *info = counter_arch_bp(bp);
428
429 /* Type */
430 switch (bp->attr.bp_type) {
431 case HW_BREAKPOINT_X:
432 info->ctrl.type = ARM_BREAKPOINT_EXECUTE;
433 break;
434 case HW_BREAKPOINT_R:
435 info->ctrl.type = ARM_BREAKPOINT_LOAD;
436 break;
437 case HW_BREAKPOINT_W:
438 info->ctrl.type = ARM_BREAKPOINT_STORE;
439 break;
440 case HW_BREAKPOINT_RW:
441 info->ctrl.type = ARM_BREAKPOINT_LOAD | ARM_BREAKPOINT_STORE;
442 break;
443 default:
444 return -EINVAL;
445 }
446
447 /* Len */
448 switch (bp->attr.bp_len) {
449 case HW_BREAKPOINT_LEN_1:
450 info->ctrl.len = ARM_BREAKPOINT_LEN_1;
451 break;
452 case HW_BREAKPOINT_LEN_2:
453 info->ctrl.len = ARM_BREAKPOINT_LEN_2;
454 break;
455 case HW_BREAKPOINT_LEN_3:
456 info->ctrl.len = ARM_BREAKPOINT_LEN_3;
457 break;
458 case HW_BREAKPOINT_LEN_4:
459 info->ctrl.len = ARM_BREAKPOINT_LEN_4;
460 break;
461 case HW_BREAKPOINT_LEN_5:
462 info->ctrl.len = ARM_BREAKPOINT_LEN_5;
463 break;
464 case HW_BREAKPOINT_LEN_6:
465 info->ctrl.len = ARM_BREAKPOINT_LEN_6;
466 break;
467 case HW_BREAKPOINT_LEN_7:
468 info->ctrl.len = ARM_BREAKPOINT_LEN_7;
469 break;
470 case HW_BREAKPOINT_LEN_8:
471 info->ctrl.len = ARM_BREAKPOINT_LEN_8;
472 break;
473 default:
474 return -EINVAL;
475 }
476
477 /*
478 * On AArch64, we only permit breakpoints of length 4, whereas
479 * AArch32 also requires breakpoints of length 2 for Thumb.
480 * Watchpoints can be of length 1, 2, 4 or 8 bytes.
481 */
482 if (info->ctrl.type == ARM_BREAKPOINT_EXECUTE) {
483 if (is_compat_task()) {
484 if (info->ctrl.len != ARM_BREAKPOINT_LEN_2 &&
485 info->ctrl.len != ARM_BREAKPOINT_LEN_4)
486 return -EINVAL;
487 } else if (info->ctrl.len != ARM_BREAKPOINT_LEN_4) {
488 /*
489 * FIXME: Some tools (I'm looking at you perf) assume
490 * that breakpoints should be sizeof(long). This
491 * is nonsense. For now, we fix up the parameter
492 * but we should probably return -EINVAL instead.
493 */
494 info->ctrl.len = ARM_BREAKPOINT_LEN_4;
495 }
496 }
497
498 /* Address */
499 info->address = bp->attr.bp_addr;
500
501 /*
502 * Privilege
503 * Note that we disallow combined EL0/EL1 breakpoints because
504 * that would complicate the stepping code.
505 */
506 if (arch_check_bp_in_kernelspace(bp))
507 info->ctrl.privilege = AARCH64_BREAKPOINT_EL1;
508 else
509 info->ctrl.privilege = AARCH64_BREAKPOINT_EL0;
510
511 /* Enabled? */
512 info->ctrl.enabled = !bp->attr.disabled;
513
514 return 0;
515 }
516
517 /*
518 * Validate the arch-specific HW Breakpoint register settings.
519 */
arch_validate_hwbkpt_settings(struct perf_event * bp)520 int arch_validate_hwbkpt_settings(struct perf_event *bp)
521 {
522 struct arch_hw_breakpoint *info = counter_arch_bp(bp);
523 int ret;
524 u64 alignment_mask, offset;
525
526 /* Build the arch_hw_breakpoint. */
527 ret = arch_build_bp_info(bp);
528 if (ret)
529 return ret;
530
531 /*
532 * Check address alignment.
533 * We don't do any clever alignment correction for watchpoints
534 * because using 64-bit unaligned addresses is deprecated for
535 * AArch64.
536 *
537 * AArch32 tasks expect some simple alignment fixups, so emulate
538 * that here.
539 */
540 if (is_compat_task()) {
541 if (info->ctrl.len == ARM_BREAKPOINT_LEN_8)
542 alignment_mask = 0x7;
543 else
544 alignment_mask = 0x3;
545 offset = info->address & alignment_mask;
546 switch (offset) {
547 case 0:
548 /* Aligned */
549 break;
550 case 1:
551 /* Allow single byte watchpoint. */
552 if (info->ctrl.len == ARM_BREAKPOINT_LEN_1)
553 break;
554 case 2:
555 /* Allow halfword watchpoints and breakpoints. */
556 if (info->ctrl.len == ARM_BREAKPOINT_LEN_2)
557 break;
558 default:
559 return -EINVAL;
560 }
561 } else {
562 if (info->ctrl.type == ARM_BREAKPOINT_EXECUTE)
563 alignment_mask = 0x3;
564 else
565 alignment_mask = 0x7;
566 offset = info->address & alignment_mask;
567 }
568
569 info->address &= ~alignment_mask;
570 info->ctrl.len <<= offset;
571
572 /*
573 * Disallow per-task kernel breakpoints since these would
574 * complicate the stepping code.
575 */
576 if (info->ctrl.privilege == AARCH64_BREAKPOINT_EL1 && bp->hw.bp_target)
577 return -EINVAL;
578
579 return 0;
580 }
581
582 /*
583 * Enable/disable all of the breakpoints active at the specified
584 * exception level at the register level.
585 * This is used when single-stepping after a breakpoint exception.
586 */
toggle_bp_registers(int reg,enum debug_el el,int enable)587 static void toggle_bp_registers(int reg, enum debug_el el, int enable)
588 {
589 int i, max_slots, privilege;
590 u32 ctrl;
591 struct perf_event **slots;
592
593 switch (reg) {
594 case AARCH64_DBG_REG_BCR:
595 slots = this_cpu_ptr(bp_on_reg);
596 max_slots = core_num_brps;
597 break;
598 case AARCH64_DBG_REG_WCR:
599 slots = this_cpu_ptr(wp_on_reg);
600 max_slots = core_num_wrps;
601 break;
602 default:
603 return;
604 }
605
606 for (i = 0; i < max_slots; ++i) {
607 if (!slots[i])
608 continue;
609
610 privilege = counter_arch_bp(slots[i])->ctrl.privilege;
611 if (debug_exception_level(privilege) != el)
612 continue;
613
614 ctrl = read_wb_reg(reg, i);
615 if (enable)
616 ctrl |= 0x1;
617 else
618 ctrl &= ~0x1;
619 write_wb_reg(reg, i, ctrl);
620 }
621 }
622
623 /*
624 * Debug exception handlers.
625 */
breakpoint_handler(unsigned long unused,unsigned int esr,struct pt_regs * regs)626 static int breakpoint_handler(unsigned long unused, unsigned int esr,
627 struct pt_regs *regs)
628 {
629 int i, step = 0, *kernel_step;
630 u32 ctrl_reg;
631 u64 addr, val;
632 struct perf_event *bp, **slots;
633 struct debug_info *debug_info;
634 struct arch_hw_breakpoint_ctrl ctrl;
635
636 slots = this_cpu_ptr(bp_on_reg);
637 addr = instruction_pointer(regs);
638 debug_info = ¤t->thread.debug;
639
640 for (i = 0; i < core_num_brps; ++i) {
641 rcu_read_lock();
642
643 bp = slots[i];
644
645 if (bp == NULL)
646 goto unlock;
647
648 /* Check if the breakpoint value matches. */
649 val = read_wb_reg(AARCH64_DBG_REG_BVR, i);
650 if (val != (addr & ~0x3))
651 goto unlock;
652
653 /* Possible match, check the byte address select to confirm. */
654 ctrl_reg = read_wb_reg(AARCH64_DBG_REG_BCR, i);
655 decode_ctrl_reg(ctrl_reg, &ctrl);
656 if (!((1 << (addr & 0x3)) & ctrl.len))
657 goto unlock;
658
659 counter_arch_bp(bp)->trigger = addr;
660 perf_bp_event(bp, regs);
661
662 /* Do we need to handle the stepping? */
663 if (!bp->overflow_handler)
664 step = 1;
665 unlock:
666 rcu_read_unlock();
667 }
668
669 if (!step)
670 return 0;
671
672 if (user_mode(regs)) {
673 debug_info->bps_disabled = 1;
674 toggle_bp_registers(AARCH64_DBG_REG_BCR, DBG_ACTIVE_EL0, 0);
675
676 /* If we're already stepping a watchpoint, just return. */
677 if (debug_info->wps_disabled)
678 return 0;
679
680 if (test_thread_flag(TIF_SINGLESTEP))
681 debug_info->suspended_step = 1;
682 else
683 user_enable_single_step(current);
684 } else {
685 toggle_bp_registers(AARCH64_DBG_REG_BCR, DBG_ACTIVE_EL1, 0);
686 kernel_step = this_cpu_ptr(&stepping_kernel_bp);
687
688 if (*kernel_step != ARM_KERNEL_STEP_NONE)
689 return 0;
690
691 if (kernel_active_single_step()) {
692 *kernel_step = ARM_KERNEL_STEP_SUSPEND;
693 } else {
694 *kernel_step = ARM_KERNEL_STEP_ACTIVE;
695 kernel_enable_single_step(regs);
696 }
697 }
698
699 return 0;
700 }
701
702 /*
703 * Arm64 hardware does not always report a watchpoint hit address that matches
704 * one of the watchpoints set. It can also report an address "near" the
705 * watchpoint if a single instruction access both watched and unwatched
706 * addresses. There is no straight-forward way, short of disassembling the
707 * offending instruction, to map that address back to the watchpoint. This
708 * function computes the distance of the memory access from the watchpoint as a
709 * heuristic for the likelyhood that a given access triggered the watchpoint.
710 *
711 * See Section D2.10.5 "Determining the memory location that caused a Watchpoint
712 * exception" of ARMv8 Architecture Reference Manual for details.
713 *
714 * The function returns the distance of the address from the bytes watched by
715 * the watchpoint. In case of an exact match, it returns 0.
716 */
get_distance_from_watchpoint(unsigned long addr,u64 val,struct arch_hw_breakpoint_ctrl * ctrl)717 static u64 get_distance_from_watchpoint(unsigned long addr, u64 val,
718 struct arch_hw_breakpoint_ctrl *ctrl)
719 {
720 u64 wp_low, wp_high;
721 u32 lens, lene;
722
723 lens = __ffs(ctrl->len);
724 lene = __fls(ctrl->len);
725
726 wp_low = val + lens;
727 wp_high = val + lene;
728 if (addr < wp_low)
729 return wp_low - addr;
730 else if (addr > wp_high)
731 return addr - wp_high;
732 else
733 return 0;
734 }
735
watchpoint_handler(unsigned long addr,unsigned int esr,struct pt_regs * regs)736 static int watchpoint_handler(unsigned long addr, unsigned int esr,
737 struct pt_regs *regs)
738 {
739 int i, step = 0, *kernel_step, access, closest_match = 0;
740 u64 min_dist = -1, dist;
741 u32 ctrl_reg;
742 u64 val;
743 struct perf_event *wp, **slots;
744 struct debug_info *debug_info;
745 struct arch_hw_breakpoint *info;
746 struct arch_hw_breakpoint_ctrl ctrl;
747
748 slots = this_cpu_ptr(wp_on_reg);
749 debug_info = ¤t->thread.debug;
750
751 /*
752 * Find all watchpoints that match the reported address. If no exact
753 * match is found. Attribute the hit to the closest watchpoint.
754 */
755 rcu_read_lock();
756 for (i = 0; i < core_num_wrps; ++i) {
757 wp = slots[i];
758 if (wp == NULL)
759 continue;
760
761 /*
762 * Check that the access type matches.
763 * 0 => load, otherwise => store
764 */
765 access = (esr & AARCH64_ESR_ACCESS_MASK) ? HW_BREAKPOINT_W :
766 HW_BREAKPOINT_R;
767 if (!(access & hw_breakpoint_type(wp)))
768 continue;
769
770 /* Check if the watchpoint value and byte select match. */
771 val = read_wb_reg(AARCH64_DBG_REG_WVR, i);
772 ctrl_reg = read_wb_reg(AARCH64_DBG_REG_WCR, i);
773 decode_ctrl_reg(ctrl_reg, &ctrl);
774 dist = get_distance_from_watchpoint(addr, val, &ctrl);
775 if (dist < min_dist) {
776 min_dist = dist;
777 closest_match = i;
778 }
779 /* Is this an exact match? */
780 if (dist != 0)
781 continue;
782
783 info = counter_arch_bp(wp);
784 info->trigger = addr;
785 perf_bp_event(wp, regs);
786
787 /* Do we need to handle the stepping? */
788 if (!wp->overflow_handler)
789 step = 1;
790 }
791 if (min_dist > 0 && min_dist != -1) {
792 /* No exact match found. */
793 wp = slots[closest_match];
794 info = counter_arch_bp(wp);
795 info->trigger = addr;
796 perf_bp_event(wp, regs);
797
798 /* Do we need to handle the stepping? */
799 if (!wp->overflow_handler)
800 step = 1;
801 }
802 rcu_read_unlock();
803
804 if (!step)
805 return 0;
806
807 /*
808 * We always disable EL0 watchpoints because the kernel can
809 * cause these to fire via an unprivileged access.
810 */
811 toggle_bp_registers(AARCH64_DBG_REG_WCR, DBG_ACTIVE_EL0, 0);
812
813 if (user_mode(regs)) {
814 debug_info->wps_disabled = 1;
815
816 /* If we're already stepping a breakpoint, just return. */
817 if (debug_info->bps_disabled)
818 return 0;
819
820 if (test_thread_flag(TIF_SINGLESTEP))
821 debug_info->suspended_step = 1;
822 else
823 user_enable_single_step(current);
824 } else {
825 toggle_bp_registers(AARCH64_DBG_REG_WCR, DBG_ACTIVE_EL1, 0);
826 kernel_step = this_cpu_ptr(&stepping_kernel_bp);
827
828 if (*kernel_step != ARM_KERNEL_STEP_NONE)
829 return 0;
830
831 if (kernel_active_single_step()) {
832 *kernel_step = ARM_KERNEL_STEP_SUSPEND;
833 } else {
834 *kernel_step = ARM_KERNEL_STEP_ACTIVE;
835 kernel_enable_single_step(regs);
836 }
837 }
838
839 return 0;
840 }
841
842 /*
843 * Handle single-step exception.
844 */
reinstall_suspended_bps(struct pt_regs * regs)845 int reinstall_suspended_bps(struct pt_regs *regs)
846 {
847 struct debug_info *debug_info = ¤t->thread.debug;
848 int handled_exception = 0, *kernel_step;
849
850 kernel_step = this_cpu_ptr(&stepping_kernel_bp);
851
852 /*
853 * Called from single-step exception handler.
854 * Return 0 if execution can resume, 1 if a SIGTRAP should be
855 * reported.
856 */
857 if (user_mode(regs)) {
858 if (debug_info->bps_disabled) {
859 debug_info->bps_disabled = 0;
860 toggle_bp_registers(AARCH64_DBG_REG_BCR, DBG_ACTIVE_EL0, 1);
861 handled_exception = 1;
862 }
863
864 if (debug_info->wps_disabled) {
865 debug_info->wps_disabled = 0;
866 toggle_bp_registers(AARCH64_DBG_REG_WCR, DBG_ACTIVE_EL0, 1);
867 handled_exception = 1;
868 }
869
870 if (handled_exception) {
871 if (debug_info->suspended_step) {
872 debug_info->suspended_step = 0;
873 /* Allow exception handling to fall-through. */
874 handled_exception = 0;
875 } else {
876 user_disable_single_step(current);
877 }
878 }
879 } else if (*kernel_step != ARM_KERNEL_STEP_NONE) {
880 toggle_bp_registers(AARCH64_DBG_REG_BCR, DBG_ACTIVE_EL1, 1);
881 toggle_bp_registers(AARCH64_DBG_REG_WCR, DBG_ACTIVE_EL1, 1);
882
883 if (!debug_info->wps_disabled)
884 toggle_bp_registers(AARCH64_DBG_REG_WCR, DBG_ACTIVE_EL0, 1);
885
886 if (*kernel_step != ARM_KERNEL_STEP_SUSPEND) {
887 kernel_disable_single_step();
888 handled_exception = 1;
889 } else {
890 handled_exception = 0;
891 }
892
893 *kernel_step = ARM_KERNEL_STEP_NONE;
894 }
895
896 return !handled_exception;
897 }
898
899 /*
900 * Context-switcher for restoring suspended breakpoints.
901 */
hw_breakpoint_thread_switch(struct task_struct * next)902 void hw_breakpoint_thread_switch(struct task_struct *next)
903 {
904 /*
905 * current next
906 * disabled: 0 0 => The usual case, NOTIFY_DONE
907 * 0 1 => Disable the registers
908 * 1 0 => Enable the registers
909 * 1 1 => NOTIFY_DONE. per-task bps will
910 * get taken care of by perf.
911 */
912
913 struct debug_info *current_debug_info, *next_debug_info;
914
915 current_debug_info = ¤t->thread.debug;
916 next_debug_info = &next->thread.debug;
917
918 /* Update breakpoints. */
919 if (current_debug_info->bps_disabled != next_debug_info->bps_disabled)
920 toggle_bp_registers(AARCH64_DBG_REG_BCR,
921 DBG_ACTIVE_EL0,
922 !next_debug_info->bps_disabled);
923
924 /* Update watchpoints. */
925 if (current_debug_info->wps_disabled != next_debug_info->wps_disabled)
926 toggle_bp_registers(AARCH64_DBG_REG_WCR,
927 DBG_ACTIVE_EL0,
928 !next_debug_info->wps_disabled);
929 }
930
931 /*
932 * CPU initialisation.
933 */
hw_breakpoint_reset(void * unused)934 static void hw_breakpoint_reset(void *unused)
935 {
936 int i;
937 struct perf_event **slots;
938 /*
939 * When a CPU goes through cold-boot, it does not have any installed
940 * slot, so it is safe to share the same function for restoring and
941 * resetting breakpoints; when a CPU is hotplugged in, it goes
942 * through the slots, which are all empty, hence it just resets control
943 * and value for debug registers.
944 * When this function is triggered on warm-boot through a CPU PM
945 * notifier some slots might be initialized; if so they are
946 * reprogrammed according to the debug slots content.
947 */
948 for (slots = this_cpu_ptr(bp_on_reg), i = 0; i < core_num_brps; ++i) {
949 if (slots[i]) {
950 hw_breakpoint_control(slots[i], HW_BREAKPOINT_RESTORE);
951 } else {
952 write_wb_reg(AARCH64_DBG_REG_BCR, i, 0UL);
953 write_wb_reg(AARCH64_DBG_REG_BVR, i, 0UL);
954 }
955 }
956
957 for (slots = this_cpu_ptr(wp_on_reg), i = 0; i < core_num_wrps; ++i) {
958 if (slots[i]) {
959 hw_breakpoint_control(slots[i], HW_BREAKPOINT_RESTORE);
960 } else {
961 write_wb_reg(AARCH64_DBG_REG_WCR, i, 0UL);
962 write_wb_reg(AARCH64_DBG_REG_WVR, i, 0UL);
963 }
964 }
965 }
966
hw_breakpoint_reset_notify(struct notifier_block * self,unsigned long action,void * hcpu)967 static int hw_breakpoint_reset_notify(struct notifier_block *self,
968 unsigned long action,
969 void *hcpu)
970 {
971 int cpu = (long)hcpu;
972 if ((action & ~CPU_TASKS_FROZEN) == CPU_ONLINE)
973 smp_call_function_single(cpu, hw_breakpoint_reset, NULL, 1);
974 return NOTIFY_OK;
975 }
976
977 static struct notifier_block hw_breakpoint_reset_nb = {
978 .notifier_call = hw_breakpoint_reset_notify,
979 };
980
981 #ifdef CONFIG_ARM64_CPU_SUSPEND
982 extern void cpu_suspend_set_dbg_restorer(void (*hw_bp_restore)(void *));
983 #else
cpu_suspend_set_dbg_restorer(void (* hw_bp_restore)(void *))984 static inline void cpu_suspend_set_dbg_restorer(void (*hw_bp_restore)(void *))
985 {
986 }
987 #endif
988
989 /*
990 * One-time initialisation.
991 */
arch_hw_breakpoint_init(void)992 static int __init arch_hw_breakpoint_init(void)
993 {
994 core_num_brps = get_num_brps();
995 core_num_wrps = get_num_wrps();
996
997 pr_info("found %d breakpoint and %d watchpoint registers.\n",
998 core_num_brps, core_num_wrps);
999
1000 cpu_notifier_register_begin();
1001
1002 /*
1003 * Reset the breakpoint resources. We assume that a halting
1004 * debugger will leave the world in a nice state for us.
1005 */
1006 smp_call_function(hw_breakpoint_reset, NULL, 1);
1007 hw_breakpoint_reset(NULL);
1008
1009 /* Register debug fault handlers. */
1010 hook_debug_fault_code(DBG_ESR_EVT_HWBP, breakpoint_handler, SIGTRAP,
1011 TRAP_HWBKPT, "hw-breakpoint handler");
1012 hook_debug_fault_code(DBG_ESR_EVT_HWWP, watchpoint_handler, SIGTRAP,
1013 TRAP_HWBKPT, "hw-watchpoint handler");
1014
1015 /* Register hotplug notifier. */
1016 __register_cpu_notifier(&hw_breakpoint_reset_nb);
1017
1018 cpu_notifier_register_done();
1019
1020 /* Register cpu_suspend hw breakpoint restore hook */
1021 cpu_suspend_set_dbg_restorer(hw_breakpoint_reset);
1022
1023 return 0;
1024 }
1025 arch_initcall(arch_hw_breakpoint_init);
1026
hw_breakpoint_pmu_read(struct perf_event * bp)1027 void hw_breakpoint_pmu_read(struct perf_event *bp)
1028 {
1029 }
1030
1031 /*
1032 * Dummy function to register with die_notifier.
1033 */
hw_breakpoint_exceptions_notify(struct notifier_block * unused,unsigned long val,void * data)1034 int hw_breakpoint_exceptions_notify(struct notifier_block *unused,
1035 unsigned long val, void *data)
1036 {
1037 return NOTIFY_DONE;
1038 }
1039