1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * Kernel Probes (KProbes)
4 *
5 * Copyright (C) IBM Corporation, 2002, 2004
6 *
7 * 2002-Oct Created by Vamsi Krishna S <vamsi_krishna@in.ibm.com> Kernel
8 * Probes initial implementation ( includes contributions from
9 * Rusty Russell).
10 * 2004-July Suparna Bhattacharya <suparna@in.ibm.com> added jumper probes
11 * interface to access function arguments.
12 * 2004-Oct Jim Keniston <jkenisto@us.ibm.com> and Prasanna S Panchamukhi
13 * <prasanna@in.ibm.com> adapted for x86_64 from i386.
14 * 2005-Mar Roland McGrath <roland@redhat.com>
15 * Fixed to handle %rip-relative addressing mode correctly.
16 * 2005-May Hien Nguyen <hien@us.ibm.com>, Jim Keniston
17 * <jkenisto@us.ibm.com> and Prasanna S Panchamukhi
18 * <prasanna@in.ibm.com> added function-return probes.
19 * 2005-May Rusty Lynch <rusty.lynch@intel.com>
20 * Added function return probes functionality
21 * 2006-Feb Masami Hiramatsu <hiramatu@sdl.hitachi.co.jp> added
22 * kprobe-booster and kretprobe-booster for i386.
23 * 2007-Dec Masami Hiramatsu <mhiramat@redhat.com> added kprobe-booster
24 * and kretprobe-booster for x86-64
25 * 2007-Dec Masami Hiramatsu <mhiramat@redhat.com>, Arjan van de Ven
26 * <arjan@infradead.org> and Jim Keniston <jkenisto@us.ibm.com>
27 * unified x86 kprobes code.
28 */
29 #include <linux/kprobes.h>
30 #include <linux/ptrace.h>
31 #include <linux/string.h>
32 #include <linux/slab.h>
33 #include <linux/hardirq.h>
34 #include <linux/preempt.h>
35 #include <linux/sched/debug.h>
36 #include <linux/perf_event.h>
37 #include <linux/extable.h>
38 #include <linux/kdebug.h>
39 #include <linux/kallsyms.h>
40 #include <linux/ftrace.h>
41 #include <linux/kasan.h>
42 #include <linux/moduleloader.h>
43 #include <linux/objtool.h>
44 #include <linux/vmalloc.h>
45 #include <linux/pgtable.h>
46
47 #include <asm/text-patching.h>
48 #include <asm/cacheflush.h>
49 #include <asm/desc.h>
50 #include <linux/uaccess.h>
51 #include <asm/alternative.h>
52 #include <asm/insn.h>
53 #include <asm/debugreg.h>
54 #include <asm/set_memory.h>
55
56 #include "common.h"
57
58 DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL;
59 DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk);
60
61 #define stack_addr(regs) ((unsigned long *)regs->sp)
62
63 #define W(row, b0, b1, b2, b3, b4, b5, b6, b7, b8, b9, ba, bb, bc, bd, be, bf)\
64 (((b0##UL << 0x0)|(b1##UL << 0x1)|(b2##UL << 0x2)|(b3##UL << 0x3) | \
65 (b4##UL << 0x4)|(b5##UL << 0x5)|(b6##UL << 0x6)|(b7##UL << 0x7) | \
66 (b8##UL << 0x8)|(b9##UL << 0x9)|(ba##UL << 0xa)|(bb##UL << 0xb) | \
67 (bc##UL << 0xc)|(bd##UL << 0xd)|(be##UL << 0xe)|(bf##UL << 0xf)) \
68 << (row % 32))
69 /*
70 * Undefined/reserved opcodes, conditional jump, Opcode Extension
71 * Groups, and some special opcodes can not boost.
72 * This is non-const and volatile to keep gcc from statically
73 * optimizing it out, as variable_test_bit makes gcc think only
74 * *(unsigned long*) is used.
75 */
76 static volatile u32 twobyte_is_boostable[256 / 32] = {
77 /* 0 1 2 3 4 5 6 7 8 9 a b c d e f */
78 /* ---------------------------------------------- */
79 W(0x00, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0) | /* 00 */
80 W(0x10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1) , /* 10 */
81 W(0x20, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) | /* 20 */
82 W(0x30, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) , /* 30 */
83 W(0x40, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* 40 */
84 W(0x50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) , /* 50 */
85 W(0x60, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1) | /* 60 */
86 W(0x70, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 1, 1) , /* 70 */
87 W(0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) | /* 80 */
88 W(0x90, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* 90 */
89 W(0xa0, 1, 1, 0, 1, 1, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1) | /* a0 */
90 W(0xb0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, 1, 1) , /* b0 */
91 W(0xc0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1) | /* c0 */
92 W(0xd0, 0, 1, 1, 1, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1) , /* d0 */
93 W(0xe0, 0, 1, 1, 0, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1) | /* e0 */
94 W(0xf0, 0, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 0, 1, 1, 1, 0) /* f0 */
95 /* ----------------------------------------------- */
96 /* 0 1 2 3 4 5 6 7 8 9 a b c d e f */
97 };
98 #undef W
99
100 struct kretprobe_blackpoint kretprobe_blacklist[] = {
101 {"__switch_to", }, /* This function switches only current task, but
102 doesn't switch kernel stack.*/
103 {NULL, NULL} /* Terminator */
104 };
105
106 const int kretprobe_blacklist_size = ARRAY_SIZE(kretprobe_blacklist);
107
108 static nokprobe_inline void
__synthesize_relative_insn(void * dest,void * from,void * to,u8 op)109 __synthesize_relative_insn(void *dest, void *from, void *to, u8 op)
110 {
111 struct __arch_relative_insn {
112 u8 op;
113 s32 raddr;
114 } __packed *insn;
115
116 insn = (struct __arch_relative_insn *)dest;
117 insn->raddr = (s32)((long)(to) - ((long)(from) + 5));
118 insn->op = op;
119 }
120
121 /* Insert a jump instruction at address 'from', which jumps to address 'to'.*/
synthesize_reljump(void * dest,void * from,void * to)122 void synthesize_reljump(void *dest, void *from, void *to)
123 {
124 __synthesize_relative_insn(dest, from, to, JMP32_INSN_OPCODE);
125 }
126 NOKPROBE_SYMBOL(synthesize_reljump);
127
128 /* Insert a call instruction at address 'from', which calls address 'to'.*/
synthesize_relcall(void * dest,void * from,void * to)129 void synthesize_relcall(void *dest, void *from, void *to)
130 {
131 __synthesize_relative_insn(dest, from, to, CALL_INSN_OPCODE);
132 }
133 NOKPROBE_SYMBOL(synthesize_relcall);
134
135 /*
136 * Skip the prefixes of the instruction.
137 */
skip_prefixes(kprobe_opcode_t * insn)138 static kprobe_opcode_t *skip_prefixes(kprobe_opcode_t *insn)
139 {
140 insn_attr_t attr;
141
142 attr = inat_get_opcode_attribute((insn_byte_t)*insn);
143 while (inat_is_legacy_prefix(attr)) {
144 insn++;
145 attr = inat_get_opcode_attribute((insn_byte_t)*insn);
146 }
147 #ifdef CONFIG_X86_64
148 if (inat_is_rex_prefix(attr))
149 insn++;
150 #endif
151 return insn;
152 }
153 NOKPROBE_SYMBOL(skip_prefixes);
154
155 /*
156 * Returns non-zero if INSN is boostable.
157 * RIP relative instructions are adjusted at copying time in 64 bits mode
158 */
can_boost(struct insn * insn,void * addr)159 int can_boost(struct insn *insn, void *addr)
160 {
161 kprobe_opcode_t opcode;
162 insn_byte_t prefix;
163 int i;
164
165 if (search_exception_tables((unsigned long)addr))
166 return 0; /* Page fault may occur on this address. */
167
168 /* 2nd-byte opcode */
169 if (insn->opcode.nbytes == 2)
170 return test_bit(insn->opcode.bytes[1],
171 (unsigned long *)twobyte_is_boostable);
172
173 if (insn->opcode.nbytes != 1)
174 return 0;
175
176 for_each_insn_prefix(insn, i, prefix) {
177 insn_attr_t attr;
178
179 attr = inat_get_opcode_attribute(prefix);
180 /* Can't boost Address-size override prefix and CS override prefix */
181 if (prefix == 0x2e || inat_is_address_size_prefix(attr))
182 return 0;
183 }
184
185 opcode = insn->opcode.bytes[0];
186
187 switch (opcode & 0xf0) {
188 case 0x60:
189 /* can't boost "bound" */
190 return (opcode != 0x62);
191 case 0x70:
192 return 0; /* can't boost conditional jump */
193 case 0x90:
194 return opcode != 0x9a; /* can't boost call far */
195 case 0xc0:
196 /* can't boost software-interruptions */
197 return (0xc1 < opcode && opcode < 0xcc) || opcode == 0xcf;
198 case 0xd0:
199 /* can boost AA* and XLAT */
200 return (opcode == 0xd4 || opcode == 0xd5 || opcode == 0xd7);
201 case 0xe0:
202 /* can boost in/out and absolute jmps */
203 return ((opcode & 0x04) || opcode == 0xea);
204 case 0xf0:
205 /* clear and set flags are boostable */
206 return (opcode == 0xf5 || (0xf7 < opcode && opcode < 0xfe));
207 default:
208 /* call is not boostable */
209 return opcode != 0x9a;
210 }
211 }
212
213 static unsigned long
__recover_probed_insn(kprobe_opcode_t * buf,unsigned long addr)214 __recover_probed_insn(kprobe_opcode_t *buf, unsigned long addr)
215 {
216 struct kprobe *kp;
217 unsigned long faddr;
218
219 kp = get_kprobe((void *)addr);
220 faddr = ftrace_location(addr);
221 /*
222 * Addresses inside the ftrace location are refused by
223 * arch_check_ftrace_location(). Something went terribly wrong
224 * if such an address is checked here.
225 */
226 if (WARN_ON(faddr && faddr != addr))
227 return 0UL;
228 /*
229 * Use the current code if it is not modified by Kprobe
230 * and it cannot be modified by ftrace.
231 */
232 if (!kp && !faddr)
233 return addr;
234
235 /*
236 * Basically, kp->ainsn.insn has an original instruction.
237 * However, RIP-relative instruction can not do single-stepping
238 * at different place, __copy_instruction() tweaks the displacement of
239 * that instruction. In that case, we can't recover the instruction
240 * from the kp->ainsn.insn.
241 *
242 * On the other hand, in case on normal Kprobe, kp->opcode has a copy
243 * of the first byte of the probed instruction, which is overwritten
244 * by int3. And the instruction at kp->addr is not modified by kprobes
245 * except for the first byte, we can recover the original instruction
246 * from it and kp->opcode.
247 *
248 * In case of Kprobes using ftrace, we do not have a copy of
249 * the original instruction. In fact, the ftrace location might
250 * be modified at anytime and even could be in an inconsistent state.
251 * Fortunately, we know that the original code is the ideal 5-byte
252 * long NOP.
253 */
254 if (copy_from_kernel_nofault(buf, (void *)addr,
255 MAX_INSN_SIZE * sizeof(kprobe_opcode_t)))
256 return 0UL;
257
258 if (faddr)
259 memcpy(buf, ideal_nops[NOP_ATOMIC5], 5);
260 else
261 buf[0] = kp->opcode;
262 return (unsigned long)buf;
263 }
264
265 /*
266 * Recover the probed instruction at addr for further analysis.
267 * Caller must lock kprobes by kprobe_mutex, or disable preemption
268 * for preventing to release referencing kprobes.
269 * Returns zero if the instruction can not get recovered (or access failed).
270 */
recover_probed_instruction(kprobe_opcode_t * buf,unsigned long addr)271 unsigned long recover_probed_instruction(kprobe_opcode_t *buf, unsigned long addr)
272 {
273 unsigned long __addr;
274
275 __addr = __recover_optprobed_insn(buf, addr);
276 if (__addr != addr)
277 return __addr;
278
279 return __recover_probed_insn(buf, addr);
280 }
281
282 /* Check if paddr is at an instruction boundary */
can_probe(unsigned long paddr)283 static int can_probe(unsigned long paddr)
284 {
285 unsigned long addr, __addr, offset = 0;
286 struct insn insn;
287 kprobe_opcode_t buf[MAX_INSN_SIZE];
288
289 if (!kallsyms_lookup_size_offset(paddr, NULL, &offset))
290 return 0;
291
292 /* Decode instructions */
293 addr = paddr - offset;
294 while (addr < paddr) {
295 /*
296 * Check if the instruction has been modified by another
297 * kprobe, in which case we replace the breakpoint by the
298 * original instruction in our buffer.
299 * Also, jump optimization will change the breakpoint to
300 * relative-jump. Since the relative-jump itself is
301 * normally used, we just go through if there is no kprobe.
302 */
303 __addr = recover_probed_instruction(buf, addr);
304 if (!__addr)
305 return 0;
306 kernel_insn_init(&insn, (void *)__addr, MAX_INSN_SIZE);
307 insn_get_length(&insn);
308
309 /*
310 * Another debugging subsystem might insert this breakpoint.
311 * In that case, we can't recover it.
312 */
313 if (insn.opcode.bytes[0] == INT3_INSN_OPCODE)
314 return 0;
315 addr += insn.length;
316 }
317
318 return (addr == paddr);
319 }
320
321 /*
322 * Returns non-zero if opcode modifies the interrupt flag.
323 */
is_IF_modifier(kprobe_opcode_t * insn)324 static int is_IF_modifier(kprobe_opcode_t *insn)
325 {
326 /* Skip prefixes */
327 insn = skip_prefixes(insn);
328
329 switch (*insn) {
330 case 0xfa: /* cli */
331 case 0xfb: /* sti */
332 case 0xcf: /* iret/iretd */
333 case 0x9d: /* popf/popfd */
334 return 1;
335 }
336
337 return 0;
338 }
339
340 /*
341 * Copy an instruction with recovering modified instruction by kprobes
342 * and adjust the displacement if the instruction uses the %rip-relative
343 * addressing mode. Note that since @real will be the final place of copied
344 * instruction, displacement must be adjust by @real, not @dest.
345 * This returns the length of copied instruction, or 0 if it has an error.
346 */
__copy_instruction(u8 * dest,u8 * src,u8 * real,struct insn * insn)347 int __copy_instruction(u8 *dest, u8 *src, u8 *real, struct insn *insn)
348 {
349 kprobe_opcode_t buf[MAX_INSN_SIZE];
350 unsigned long recovered_insn =
351 recover_probed_instruction(buf, (unsigned long)src);
352
353 if (!recovered_insn || !insn)
354 return 0;
355
356 /* This can access kernel text if given address is not recovered */
357 if (copy_from_kernel_nofault(dest, (void *)recovered_insn,
358 MAX_INSN_SIZE))
359 return 0;
360
361 kernel_insn_init(insn, dest, MAX_INSN_SIZE);
362 insn_get_length(insn);
363
364 /* We can not probe force emulate prefixed instruction */
365 if (insn_has_emulate_prefix(insn))
366 return 0;
367
368 /* Another subsystem puts a breakpoint, failed to recover */
369 if (insn->opcode.bytes[0] == INT3_INSN_OPCODE)
370 return 0;
371
372 /* We should not singlestep on the exception masking instructions */
373 if (insn_masking_exception(insn))
374 return 0;
375
376 #ifdef CONFIG_X86_64
377 /* Only x86_64 has RIP relative instructions */
378 if (insn_rip_relative(insn)) {
379 s64 newdisp;
380 u8 *disp;
381 /*
382 * The copied instruction uses the %rip-relative addressing
383 * mode. Adjust the displacement for the difference between
384 * the original location of this instruction and the location
385 * of the copy that will actually be run. The tricky bit here
386 * is making sure that the sign extension happens correctly in
387 * this calculation, since we need a signed 32-bit result to
388 * be sign-extended to 64 bits when it's added to the %rip
389 * value and yield the same 64-bit result that the sign-
390 * extension of the original signed 32-bit displacement would
391 * have given.
392 */
393 newdisp = (u8 *) src + (s64) insn->displacement.value
394 - (u8 *) real;
395 if ((s64) (s32) newdisp != newdisp) {
396 pr_err("Kprobes error: new displacement does not fit into s32 (%llx)\n", newdisp);
397 return 0;
398 }
399 disp = (u8 *) dest + insn_offset_displacement(insn);
400 *(s32 *) disp = (s32) newdisp;
401 }
402 #endif
403 return insn->length;
404 }
405
406 /* Prepare reljump right after instruction to boost */
prepare_boost(kprobe_opcode_t * buf,struct kprobe * p,struct insn * insn)407 static int prepare_boost(kprobe_opcode_t *buf, struct kprobe *p,
408 struct insn *insn)
409 {
410 int len = insn->length;
411
412 if (can_boost(insn, p->addr) &&
413 MAX_INSN_SIZE - len >= JMP32_INSN_SIZE) {
414 /*
415 * These instructions can be executed directly if it
416 * jumps back to correct address.
417 */
418 synthesize_reljump(buf + len, p->ainsn.insn + len,
419 p->addr + insn->length);
420 len += JMP32_INSN_SIZE;
421 p->ainsn.boostable = true;
422 } else {
423 p->ainsn.boostable = false;
424 }
425
426 return len;
427 }
428
429 /* Make page to RO mode when allocate it */
alloc_insn_page(void)430 void *alloc_insn_page(void)
431 {
432 void *page;
433
434 page = module_alloc(PAGE_SIZE);
435 if (!page)
436 return NULL;
437
438 set_vm_flush_reset_perms(page);
439 /*
440 * First make the page read-only, and only then make it executable to
441 * prevent it from being W+X in between.
442 */
443 set_memory_ro((unsigned long)page, 1);
444
445 /*
446 * TODO: Once additional kernel code protection mechanisms are set, ensure
447 * that the page was not maliciously altered and it is still zeroed.
448 */
449 set_memory_x((unsigned long)page, 1);
450
451 return page;
452 }
453
454 /* Recover page to RW mode before releasing it */
free_insn_page(void * page)455 void free_insn_page(void *page)
456 {
457 module_memfree(page);
458 }
459
arch_copy_kprobe(struct kprobe * p)460 static int arch_copy_kprobe(struct kprobe *p)
461 {
462 struct insn insn;
463 kprobe_opcode_t buf[MAX_INSN_SIZE];
464 int len;
465
466 /* Copy an instruction with recovering if other optprobe modifies it.*/
467 len = __copy_instruction(buf, p->addr, p->ainsn.insn, &insn);
468 if (!len)
469 return -EINVAL;
470
471 /*
472 * __copy_instruction can modify the displacement of the instruction,
473 * but it doesn't affect boostable check.
474 */
475 len = prepare_boost(buf, p, &insn);
476
477 /* Check whether the instruction modifies Interrupt Flag or not */
478 p->ainsn.if_modifier = is_IF_modifier(buf);
479
480 /* Also, displacement change doesn't affect the first byte */
481 p->opcode = buf[0];
482
483 p->ainsn.tp_len = len;
484 perf_event_text_poke(p->ainsn.insn, NULL, 0, buf, len);
485
486 /* OK, write back the instruction(s) into ROX insn buffer */
487 text_poke(p->ainsn.insn, buf, len);
488
489 return 0;
490 }
491
arch_prepare_kprobe(struct kprobe * p)492 int arch_prepare_kprobe(struct kprobe *p)
493 {
494 int ret;
495
496 if (alternatives_text_reserved(p->addr, p->addr))
497 return -EINVAL;
498
499 if (!can_probe((unsigned long)p->addr))
500 return -EILSEQ;
501 /* insn: must be on special executable page on x86. */
502 p->ainsn.insn = get_insn_slot();
503 if (!p->ainsn.insn)
504 return -ENOMEM;
505
506 ret = arch_copy_kprobe(p);
507 if (ret) {
508 free_insn_slot(p->ainsn.insn, 0);
509 p->ainsn.insn = NULL;
510 }
511
512 return ret;
513 }
514
arch_arm_kprobe(struct kprobe * p)515 void arch_arm_kprobe(struct kprobe *p)
516 {
517 u8 int3 = INT3_INSN_OPCODE;
518
519 text_poke(p->addr, &int3, 1);
520 text_poke_sync();
521 perf_event_text_poke(p->addr, &p->opcode, 1, &int3, 1);
522 }
523
arch_disarm_kprobe(struct kprobe * p)524 void arch_disarm_kprobe(struct kprobe *p)
525 {
526 u8 int3 = INT3_INSN_OPCODE;
527
528 perf_event_text_poke(p->addr, &int3, 1, &p->opcode, 1);
529 text_poke(p->addr, &p->opcode, 1);
530 text_poke_sync();
531 }
532
arch_remove_kprobe(struct kprobe * p)533 void arch_remove_kprobe(struct kprobe *p)
534 {
535 if (p->ainsn.insn) {
536 /* Record the perf event before freeing the slot */
537 perf_event_text_poke(p->ainsn.insn, p->ainsn.insn,
538 p->ainsn.tp_len, NULL, 0);
539 free_insn_slot(p->ainsn.insn, p->ainsn.boostable);
540 p->ainsn.insn = NULL;
541 }
542 }
543
544 static nokprobe_inline void
save_previous_kprobe(struct kprobe_ctlblk * kcb)545 save_previous_kprobe(struct kprobe_ctlblk *kcb)
546 {
547 kcb->prev_kprobe.kp = kprobe_running();
548 kcb->prev_kprobe.status = kcb->kprobe_status;
549 kcb->prev_kprobe.old_flags = kcb->kprobe_old_flags;
550 kcb->prev_kprobe.saved_flags = kcb->kprobe_saved_flags;
551 }
552
553 static nokprobe_inline void
restore_previous_kprobe(struct kprobe_ctlblk * kcb)554 restore_previous_kprobe(struct kprobe_ctlblk *kcb)
555 {
556 __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp);
557 kcb->kprobe_status = kcb->prev_kprobe.status;
558 kcb->kprobe_old_flags = kcb->prev_kprobe.old_flags;
559 kcb->kprobe_saved_flags = kcb->prev_kprobe.saved_flags;
560 }
561
562 static nokprobe_inline void
set_current_kprobe(struct kprobe * p,struct pt_regs * regs,struct kprobe_ctlblk * kcb)563 set_current_kprobe(struct kprobe *p, struct pt_regs *regs,
564 struct kprobe_ctlblk *kcb)
565 {
566 __this_cpu_write(current_kprobe, p);
567 kcb->kprobe_saved_flags = kcb->kprobe_old_flags
568 = (regs->flags & (X86_EFLAGS_TF | X86_EFLAGS_IF));
569 if (p->ainsn.if_modifier)
570 kcb->kprobe_saved_flags &= ~X86_EFLAGS_IF;
571 }
572
clear_btf(void)573 static nokprobe_inline void clear_btf(void)
574 {
575 if (test_thread_flag(TIF_BLOCKSTEP)) {
576 unsigned long debugctl = get_debugctlmsr();
577
578 debugctl &= ~DEBUGCTLMSR_BTF;
579 update_debugctlmsr(debugctl);
580 }
581 }
582
restore_btf(void)583 static nokprobe_inline void restore_btf(void)
584 {
585 if (test_thread_flag(TIF_BLOCKSTEP)) {
586 unsigned long debugctl = get_debugctlmsr();
587
588 debugctl |= DEBUGCTLMSR_BTF;
589 update_debugctlmsr(debugctl);
590 }
591 }
592
arch_prepare_kretprobe(struct kretprobe_instance * ri,struct pt_regs * regs)593 void arch_prepare_kretprobe(struct kretprobe_instance *ri, struct pt_regs *regs)
594 {
595 unsigned long *sara = stack_addr(regs);
596
597 ri->ret_addr = (kprobe_opcode_t *) *sara;
598 ri->fp = sara;
599
600 /* Replace the return addr with trampoline addr */
601 *sara = (unsigned long) &kretprobe_trampoline;
602 }
603 NOKPROBE_SYMBOL(arch_prepare_kretprobe);
604
setup_singlestep(struct kprobe * p,struct pt_regs * regs,struct kprobe_ctlblk * kcb,int reenter)605 static void setup_singlestep(struct kprobe *p, struct pt_regs *regs,
606 struct kprobe_ctlblk *kcb, int reenter)
607 {
608 if (setup_detour_execution(p, regs, reenter))
609 return;
610
611 #if !defined(CONFIG_PREEMPTION)
612 if (p->ainsn.boostable && !p->post_handler) {
613 /* Boost up -- we can execute copied instructions directly */
614 if (!reenter)
615 reset_current_kprobe();
616 /*
617 * Reentering boosted probe doesn't reset current_kprobe,
618 * nor set current_kprobe, because it doesn't use single
619 * stepping.
620 */
621 regs->ip = (unsigned long)p->ainsn.insn;
622 return;
623 }
624 #endif
625 if (reenter) {
626 save_previous_kprobe(kcb);
627 set_current_kprobe(p, regs, kcb);
628 kcb->kprobe_status = KPROBE_REENTER;
629 } else
630 kcb->kprobe_status = KPROBE_HIT_SS;
631 /* Prepare real single stepping */
632 clear_btf();
633 regs->flags |= X86_EFLAGS_TF;
634 regs->flags &= ~X86_EFLAGS_IF;
635 /* single step inline if the instruction is an int3 */
636 if (p->opcode == INT3_INSN_OPCODE)
637 regs->ip = (unsigned long)p->addr;
638 else
639 regs->ip = (unsigned long)p->ainsn.insn;
640 }
641 NOKPROBE_SYMBOL(setup_singlestep);
642
643 /*
644 * We have reentered the kprobe_handler(), since another probe was hit while
645 * within the handler. We save the original kprobes variables and just single
646 * step on the instruction of the new probe without calling any user handlers.
647 */
reenter_kprobe(struct kprobe * p,struct pt_regs * regs,struct kprobe_ctlblk * kcb)648 static int reenter_kprobe(struct kprobe *p, struct pt_regs *regs,
649 struct kprobe_ctlblk *kcb)
650 {
651 switch (kcb->kprobe_status) {
652 case KPROBE_HIT_SSDONE:
653 case KPROBE_HIT_ACTIVE:
654 case KPROBE_HIT_SS:
655 kprobes_inc_nmissed_count(p);
656 setup_singlestep(p, regs, kcb, 1);
657 break;
658 case KPROBE_REENTER:
659 /* A probe has been hit in the codepath leading up to, or just
660 * after, single-stepping of a probed instruction. This entire
661 * codepath should strictly reside in .kprobes.text section.
662 * Raise a BUG or we'll continue in an endless reentering loop
663 * and eventually a stack overflow.
664 */
665 pr_err("Unrecoverable kprobe detected.\n");
666 dump_kprobe(p);
667 BUG();
668 default:
669 /* impossible cases */
670 WARN_ON(1);
671 return 0;
672 }
673
674 return 1;
675 }
676 NOKPROBE_SYMBOL(reenter_kprobe);
677
678 /*
679 * Interrupts are disabled on entry as trap3 is an interrupt gate and they
680 * remain disabled throughout this function.
681 */
kprobe_int3_handler(struct pt_regs * regs)682 int kprobe_int3_handler(struct pt_regs *regs)
683 {
684 kprobe_opcode_t *addr;
685 struct kprobe *p;
686 struct kprobe_ctlblk *kcb;
687
688 if (user_mode(regs))
689 return 0;
690
691 addr = (kprobe_opcode_t *)(regs->ip - sizeof(kprobe_opcode_t));
692 /*
693 * We don't want to be preempted for the entire duration of kprobe
694 * processing. Since int3 and debug trap disables irqs and we clear
695 * IF while singlestepping, it must be no preemptible.
696 */
697
698 kcb = get_kprobe_ctlblk();
699 p = get_kprobe(addr);
700
701 if (p) {
702 if (kprobe_running()) {
703 if (reenter_kprobe(p, regs, kcb))
704 return 1;
705 } else {
706 set_current_kprobe(p, regs, kcb);
707 kcb->kprobe_status = KPROBE_HIT_ACTIVE;
708
709 /*
710 * If we have no pre-handler or it returned 0, we
711 * continue with normal processing. If we have a
712 * pre-handler and it returned non-zero, that means
713 * user handler setup registers to exit to another
714 * instruction, we must skip the single stepping.
715 */
716 if (!p->pre_handler || !p->pre_handler(p, regs))
717 setup_singlestep(p, regs, kcb, 0);
718 else
719 reset_current_kprobe();
720 return 1;
721 }
722 } else if (*addr != INT3_INSN_OPCODE) {
723 /*
724 * The breakpoint instruction was removed right
725 * after we hit it. Another cpu has removed
726 * either a probepoint or a debugger breakpoint
727 * at this address. In either case, no further
728 * handling of this interrupt is appropriate.
729 * Back up over the (now missing) int3 and run
730 * the original instruction.
731 */
732 regs->ip = (unsigned long)addr;
733 return 1;
734 } /* else: not a kprobe fault; let the kernel handle it */
735
736 return 0;
737 }
738 NOKPROBE_SYMBOL(kprobe_int3_handler);
739
740 /*
741 * When a retprobed function returns, this code saves registers and
742 * calls trampoline_handler() runs, which calls the kretprobe's handler.
743 */
744 asm(
745 ".text\n"
746 ".global kretprobe_trampoline\n"
747 ".type kretprobe_trampoline, @function\n"
748 "kretprobe_trampoline:\n"
749 /* We don't bother saving the ss register */
750 #ifdef CONFIG_X86_64
751 " pushq %rsp\n"
752 " pushfq\n"
753 SAVE_REGS_STRING
754 " movq %rsp, %rdi\n"
755 " call trampoline_handler\n"
756 /* Replace saved sp with true return address. */
757 " movq %rax, 19*8(%rsp)\n"
758 RESTORE_REGS_STRING
759 " popfq\n"
760 #else
761 " pushl %esp\n"
762 " pushfl\n"
763 SAVE_REGS_STRING
764 " movl %esp, %eax\n"
765 " call trampoline_handler\n"
766 /* Replace saved sp with true return address. */
767 " movl %eax, 15*4(%esp)\n"
768 RESTORE_REGS_STRING
769 " popfl\n"
770 #endif
771 ASM_RET
772 ".size kretprobe_trampoline, .-kretprobe_trampoline\n"
773 );
774 NOKPROBE_SYMBOL(kretprobe_trampoline);
775 STACK_FRAME_NON_STANDARD(kretprobe_trampoline);
776
777
778 /*
779 * Called from kretprobe_trampoline
780 */
trampoline_handler(struct pt_regs * regs)781 __used __visible void *trampoline_handler(struct pt_regs *regs)
782 {
783 /* fixup registers */
784 regs->cs = __KERNEL_CS;
785 #ifdef CONFIG_X86_32
786 regs->gs = 0;
787 #endif
788 regs->ip = (unsigned long)&kretprobe_trampoline;
789 regs->orig_ax = ~0UL;
790
791 return (void *)kretprobe_trampoline_handler(regs, &kretprobe_trampoline, ®s->sp);
792 }
793 NOKPROBE_SYMBOL(trampoline_handler);
794
795 /*
796 * Called after single-stepping. p->addr is the address of the
797 * instruction whose first byte has been replaced by the "int 3"
798 * instruction. To avoid the SMP problems that can occur when we
799 * temporarily put back the original opcode to single-step, we
800 * single-stepped a copy of the instruction. The address of this
801 * copy is p->ainsn.insn.
802 *
803 * This function prepares to return from the post-single-step
804 * interrupt. We have to fix up the stack as follows:
805 *
806 * 0) Except in the case of absolute or indirect jump or call instructions,
807 * the new ip is relative to the copied instruction. We need to make
808 * it relative to the original instruction.
809 *
810 * 1) If the single-stepped instruction was pushfl, then the TF and IF
811 * flags are set in the just-pushed flags, and may need to be cleared.
812 *
813 * 2) If the single-stepped instruction was a call, the return address
814 * that is atop the stack is the address following the copied instruction.
815 * We need to make it the address following the original instruction.
816 *
817 * If this is the first time we've single-stepped the instruction at
818 * this probepoint, and the instruction is boostable, boost it: add a
819 * jump instruction after the copied instruction, that jumps to the next
820 * instruction after the probepoint.
821 */
resume_execution(struct kprobe * p,struct pt_regs * regs,struct kprobe_ctlblk * kcb)822 static void resume_execution(struct kprobe *p, struct pt_regs *regs,
823 struct kprobe_ctlblk *kcb)
824 {
825 unsigned long *tos = stack_addr(regs);
826 unsigned long copy_ip = (unsigned long)p->ainsn.insn;
827 unsigned long orig_ip = (unsigned long)p->addr;
828 kprobe_opcode_t *insn = p->ainsn.insn;
829
830 /* Skip prefixes */
831 insn = skip_prefixes(insn);
832
833 regs->flags &= ~X86_EFLAGS_TF;
834 switch (*insn) {
835 case 0x9c: /* pushfl */
836 *tos &= ~(X86_EFLAGS_TF | X86_EFLAGS_IF);
837 *tos |= kcb->kprobe_old_flags;
838 break;
839 case 0xc2: /* iret/ret/lret */
840 case 0xc3:
841 case 0xca:
842 case 0xcb:
843 case 0xcf:
844 case 0xea: /* jmp absolute -- ip is correct */
845 /* ip is already adjusted, no more changes required */
846 p->ainsn.boostable = true;
847 goto no_change;
848 case 0xe8: /* call relative - Fix return addr */
849 *tos = orig_ip + (*tos - copy_ip);
850 break;
851 #ifdef CONFIG_X86_32
852 case 0x9a: /* call absolute -- same as call absolute, indirect */
853 *tos = orig_ip + (*tos - copy_ip);
854 goto no_change;
855 #endif
856 case 0xff:
857 if ((insn[1] & 0x30) == 0x10) {
858 /*
859 * call absolute, indirect
860 * Fix return addr; ip is correct.
861 * But this is not boostable
862 */
863 *tos = orig_ip + (*tos - copy_ip);
864 goto no_change;
865 } else if (((insn[1] & 0x31) == 0x20) ||
866 ((insn[1] & 0x31) == 0x21)) {
867 /*
868 * jmp near and far, absolute indirect
869 * ip is correct. And this is boostable
870 */
871 p->ainsn.boostable = true;
872 goto no_change;
873 }
874 default:
875 break;
876 }
877
878 regs->ip += orig_ip - copy_ip;
879
880 no_change:
881 restore_btf();
882 }
883 NOKPROBE_SYMBOL(resume_execution);
884
885 /*
886 * Interrupts are disabled on entry as trap1 is an interrupt gate and they
887 * remain disabled throughout this function.
888 */
kprobe_debug_handler(struct pt_regs * regs)889 int kprobe_debug_handler(struct pt_regs *regs)
890 {
891 struct kprobe *cur = kprobe_running();
892 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
893
894 if (!cur)
895 return 0;
896
897 resume_execution(cur, regs, kcb);
898 regs->flags |= kcb->kprobe_saved_flags;
899
900 if ((kcb->kprobe_status != KPROBE_REENTER) && cur->post_handler) {
901 kcb->kprobe_status = KPROBE_HIT_SSDONE;
902 cur->post_handler(cur, regs, 0);
903 }
904
905 /* Restore back the original saved kprobes variables and continue. */
906 if (kcb->kprobe_status == KPROBE_REENTER) {
907 restore_previous_kprobe(kcb);
908 goto out;
909 }
910 reset_current_kprobe();
911 out:
912 /*
913 * if somebody else is singlestepping across a probe point, flags
914 * will have TF set, in which case, continue the remaining processing
915 * of do_debug, as if this is not a probe hit.
916 */
917 if (regs->flags & X86_EFLAGS_TF)
918 return 0;
919
920 return 1;
921 }
922 NOKPROBE_SYMBOL(kprobe_debug_handler);
923
kprobe_fault_handler(struct pt_regs * regs,int trapnr)924 int kprobe_fault_handler(struct pt_regs *regs, int trapnr)
925 {
926 struct kprobe *cur = kprobe_running();
927 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
928
929 if (unlikely(regs->ip == (unsigned long)cur->ainsn.insn)) {
930 /* This must happen on single-stepping */
931 WARN_ON(kcb->kprobe_status != KPROBE_HIT_SS &&
932 kcb->kprobe_status != KPROBE_REENTER);
933 /*
934 * We are here because the instruction being single
935 * stepped caused a page fault. We reset the current
936 * kprobe and the ip points back to the probe address
937 * and allow the page fault handler to continue as a
938 * normal page fault.
939 */
940 regs->ip = (unsigned long)cur->addr;
941 /*
942 * Trap flag (TF) has been set here because this fault
943 * happened where the single stepping will be done.
944 * So clear it by resetting the current kprobe:
945 */
946 regs->flags &= ~X86_EFLAGS_TF;
947 /*
948 * Since the single step (trap) has been cancelled,
949 * we need to restore BTF here.
950 */
951 restore_btf();
952
953 /*
954 * If the TF flag was set before the kprobe hit,
955 * don't touch it:
956 */
957 regs->flags |= kcb->kprobe_old_flags;
958
959 if (kcb->kprobe_status == KPROBE_REENTER)
960 restore_previous_kprobe(kcb);
961 else
962 reset_current_kprobe();
963 } else if (kcb->kprobe_status == KPROBE_HIT_ACTIVE ||
964 kcb->kprobe_status == KPROBE_HIT_SSDONE) {
965 /*
966 * We increment the nmissed count for accounting,
967 * we can also use npre/npostfault count for accounting
968 * these specific fault cases.
969 */
970 kprobes_inc_nmissed_count(cur);
971
972 /*
973 * We come here because instructions in the pre/post
974 * handler caused the page_fault, this could happen
975 * if handler tries to access user space by
976 * copy_from_user(), get_user() etc. Let the
977 * user-specified handler try to fix it first.
978 */
979 if (cur->fault_handler && cur->fault_handler(cur, regs, trapnr))
980 return 1;
981 }
982
983 return 0;
984 }
985 NOKPROBE_SYMBOL(kprobe_fault_handler);
986
arch_populate_kprobe_blacklist(void)987 int __init arch_populate_kprobe_blacklist(void)
988 {
989 return kprobe_add_area_blacklist((unsigned long)__entry_text_start,
990 (unsigned long)__entry_text_end);
991 }
992
arch_init_kprobes(void)993 int __init arch_init_kprobes(void)
994 {
995 return 0;
996 }
997
arch_trampoline_kprobe(struct kprobe * p)998 int arch_trampoline_kprobe(struct kprobe *p)
999 {
1000 return 0;
1001 }
1002