1 // SPDX-License-Identifier: GPL-2.0-or-later
24 #include <asm/text-patching.h>
33 #include <asm/nospec-branch.h>
45 		kp = get_kprobe((void *)addr - i);  in __recover_optprobed_insn()
46 		/* This function only handles jump-optimized kprobe */  in __recover_optprobed_insn()
50 			if (list_empty(&op->list) || optprobe_queued_unopt(op))  in __recover_optprobed_insn()
60 	 * bytes must be recovered from op->optinsn.copied_insn buffer.  in __recover_optprobed_insn()
66 	if (addr == (unsigned long)kp->addr) {  in __recover_optprobed_insn()
67 		buf[0] = kp->opcode;  in __recover_optprobed_insn()
68 		memcpy(buf + 1, op->optinsn.copied_insn, DISP32_SIZE);  in __recover_optprobed_insn()
70 		offs = addr - (unsigned long)kp->addr - 1;  in __recover_optprobed_insn()
71 		memcpy(buf, op->optinsn.copied_insn + offs, DISP32_SIZE - offs);  in __recover_optprobed_insn()
107 			".global optprobe_template_entry\n"
113 			".global optprobe_template_clac\n"
118 			".global optprobe_template_val\n"
122 			".global optprobe_template_call\n"
135 			".global optprobe_template_clac\n"
140 			".global optprobe_template_val\n"
143 			".global optprobe_template_call\n"
154 			".global optprobe_template_end\n"
162 	((long)optprobe_template_clac - (long)optprobe_template_entry)
164 	((long)optprobe_template_val - (long)optprobe_template_entry)
166 	((long)optprobe_template_call - (long)optprobe_template_entry)
168 	((long)optprobe_template_end - (long)optprobe_template_entry)
172 optimized_callback(struct optimized_kprobe *op, struct pt_regs *regs)  in optimized_callback()  argument
175 	if (kprobe_disabled(&op->kp))  in optimized_callback()
180 		kprobes_inc_nmissed_count(&op->kp);  in optimized_callback()
184 		regs->cs = __KERNEL_CS;  in optimized_callback()
186 		regs->gs = 0;  in optimized_callback()
188 		regs->ip = (unsigned long)op->kp.addr + INT3_INSN_SIZE;  in optimized_callback()
189 		regs->orig_ax = ~0UL;  in optimized_callback()
191 		__this_cpu_write(current_kprobe, &op->kp);  in optimized_callback()
192 		kcb->kprobe_status = KPROBE_HIT_ACTIVE;  in optimized_callback()
193 		opt_pre_handler(&op->kp, regs);  in optimized_callback()
208 			return -EINVAL;  in copy_optimized_instructions()
212 	if (ftrace_text_reserved(src, src + len - 1) ||  in copy_optimized_instructions()
213 	    alternatives_text_reserved(src, src + len - 1) ||  in copy_optimized_instructions()
214 	    jump_label_text_reserved(src, src + len - 1) ||  in copy_optimized_instructions()
215 	    static_call_text_reserved(src, src + len - 1))  in copy_optimized_instructions()
216 		return -EBUSY;  in copy_optimized_instructions()
224 	return ((insn->opcode.bytes[0] == 0xff &&  in __insn_is_indirect_jump()
225 		(X86_MODRM_REG(insn->modrm.value) & 6) == 4) || /* Jump */  in __insn_is_indirect_jump()
226 		insn->opcode.bytes[0] == 0xea);	/* Segment based jump */  in __insn_is_indirect_jump()
234 	switch (insn->opcode.bytes[0]) {  in insn_jump_into_range()
243 		if ((insn->opcode.bytes[1] & 0xf0) == 0x80) /* jcc near */  in insn_jump_into_range()
247 		if ((insn->opcode.bytes[0] & 0xf0) == 0x70) /* jcc short */  in insn_jump_into_range()
251 	target = (unsigned long)insn->next_byte + insn->immediate.value;  in insn_jump_into_range()
265 	 * replace indirect-jump check.  in insn_is_indirect_jump()
270 				(unsigned long)__indirect_thunk_end -  in insn_is_indirect_jump()
296 	if (size - offset < JMP32_INSN_SIZE)  in can_optimize()
300 	addr = paddr - offset;  in can_optimize()
301 	while (addr < paddr - offset + size) { /* Decode until function end */  in can_optimize()
347 	for (i = 1; i < op->optinsn.size; i++) {  in arch_check_optimized_kprobe()
348 		p = get_kprobe(op->kp.addr + i);  in arch_check_optimized_kprobe()
350 			return -EEXIST;  in arch_check_optimized_kprobe()
360 	return ((unsigned long)op->kp.addr <= addr &&  in arch_within_optimized_kprobe()
361 		(unsigned long)op->kp.addr + op->optinsn.size > addr);  in arch_within_optimized_kprobe()
368 	u8 *slot = op->optinsn.insn;  in __arch_remove_optimized_kprobe()
370 		int len = TMPL_END_IDX + op->optinsn.size + JMP32_INSN_SIZE;  in __arch_remove_optimized_kprobe()
377 		op->optinsn.insn = NULL;  in __arch_remove_optimized_kprobe()
378 		op->optinsn.size = 0;  in __arch_remove_optimized_kprobe()
399 	if (!can_optimize((unsigned long)op->kp.addr))  in arch_prepare_optimized_kprobe()
400 		return -EILSEQ;  in arch_prepare_optimized_kprobe()
404 		return -ENOMEM;  in arch_prepare_optimized_kprobe()
406 	op->optinsn.insn = slot = get_optinsn_slot();  in arch_prepare_optimized_kprobe()
408 		ret = -ENOMEM;  in arch_prepare_optimized_kprobe()
416 	rel = (long)slot - (long)op->kp.addr + JMP32_INSN_SIZE;  in arch_prepare_optimized_kprobe()
418 		ret = -ERANGE;  in arch_prepare_optimized_kprobe()
422 	/* Copy arch-dep-instance from template */  in arch_prepare_optimized_kprobe()
425 	/* Copy instructions into the out-of-line buffer */  in arch_prepare_optimized_kprobe()
426 	ret = copy_optimized_instructions(buf + TMPL_END_IDX, op->kp.addr,  in arch_prepare_optimized_kprobe()
430 	op->optinsn.size = ret;  in arch_prepare_optimized_kprobe()
431 	len = TMPL_END_IDX + op->optinsn.size;  in arch_prepare_optimized_kprobe()
442 	/* Set returning jmp instruction at the tail of out-of-line buffer */  in arch_prepare_optimized_kprobe()
444 			   (u8 *)op->kp.addr + op->optinsn.size);  in arch_prepare_optimized_kprobe()
448 	 * Note	len = TMPL_END_IDX + op->optinsn.size + JMP32_INSN_SIZE is also  in arch_prepare_optimized_kprobe()
480 		s32 rel = (s32)((long)op->optinsn.insn -  in arch_optimize_kprobes()
481 			((long)op->kp.addr + JMP32_INSN_SIZE));  in arch_optimize_kprobes()
483 		WARN_ON(kprobe_disabled(&op->kp));  in arch_optimize_kprobes()
486 		memcpy(op->optinsn.copied_insn, op->kp.addr + INT3_INSN_SIZE,  in arch_optimize_kprobes()
492 		text_poke_bp(op->kp.addr, insn_buff, JMP32_INSN_SIZE, NULL);  in arch_optimize_kprobes()
494 		list_del_init(&op->list);  in arch_optimize_kprobes()
509 	u8 *addr = op->kp.addr;  in arch_unoptimize_kprobe()
511 	memcpy(old, op->kp.addr, JMP32_INSN_SIZE);  in arch_unoptimize_kprobe()
513 	       op->optinsn.copied_insn,  in arch_unoptimize_kprobe()
514 	       JMP32_INSN_SIZE - INT3_INSN_SIZE);  in arch_unoptimize_kprobe()
520 		  JMP32_INSN_SIZE - INT3_INSN_SIZE);  in arch_unoptimize_kprobe()
523 	perf_event_text_poke(op->kp.addr, old, JMP32_INSN_SIZE, new, JMP32_INSN_SIZE);  in arch_unoptimize_kprobe()
537 		list_move(&op->list, done_list);  in arch_unoptimize_kprobes()
541 int setup_detour_execution(struct kprobe *p, struct pt_regs *regs, int reenter)  in setup_detour_execution()  argument
545 	if (p->flags & KPROBE_FLAG_OPTIMIZED) {  in setup_detour_execution()
549 		regs->ip = (unsigned long)op->optinsn.insn + TMPL_END_IDX;  in setup_detour_execution()