Lines Matching +full:0 +full:x48
34 do { prog = emit_code(prog, bytes, len); cnt += len; } while (0)
42 do { EMIT1(b1); EMIT(off, 4); } while (0)
44 do { EMIT2(b1, b2); EMIT(off, 4); } while (0)
46 do { EMIT3(b1, b2, b3); EMIT(off, 4); } while (0)
48 do { EMIT4(b1, b2, b3, b4); EMIT(off, 4); } while (0)
69 EMIT3(add_2mod(0x48, DST, SRC), 0x89, add_2reg(0xC0, DST, SRC)); \
70 } while (0)
83 return 0; in bpf_size_to_x86_bytes()
88 * Add 0x10 (and an extra 0x0f) to generate far jumps (. + s32)
90 #define X86_JB 0x72
91 #define X86_JAE 0x73
92 #define X86_JE 0x74
93 #define X86_JNE 0x75
94 #define X86_JBE 0x76
95 #define X86_JA 0x77
96 #define X86_JL 0x7C
97 #define X86_JGE 0x7D
98 #define X86_JLE 0x7E
99 #define X86_JG 0x7F
115 [BPF_REG_0] = 0, /* RAX */
120 [BPF_REG_5] = 0, /* R8 */
195 memset(area, 0xcc, size); in jit_fill_hole()
215 int cnt = 0; in emit_prologue()
217 EMIT1(0x55); /* push rbp */ in emit_prologue()
218 EMIT3(0x48, 0x89, 0xE5); /* mov rbp, rsp */ in emit_prologue()
220 EMIT3_off32(0x48, 0x81, 0xEC, round_up(stack_depth, 8)); in emit_prologue()
221 EMIT1(0x53); /* push rbx */ in emit_prologue()
222 EMIT2(0x41, 0x55); /* push r13 */ in emit_prologue()
223 EMIT2(0x41, 0x56); /* push r14 */ in emit_prologue()
224 EMIT2(0x41, 0x57); /* push r15 */ in emit_prologue()
227 EMIT2(0x6a, 0x00); in emit_prologue()
251 int cnt = 0; in emit_bpf_tail_call()
263 EMIT2(0x89, 0xD2); /* mov edx, edx */ in emit_bpf_tail_call()
264 EMIT3(0x39, 0x56, /* cmp dword ptr [rsi + 16], edx */ in emit_bpf_tail_call()
274 EMIT2_off32(0x8B, 0x85, -36 - MAX_BPF_STACK); /* mov eax, dword ptr [rbp - 548] */ in emit_bpf_tail_call()
275 EMIT3(0x83, 0xF8, MAX_TAIL_CALL_CNT); /* cmp eax, MAX_TAIL_CALL_CNT */ in emit_bpf_tail_call()
279 EMIT3(0x83, 0xC0, 0x01); /* add eax, 1 */ in emit_bpf_tail_call()
280 EMIT2_off32(0x89, 0x85, -36 - MAX_BPF_STACK); /* mov dword ptr [rbp -548], eax */ in emit_bpf_tail_call()
283 EMIT4_off32(0x48, 0x8B, 0x84, 0xD6, /* mov rax, [rsi + rdx * 8 + offsetof(...)] */ in emit_bpf_tail_call()
290 EMIT3(0x48, 0x85, 0xC0); /* test rax,rax */ in emit_bpf_tail_call()
296 EMIT4(0x48, 0x8B, 0x40, /* mov rax, qword ptr [rax + 32] */ in emit_bpf_tail_call()
298 EMIT4(0x48, 0x83, 0xC0, PROLOGUE_SIZE); /* add rax, prologue_size */ in emit_bpf_tail_call()
319 int cnt = 0; in emit_mov_imm32()
325 if (sign_propagate && (s32)imm32 < 0) { in emit_mov_imm32()
327 b1 = add_1mod(0x48, dst_reg); in emit_mov_imm32()
328 b2 = 0xC7; in emit_mov_imm32()
329 b3 = 0xC0; in emit_mov_imm32()
338 if (imm32 == 0) { in emit_mov_imm32()
340 EMIT1(add_2mod(0x40, dst_reg, dst_reg)); in emit_mov_imm32()
341 b2 = 0x31; /* xor */ in emit_mov_imm32()
342 b3 = 0xC0; in emit_mov_imm32()
349 EMIT1(add_1mod(0x40, dst_reg)); in emit_mov_imm32()
350 EMIT1_off32(add_1reg(0xB8, dst_reg), imm32); in emit_mov_imm32()
359 int cnt = 0; in emit_mov_imm64()
371 EMIT2(add_1mod(0x48, dst_reg), add_1reg(0xB8, dst_reg)); in emit_mov_imm64()
382 int cnt = 0; in emit_mov_reg()
390 EMIT1(add_2mod(0x40, dst_reg, src_reg)); in emit_mov_reg()
391 EMIT2(0x89, add_2reg(0xC0, dst_reg, src_reg)); in emit_mov_reg()
404 int i, cnt = 0; in do_jit()
405 int proglen = 0; in do_jit()
411 for (i = 0; i < insn_cnt; i++, insn++) { in do_jit()
415 u8 b2 = 0, b3 = 0; in do_jit()
434 case BPF_ADD: b2 = 0x01; break; in do_jit()
435 case BPF_SUB: b2 = 0x29; break; in do_jit()
436 case BPF_AND: b2 = 0x21; break; in do_jit()
437 case BPF_OR: b2 = 0x09; break; in do_jit()
438 case BPF_XOR: b2 = 0x31; break; in do_jit()
441 EMIT1(add_2mod(0x48, dst_reg, src_reg)); in do_jit()
443 EMIT1(add_2mod(0x40, dst_reg, src_reg)); in do_jit()
444 EMIT2(b2, add_2reg(0xC0, dst_reg, src_reg)); in do_jit()
458 EMIT1(add_1mod(0x48, dst_reg)); in do_jit()
460 EMIT1(add_1mod(0x40, dst_reg)); in do_jit()
461 EMIT2(0xF7, add_1reg(0xD8, dst_reg)); in do_jit()
475 EMIT1(add_1mod(0x48, dst_reg)); in do_jit()
477 EMIT1(add_1mod(0x40, dst_reg)); in do_jit()
485 b3 = 0xC0; in do_jit()
486 b2 = 0x05; in do_jit()
489 b3 = 0xE8; in do_jit()
490 b2 = 0x2D; in do_jit()
493 b3 = 0xE0; in do_jit()
494 b2 = 0x25; in do_jit()
497 b3 = 0xC8; in do_jit()
498 b2 = 0x0D; in do_jit()
501 b3 = 0xF0; in do_jit()
502 b2 = 0x35; in do_jit()
507 EMIT3(0x83, add_1reg(b3, dst_reg), imm32); in do_jit()
511 EMIT2_off32(0x81, add_1reg(b3, dst_reg), imm32); in do_jit()
521 emit_mov_imm64(&prog, dst_reg, insn[1].imm, insn[0].imm); in do_jit()
535 EMIT1(0x50); /* push rax */ in do_jit()
536 EMIT1(0x52); /* push rdx */ in do_jit()
543 EMIT3_off32(0x49, 0xC7, 0xC3, imm32); in do_jit()
552 EMIT2(0x31, 0xd2); in do_jit()
556 EMIT3(0x49, 0xF7, 0xF3); in do_jit()
559 EMIT3(0x41, 0xF7, 0xF3); in do_jit()
563 EMIT3(0x49, 0x89, 0xD3); in do_jit()
566 EMIT3(0x49, 0x89, 0xC3); in do_jit()
568 EMIT1(0x5A); /* pop rdx */ in do_jit()
569 EMIT1(0x58); /* pop rax */ in do_jit()
583 EMIT1(0x50); /* push rax */ in do_jit()
585 EMIT1(0x52); /* push rdx */ in do_jit()
596 EMIT1(add_1mod(0x48, AUX_REG)); in do_jit()
598 EMIT1(add_1mod(0x40, AUX_REG)); in do_jit()
600 EMIT2(0xF7, add_1reg(0xE0, AUX_REG)); in do_jit()
603 EMIT1(0x5A); /* pop rdx */ in do_jit()
607 EMIT1(0x58); /* pop rax */ in do_jit()
619 EMIT1(add_1mod(0x48, dst_reg)); in do_jit()
621 EMIT1(add_1mod(0x40, dst_reg)); in do_jit()
624 case BPF_LSH: b3 = 0xE0; break; in do_jit()
625 case BPF_RSH: b3 = 0xE8; break; in do_jit()
626 case BPF_ARSH: b3 = 0xF8; break; in do_jit()
630 EMIT2(0xD1, add_1reg(b3, dst_reg)); in do_jit()
632 EMIT3(0xC1, add_1reg(b3, dst_reg), imm32); in do_jit()
650 EMIT1(0x51); /* push rcx */ in do_jit()
658 EMIT1(add_1mod(0x48, dst_reg)); in do_jit()
660 EMIT1(add_1mod(0x40, dst_reg)); in do_jit()
663 case BPF_LSH: b3 = 0xE0; break; in do_jit()
664 case BPF_RSH: b3 = 0xE8; break; in do_jit()
665 case BPF_ARSH: b3 = 0xF8; break; in do_jit()
667 EMIT2(0xD3, add_1reg(b3, dst_reg)); in do_jit()
670 EMIT1(0x59); /* pop rcx */ in do_jit()
681 EMIT1(0x66); in do_jit()
683 EMIT1(0x41); in do_jit()
684 EMIT3(0xC1, add_1reg(0xC8, dst_reg), 8); in do_jit()
688 EMIT3(0x45, 0x0F, 0xB7); in do_jit()
690 EMIT2(0x0F, 0xB7); in do_jit()
691 EMIT1(add_2reg(0xC0, dst_reg, dst_reg)); in do_jit()
696 EMIT2(0x41, 0x0F); in do_jit()
698 EMIT1(0x0F); in do_jit()
699 EMIT1(add_1reg(0xC8, dst_reg)); in do_jit()
703 EMIT3(add_1mod(0x48, dst_reg), 0x0F, in do_jit()
704 add_1reg(0xC8, dst_reg)); in do_jit()
717 EMIT3(0x45, 0x0F, 0xB7); in do_jit()
719 EMIT2(0x0F, 0xB7); in do_jit()
720 EMIT1(add_2reg(0xC0, dst_reg, dst_reg)); in do_jit()
725 EMIT1(0x45); in do_jit()
726 EMIT2(0x89, add_2reg(0xC0, dst_reg, dst_reg)); in do_jit()
738 EMIT3(0x0F, 0xAE, 0xE8); in do_jit()
744 EMIT2(0x41, 0xC6); in do_jit()
746 EMIT1(0xC6); in do_jit()
750 EMIT3(0x66, 0x41, 0xC7); in do_jit()
752 EMIT2(0x66, 0xC7); in do_jit()
756 EMIT2(0x41, 0xC7); in do_jit()
758 EMIT1(0xC7); in do_jit()
761 EMIT2(add_1mod(0x48, dst_reg), 0xC7); in do_jit()
764 EMIT2(add_1reg(0x40, dst_reg), insn->off); in do_jit()
766 EMIT1_off32(add_1reg(0x80, dst_reg), insn->off); in do_jit()
776 EMIT2(add_2mod(0x40, dst_reg, src_reg), 0x88); in do_jit()
778 EMIT1(0x88); in do_jit()
782 EMIT3(0x66, add_2mod(0x40, dst_reg, src_reg), 0x89); in do_jit()
784 EMIT2(0x66, 0x89); in do_jit()
788 EMIT2(add_2mod(0x40, dst_reg, src_reg), 0x89); in do_jit()
790 EMIT1(0x89); in do_jit()
793 EMIT2(add_2mod(0x48, dst_reg, src_reg), 0x89); in do_jit()
795 EMIT2(add_2reg(0x40, dst_reg, src_reg), insn->off); in do_jit()
797 EMIT1_off32(add_2reg(0x80, dst_reg, src_reg), in do_jit()
804 EMIT3(add_2mod(0x48, src_reg, dst_reg), 0x0F, 0xB6); in do_jit()
808 EMIT3(add_2mod(0x48, src_reg, dst_reg), 0x0F, 0xB7); in do_jit()
811 /* Emit 'mov eax, dword ptr [rax+0x14]' */ in do_jit()
813 EMIT2(add_2mod(0x40, src_reg, dst_reg), 0x8B); in do_jit()
815 EMIT1(0x8B); in do_jit()
818 /* Emit 'mov rax, qword ptr [rax+0x14]' */ in do_jit()
819 EMIT2(add_2mod(0x48, src_reg, dst_reg), 0x8B); in do_jit()
821 * If insn->off == 0 we can save one extra byte, but in do_jit()
826 EMIT2(add_2reg(0x40, src_reg, dst_reg), insn->off); in do_jit()
828 EMIT1_off32(add_2reg(0x80, src_reg, dst_reg), in do_jit()
836 EMIT3(0xF0, add_2mod(0x40, dst_reg, src_reg), 0x01); in do_jit()
838 EMIT2(0xF0, 0x01); in do_jit()
841 EMIT3(0xF0, add_2mod(0x48, dst_reg, src_reg), 0x01); in do_jit()
843 EMIT2(add_2reg(0x40, dst_reg, src_reg), insn->off); in do_jit()
845 EMIT1_off32(add_2reg(0x80, dst_reg, src_reg), in do_jit()
858 EMIT1_off32(0xE8, jmp_offset); in do_jit()
888 EMIT1(add_2mod(0x48, dst_reg, src_reg)); in do_jit()
890 EMIT1(add_2mod(0x40, dst_reg, src_reg)); in do_jit()
891 EMIT2(0x39, add_2reg(0xC0, dst_reg, src_reg)); in do_jit()
898 EMIT1(add_2mod(0x48, dst_reg, src_reg)); in do_jit()
900 EMIT1(add_2mod(0x40, dst_reg, src_reg)); in do_jit()
901 EMIT2(0x85, add_2reg(0xC0, dst_reg, src_reg)); in do_jit()
908 EMIT1(add_1mod(0x48, dst_reg)); in do_jit()
910 EMIT1(add_1mod(0x40, dst_reg)); in do_jit()
911 EMIT2_off32(0xF7, add_1reg(0xC0, dst_reg), imm32); in do_jit()
936 EMIT1(add_1mod(0x48, dst_reg)); in do_jit()
938 EMIT1(add_1mod(0x40, dst_reg)); in do_jit()
941 EMIT3(0x83, add_1reg(0xF8, dst_reg), imm32); in do_jit()
943 EMIT2_off32(0x81, add_1reg(0xF8, dst_reg), imm32); in do_jit()
993 EMIT2_off32(0x0F, jmp_cond + 0x10, jmp_offset); in do_jit()
1018 EMIT2(0xEB, jmp_offset); in do_jit()
1020 EMIT1_off32(0xE9, jmp_offset); in do_jit()
1036 EMIT1(0x5B); /* get rid of tail_call_cnt */ in do_jit()
1037 EMIT2(0x41, 0x5F); /* pop r15 */ in do_jit()
1038 EMIT2(0x41, 0x5E); /* pop r14 */ in do_jit()
1039 EMIT2(0x41, 0x5D); /* pop r13 */ in do_jit()
1040 EMIT1(0x5B); /* pop rbx */ in do_jit()
1041 EMIT1(0xC9); /* leave */ in do_jit()
1042 EMIT1(0xC3); /* ret */ in do_jit()
1098 int proglen, oldproglen = 0; in bpf_int_jit_compile()
1150 for (proglen = 0, i = 0; i < prog->len; i++) { in bpf_int_jit_compile()
1163 for (pass = 0; pass < 20 || image; pass++) { in bpf_int_jit_compile()
1165 if (proglen <= 0) { in bpf_int_jit_compile()