Lines Matching full:emit
243 * Emit an instruction that will be executed unconditionally.
245 static inline void emit(u32 inst, struct jit_ctx *ctx) in emit() function
429 emit(ARM_LDR_I(rd, ARM_PC, imm_offset(val, ctx)), ctx); in emit_mov_i_no8m()
431 emit(ARM_MOVW(rd, val & 0xffff), ctx); in emit_mov_i_no8m()
433 emit(ARM_MOVT(rd, val >> 16), ctx); in emit_mov_i_no8m()
442 emit(ARM_MOV_I(rd, imm12), ctx); in emit_mov_i()
450 emit(ARM_BX(tgt_reg), ctx); in emit_bx_r()
452 emit(ARM_MOV_R(ARM_PC, tgt_reg), ctx); in emit_bx_r()
458 emit(ARM_MOV_R(ARM_LR, ARM_PC), ctx); in emit_blx_r()
461 emit(ARM_BLX_R(tgt_reg), ctx); in emit_blx_r()
485 emit(ARM_UDIV(rd, rm, rn), ctx); in emit_udivmod()
487 emit(ARM_UDIV(ARM_IP, rm, rn), ctx); in emit_udivmod()
488 emit(ARM_MLS(rd, rn, ARM_IP, rm), ctx); in emit_udivmod()
503 emit(ARM_MOV_R(tmp[0], ARM_R1), ctx); in emit_udivmod()
504 emit(ARM_MOV_R(ARM_R1, rn), ctx); in emit_udivmod()
507 emit(ARM_MOV_R(tmp[1], ARM_R0), ctx); in emit_udivmod()
508 emit(ARM_MOV_R(ARM_R0, rm), ctx); in emit_udivmod()
512 emit(ARM_PUSH(CALLER_MASK & ~exclude_mask), ctx); in emit_udivmod()
520 emit(ARM_POP(CALLER_MASK & ~exclude_mask), ctx); in emit_udivmod()
524 emit(ARM_MOV_R(rd, ARM_R0), ctx); in emit_udivmod()
528 emit(ARM_MOV_R(ARM_R1, tmp[0]), ctx); in emit_udivmod()
530 emit(ARM_MOV_R(ARM_R0, tmp[1]), ctx); in emit_udivmod()
546 emit(ARM_LDR_I(tmp, ARM_FP, EBPF_SCRATCH_TO_ARM_FP(reg)), ctx); in arm_bpf_get_reg32()
558 emit(ARM_LDRD_I(tmp[1], ARM_FP, in arm_bpf_get_reg64()
561 emit(ARM_LDR_I(tmp[1], ARM_FP, in arm_bpf_get_reg64()
563 emit(ARM_LDR_I(tmp[0], ARM_FP, in arm_bpf_get_reg64()
578 emit(ARM_STR_I(src, ARM_FP, EBPF_SCRATCH_TO_ARM_FP(reg)), ctx); in arm_bpf_put_reg32()
580 emit(ARM_MOV_R(reg, src), ctx); in arm_bpf_put_reg32()
589 emit(ARM_STRD_I(src[1], ARM_FP, in arm_bpf_put_reg64()
592 emit(ARM_STR_I(src[1], ARM_FP, in arm_bpf_put_reg64()
594 emit(ARM_STR_I(src[0], ARM_FP, in arm_bpf_put_reg64()
599 emit(ARM_MOV_R(reg[1], src[1]), ctx); in arm_bpf_put_reg64()
601 emit(ARM_MOV_R(reg[0], src[0]), ctx); in arm_bpf_put_reg64()
649 emit(ARM_ADDS_R(dst, dst, src), ctx); in emit_a32_add_r()
651 emit(ARM_ADC_R(dst, dst, src), ctx); in emit_a32_add_r()
653 emit(ARM_ADD_R(dst, dst, src), ctx); in emit_a32_add_r()
666 emit(ARM_SUBS_R(dst, dst, src), ctx); in emit_a32_sub_r()
668 emit(ARM_SBC_R(dst, dst, src), ctx); in emit_a32_sub_r()
670 emit(ARM_SUB_R(dst, dst, src), ctx); in emit_a32_sub_r()
686 emit(ARM_ORR_R(dst, dst, src), ctx); in emit_alu_r()
690 emit(ARM_AND_R(dst, dst, src), ctx); in emit_alu_r()
694 emit(ARM_EOR_R(dst, dst, src), ctx); in emit_alu_r()
698 emit(ARM_MUL(dst, dst, src), ctx); in emit_alu_r()
702 emit(ARM_LSL_R(dst, dst, src), ctx); in emit_alu_r()
706 emit(ARM_LSR_R(dst, dst, src), ctx); in emit_alu_r()
710 emit(ARM_MOV_SR(dst, dst, SRTYPE_ASR, src), ctx); in emit_alu_r()
789 emit(ARM_LDRD_I(tmp[1], ARM_FP, EBPF_SCRATCH_TO_ARM_FP(src_lo)), ctx); in emit_a32_mov_r64()
790 emit(ARM_STRD_I(tmp[1], ARM_FP, EBPF_SCRATCH_TO_ARM_FP(dst_lo)), ctx); in emit_a32_mov_r64()
792 emit(ARM_LDRD_I(dst[1], ARM_FP, EBPF_SCRATCH_TO_ARM_FP(src_lo)), ctx); in emit_a32_mov_r64()
794 emit(ARM_STRD_I(src[1], ARM_FP, EBPF_SCRATCH_TO_ARM_FP(dst_lo)), ctx); in emit_a32_mov_r64()
796 emit(ARM_MOV_R(dst[0], src[0]), ctx); in emit_a32_mov_r64()
797 emit(ARM_MOV_R(dst[1], src[1]), ctx); in emit_a32_mov_r64()
812 emit(ARM_LSL_I(rd, rd, val), ctx); in emit_a32_alu_i()
815 emit(ARM_LSR_I(rd, rd, val), ctx); in emit_a32_alu_i()
818 emit(ARM_ASR_I(rd, rd, val), ctx); in emit_a32_alu_i()
821 emit(ARM_RSB_I(rd, rd, val), ctx); in emit_a32_alu_i()
838 emit(ARM_RSBS_I(rd[1], rd[1], 0), ctx); in emit_a32_neg64()
839 emit(ARM_RSC_I(rd[0], rd[0], 0), ctx); in emit_a32_neg64()
857 emit(ARM_SUB_I(ARM_IP, rt, 32), ctx); in emit_a32_lsh_r64()
858 emit(ARM_RSB_I(tmp2[0], rt, 32), ctx); in emit_a32_lsh_r64()
859 emit(ARM_MOV_SR(ARM_LR, rd[0], SRTYPE_ASL, rt), ctx); in emit_a32_lsh_r64()
860 emit(ARM_ORR_SR(ARM_LR, ARM_LR, rd[1], SRTYPE_ASL, ARM_IP), ctx); in emit_a32_lsh_r64()
861 emit(ARM_ORR_SR(ARM_IP, ARM_LR, rd[1], SRTYPE_LSR, tmp2[0]), ctx); in emit_a32_lsh_r64()
862 emit(ARM_MOV_SR(ARM_LR, rd[1], SRTYPE_ASL, rt), ctx); in emit_a32_lsh_r64()
881 emit(ARM_RSB_I(ARM_IP, rt, 32), ctx); in emit_a32_arsh_r64()
882 emit(ARM_SUBS_I(tmp2[0], rt, 32), ctx); in emit_a32_arsh_r64()
883 emit(ARM_MOV_SR(ARM_LR, rd[1], SRTYPE_LSR, rt), ctx); in emit_a32_arsh_r64()
884 emit(ARM_ORR_SR(ARM_LR, ARM_LR, rd[0], SRTYPE_ASL, ARM_IP), ctx); in emit_a32_arsh_r64()
887 emit(ARM_MOV_SR(ARM_IP, rd[0], SRTYPE_ASR, rt), ctx); in emit_a32_arsh_r64()
906 emit(ARM_RSB_I(ARM_IP, rt, 32), ctx); in emit_a32_rsh_r64()
907 emit(ARM_SUBS_I(tmp2[0], rt, 32), ctx); in emit_a32_rsh_r64()
908 emit(ARM_MOV_SR(ARM_LR, rd[1], SRTYPE_LSR, rt), ctx); in emit_a32_rsh_r64()
909 emit(ARM_ORR_SR(ARM_LR, ARM_LR, rd[0], SRTYPE_ASL, ARM_IP), ctx); in emit_a32_rsh_r64()
910 emit(ARM_ORR_SR(ARM_LR, ARM_LR, rd[0], SRTYPE_LSR, tmp2[0]), ctx); in emit_a32_rsh_r64()
911 emit(ARM_MOV_SR(ARM_IP, rd[0], SRTYPE_LSR, rt), ctx); in emit_a32_rsh_r64()
929 emit(ARM_MOV_SI(tmp2[0], rd[0], SRTYPE_ASL, val), ctx); in emit_a32_lsh_i64()
930 emit(ARM_ORR_SI(rd[0], tmp2[0], rd[1], SRTYPE_LSR, 32 - val), ctx); in emit_a32_lsh_i64()
931 emit(ARM_MOV_SI(rd[1], rd[1], SRTYPE_ASL, val), ctx); in emit_a32_lsh_i64()
934 emit(ARM_MOV_R(rd[0], rd[1]), ctx); in emit_a32_lsh_i64()
936 emit(ARM_MOV_SI(rd[0], rd[1], SRTYPE_ASL, val - 32), ctx); in emit_a32_lsh_i64()
937 emit(ARM_EOR_R(rd[1], rd[1], rd[1]), ctx); in emit_a32_lsh_i64()
959 emit(ARM_MOV_SI(tmp2[1], rd[1], SRTYPE_LSR, val), ctx); in emit_a32_rsh_i64()
960 emit(ARM_ORR_SI(rd[1], tmp2[1], rd[0], SRTYPE_ASL, 32 - val), ctx); in emit_a32_rsh_i64()
961 emit(ARM_MOV_SI(rd[0], rd[0], SRTYPE_LSR, val), ctx); in emit_a32_rsh_i64()
963 emit(ARM_MOV_R(rd[1], rd[0]), ctx); in emit_a32_rsh_i64()
964 emit(ARM_MOV_I(rd[0], 0), ctx); in emit_a32_rsh_i64()
966 emit(ARM_MOV_SI(rd[1], rd[0], SRTYPE_LSR, val - 32), ctx); in emit_a32_rsh_i64()
967 emit(ARM_MOV_I(rd[0], 0), ctx); in emit_a32_rsh_i64()
989 emit(ARM_MOV_SI(tmp2[1], rd[1], SRTYPE_LSR, val), ctx); in emit_a32_arsh_i64()
990 emit(ARM_ORR_SI(rd[1], tmp2[1], rd[0], SRTYPE_ASL, 32 - val), ctx); in emit_a32_arsh_i64()
991 emit(ARM_MOV_SI(rd[0], rd[0], SRTYPE_ASR, val), ctx); in emit_a32_arsh_i64()
993 emit(ARM_MOV_R(rd[1], rd[0]), ctx); in emit_a32_arsh_i64()
994 emit(ARM_MOV_SI(rd[0], rd[0], SRTYPE_ASR, 31), ctx); in emit_a32_arsh_i64()
996 emit(ARM_MOV_SI(rd[1], rd[0], SRTYPE_ASR, val - 32), ctx); in emit_a32_arsh_i64()
997 emit(ARM_MOV_SI(rd[0], rd[0], SRTYPE_ASR, 31), ctx); in emit_a32_arsh_i64()
1014 emit(ARM_MUL(ARM_IP, rd[1], rt[0]), ctx); in emit_a32_mul_r64()
1015 emit(ARM_MUL(ARM_LR, rd[0], rt[1]), ctx); in emit_a32_mul_r64()
1016 emit(ARM_ADD_R(ARM_LR, ARM_IP, ARM_LR), ctx); in emit_a32_mul_r64()
1018 emit(ARM_UMULL(ARM_IP, rd[0], rd[1], rt[1]), ctx); in emit_a32_mul_r64()
1019 emit(ARM_ADD_R(rd[0], ARM_LR, rd[0]), ctx); in emit_a32_mul_r64()
1055 emit(ARM_ADD_R(tmp[0], tmp[0], rd), ctx); in emit_str_r()
1062 emit(ARM_STRB_I(src_lo, rd, off), ctx); in emit_str_r()
1066 emit(ARM_STRH_I(src_lo, rd, off), ctx); in emit_str_r()
1070 emit(ARM_STR_I(src_lo, rd, off), ctx); in emit_str_r()
1074 emit(ARM_STR_I(src_lo, rd, off), ctx); in emit_str_r()
1075 emit(ARM_STR_I(src_hi, rd, off + 4), ctx); in emit_str_r()
1089 emit(ARM_ADD_R(tmp[0], tmp[0], src), ctx); in emit_ldx_r()
1093 emit(ARM_MOV_R(tmp[0], rm), ctx); in emit_ldx_r()
1099 emit(ARM_LDRB_I(rd[1], rm, off), ctx); in emit_ldx_r()
1105 emit(ARM_LDRH_I(rd[1], rm, off), ctx); in emit_ldx_r()
1111 emit(ARM_LDR_I(rd[1], rm, off), ctx); in emit_ldx_r()
1117 emit(ARM_LDR_I(rd[1], rm, off), ctx); in emit_ldx_r()
1118 emit(ARM_LDR_I(rd[0], rm, off + 4), ctx); in emit_ldx_r()
1131 emit(ARM_AND_R(ARM_IP, rt, rn), ctx); in emit_ar_r()
1132 emit(ARM_AND_R(ARM_LR, rd, rm), ctx); in emit_ar_r()
1133 emit(ARM_ORRS_R(ARM_IP, ARM_LR, ARM_IP), ctx); in emit_ar_r()
1135 emit(ARM_ANDS_R(ARM_IP, rt, rn), ctx); in emit_ar_r()
1145 emit(ARM_CMP_R(rd, rm), ctx); in emit_ar_r()
1149 emit(ARM_CMP_R(rt, rn), ctx); in emit_ar_r()
1154 emit(ARM_CMP_R(rn, rt), ctx); in emit_ar_r()
1156 emit(ARM_SBCS_R(ARM_IP, rm, rd), ctx); in emit_ar_r()
1160 emit(ARM_CMP_R(rt, rn), ctx); in emit_ar_r()
1162 emit(ARM_SBCS_R(ARM_IP, rd, rm), ctx); in emit_ar_r()
1195 emit(ARM_LDR_I(tmp[1], r_array, off), ctx); in emit_bpf_tail_call()
1197 emit(ARM_CMP_R(r_index, tmp[1]), ctx); in emit_bpf_tail_call()
1209 emit(ARM_CMP_I(tc[0], hi), ctx); in emit_bpf_tail_call()
1212 emit(ARM_ADDS_I(tc[1], tc[1], 1), ctx); in emit_bpf_tail_call()
1213 emit(ARM_ADC_I(tc[0], tc[0], 0), ctx); in emit_bpf_tail_call()
1222 emit(ARM_ADD_I(tmp[1], r_array, off), ctx); in emit_bpf_tail_call()
1223 emit(ARM_LDR_R_SI(tmp[1], tmp[1], r_index, SRTYPE_ASL, 2), ctx); in emit_bpf_tail_call()
1224 emit(ARM_CMP_I(tmp[1], 0), ctx); in emit_bpf_tail_call()
1231 emit(ARM_LDR_I(tmp[1], tmp[1], off), ctx); in emit_bpf_tail_call()
1232 emit(ARM_ADD_I(tmp[1], tmp[1], ctx->prologue_bytes), ctx); in emit_bpf_tail_call()
1254 emit(ARM_AND_I(tmp2[1], rn, 0xff), ctx); in emit_rev16()
1255 emit(ARM_MOV_SI(tmp2[0], rn, SRTYPE_LSR, 8), ctx); in emit_rev16()
1256 emit(ARM_AND_I(tmp2[0], tmp2[0], 0xff), ctx); in emit_rev16()
1257 emit(ARM_ORR_SI(rd, tmp2[0], tmp2[1], SRTYPE_LSL, 8), ctx); in emit_rev16()
1259 emit(ARM_REV16(rd, rn), ctx); in emit_rev16()
1269 emit(ARM_AND_I(tmp2[1], rn, 0xff), ctx); in emit_rev32()
1270 emit(ARM_MOV_SI(tmp2[0], rn, SRTYPE_LSR, 24), ctx); in emit_rev32()
1271 emit(ARM_ORR_SI(ARM_IP, tmp2[0], tmp2[1], SRTYPE_LSL, 24), ctx); in emit_rev32()
1273 emit(ARM_MOV_SI(tmp2[1], rn, SRTYPE_LSR, 8), ctx); in emit_rev32()
1274 emit(ARM_AND_I(tmp2[1], tmp2[1], 0xff), ctx); in emit_rev32()
1275 emit(ARM_MOV_SI(tmp2[0], rn, SRTYPE_LSR, 16), ctx); in emit_rev32()
1276 emit(ARM_AND_I(tmp2[0], tmp2[0], 0xff), ctx); in emit_rev32()
1277 emit(ARM_MOV_SI(tmp2[0], tmp2[0], SRTYPE_LSL, 8), ctx); in emit_rev32()
1278 emit(ARM_ORR_SI(tmp2[0], tmp2[0], tmp2[1], SRTYPE_LSL, 16), ctx); in emit_rev32()
1279 emit(ARM_ORR_R(rd, ARM_IP, tmp2[0]), ctx); in emit_rev32()
1282 emit(ARM_REV(rd, rn), ctx); in emit_rev32()
1296 emit(ARM_PUSH(reg_set), ctx); in emit_push_r64()
1309 emit(ARM_MOV_R(ARM_IP, ARM_SP), ctx); in build_prologue()
1310 emit(ARM_PUSH(reg_set), ctx); in build_prologue()
1311 emit(ARM_SUB_I(ARM_FP, ARM_IP, 4), ctx); in build_prologue()
1313 emit(ARM_PUSH(CALLEE_PUSH_MASK), ctx); in build_prologue()
1314 emit(ARM_MOV_R(ARM_FP, ARM_SP), ctx); in build_prologue()
1318 emit(ARM_MOV_I(bpf_r1[0], 0), ctx); in build_prologue()
1319 emit(ARM_SUB_I(bpf_r1[1], ARM_SP, SCRATCH_SIZE), ctx); in build_prologue()
1324 emit(ARM_SUB_I(ARM_SP, ARM_SP, ctx->stack_size), ctx); in build_prologue()
1330 emit(ARM_MOV_I(bpf_r1[1], 0), ctx); in build_prologue()
1334 emit(ARM_MOV_R(bpf_r1[1], arm_r0), ctx); in build_prologue()
1346 emit(ARM_SUB_I(ARM_SP, ARM_FP, hweight16(reg_set) * 4), ctx); in build_epilogue()
1347 emit(ARM_LDM(ARM_SP, reg_set), ctx); in build_epilogue()
1350 emit(ARM_MOV_R(ARM_SP, ARM_FP), ctx); in build_epilogue()
1351 emit(ARM_POP(CALLEE_POP_MASK), ctx); in build_epilogue()
1578 emit(ARM_MOV_R(rd[0], ARM_LR), ctx); in build_insn()
1588 emit(ARM_AND_R(rd[1], rd[1], tmp2[1]), ctx); in build_insn()
1590 emit(ARM_UXTH(rd[1], rd[1]), ctx); in build_insn()
1593 emit(ARM_EOR_R(rd[0], rd[0], rd[0]), ctx); in build_insn()
1598 emit(ARM_EOR_R(rd[0], rd[0], rd[0]), ctx); in build_insn()
1785 emit(ARM_B(jmp_offset), ctx); in build_insn()
1813 emit(ARM_ADD_I(ARM_SP, ARM_SP, imm8m(24)), ctx); // callee clean in build_insn()
1825 emit(ARM_B(jmp_offset), ctx); in build_insn()