Lines Matching refs:ctx
119 static enum reg_val_type get_reg_val_type(const struct jit_ctx *ctx, in get_reg_val_type() argument
122 return (ctx->reg_val_types[index] >> (reg * 3)) & 7; in get_reg_val_type()
126 #define emit_instr_long(ctx, func64, func32, ...) \ argument
128 if ((ctx)->target != NULL) { \
129 u32 *p = &(ctx)->target[ctx->idx]; \
135 (ctx)->idx++; \
138 #define emit_instr(ctx, func, ...) \ argument
139 emit_instr_long(ctx, func, func, ##__VA_ARGS__)
141 static unsigned int j_target(struct jit_ctx *ctx, int target_idx) in j_target() argument
146 if (!ctx->target) in j_target()
149 base_va = (unsigned long)ctx->target; in j_target()
150 target_va = base_va + (ctx->offsets[target_idx] & ~OFFSETS_B_CONV); in j_target()
159 static u32 b_imm(unsigned int tgt, struct jit_ctx *ctx) in b_imm() argument
161 if (!ctx->gen_b_offsets) in b_imm()
176 return (ctx->offsets[tgt] & ~OFFSETS_B_CONV) - in b_imm()
177 (ctx->idx * 4) - 4; in b_imm()
193 static int ebpf_to_mips_reg(struct jit_ctx *ctx, in ebpf_to_mips_reg() argument
214 ctx->flags |= EBPF_SAVE_S0; in ebpf_to_mips_reg()
217 ctx->flags |= EBPF_SAVE_S1; in ebpf_to_mips_reg()
220 ctx->flags |= EBPF_SAVE_S2; in ebpf_to_mips_reg()
223 ctx->flags |= EBPF_SAVE_S3; in ebpf_to_mips_reg()
228 ctx->flags |= EBPF_SEEN_FP; in ebpf_to_mips_reg()
270 static int gen_int_prologue(struct jit_ctx *ctx) in gen_int_prologue() argument
276 if (ctx->flags & EBPF_SAVE_RA) in gen_int_prologue()
282 if (ctx->flags & EBPF_SAVE_S0) in gen_int_prologue()
284 if (ctx->flags & EBPF_SAVE_S1) in gen_int_prologue()
286 if (ctx->flags & EBPF_SAVE_S2) in gen_int_prologue()
288 if (ctx->flags & EBPF_SAVE_S3) in gen_int_prologue()
290 if (ctx->flags & EBPF_SAVE_S4) in gen_int_prologue()
294 locals_size = (ctx->flags & EBPF_SEEN_FP) ? MAX_BPF_STACK : 0; in gen_int_prologue()
298 ctx->stack_size = stack_adjust; in gen_int_prologue()
305 emit_instr(ctx, addiu, MIPS_R_V1, MIPS_R_ZERO, MAX_TAIL_CALL_CNT); in gen_int_prologue()
307 emit_instr_long(ctx, daddiu, addiu, in gen_int_prologue()
314 if (ctx->flags & EBPF_SAVE_RA) { in gen_int_prologue()
315 emit_instr_long(ctx, sd, sw, in gen_int_prologue()
319 if (ctx->flags & EBPF_SAVE_S0) { in gen_int_prologue()
320 emit_instr_long(ctx, sd, sw, in gen_int_prologue()
324 if (ctx->flags & EBPF_SAVE_S1) { in gen_int_prologue()
325 emit_instr_long(ctx, sd, sw, in gen_int_prologue()
329 if (ctx->flags & EBPF_SAVE_S2) { in gen_int_prologue()
330 emit_instr_long(ctx, sd, sw, in gen_int_prologue()
334 if (ctx->flags & EBPF_SAVE_S3) { in gen_int_prologue()
335 emit_instr_long(ctx, sd, sw, in gen_int_prologue()
339 if (ctx->flags & EBPF_SAVE_S4) { in gen_int_prologue()
340 emit_instr_long(ctx, sd, sw, in gen_int_prologue()
345 if ((ctx->flags & EBPF_SEEN_TC) && !(ctx->flags & EBPF_TCC_IN_V1)) in gen_int_prologue()
346 emit_instr_long(ctx, daddu, addu, in gen_int_prologue()
352 static int build_int_epilogue(struct jit_ctx *ctx, int dest_reg) in build_int_epilogue() argument
354 const struct bpf_prog *prog = ctx->skf; in build_int_epilogue()
355 int stack_adjust = ctx->stack_size; in build_int_epilogue()
362 td = get_reg_val_type(ctx, prog->len, BPF_REG_0); in build_int_epilogue()
364 emit_instr(ctx, sll, r0, r0, 0); in build_int_epilogue()
367 if (ctx->flags & EBPF_SAVE_RA) { in build_int_epilogue()
368 emit_instr_long(ctx, ld, lw, in build_int_epilogue()
372 if (ctx->flags & EBPF_SAVE_S0) { in build_int_epilogue()
373 emit_instr_long(ctx, ld, lw, in build_int_epilogue()
377 if (ctx->flags & EBPF_SAVE_S1) { in build_int_epilogue()
378 emit_instr_long(ctx, ld, lw, in build_int_epilogue()
382 if (ctx->flags & EBPF_SAVE_S2) { in build_int_epilogue()
383 emit_instr_long(ctx, ld, lw, in build_int_epilogue()
387 if (ctx->flags & EBPF_SAVE_S3) { in build_int_epilogue()
388 emit_instr_long(ctx, ld, lw, in build_int_epilogue()
392 if (ctx->flags & EBPF_SAVE_S4) { in build_int_epilogue()
393 emit_instr_long(ctx, ld, lw, in build_int_epilogue()
397 emit_instr(ctx, jr, dest_reg); in build_int_epilogue()
400 emit_instr_long(ctx, daddiu, addiu, in build_int_epilogue()
403 emit_instr(ctx, nop); in build_int_epilogue()
409 struct jit_ctx *ctx) in gen_imm_to_reg() argument
412 emit_instr(ctx, addiu, reg, MIPS_R_ZERO, insn->imm); in gen_imm_to_reg()
417 emit_instr(ctx, lui, reg, upper >> 16); in gen_imm_to_reg()
418 emit_instr(ctx, addiu, reg, reg, lower); in gen_imm_to_reg()
422 static int gen_imm_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, in gen_imm_insn() argument
426 int dst = ebpf_to_mips_reg(ctx, insn, dst_reg); in gen_imm_insn()
464 get_reg_val_type(ctx, idx, insn->dst_reg) == REG_32BIT) in gen_imm_insn()
465 emit_instr(ctx, dinsu, dst, MIPS_R_ZERO, 32, 32); in gen_imm_insn()
470 get_reg_val_type(ctx, idx, insn->dst_reg) != REG_32BIT) in gen_imm_insn()
471 emit_instr(ctx, sll, dst, dst, 0); in gen_imm_insn()
477 emit_instr(ctx, daddiu, dst, MIPS_R_ZERO, insn->imm); in gen_imm_insn()
481 emit_instr(ctx, andi, dst, dst, insn->imm); in gen_imm_insn()
485 emit_instr(ctx, ori, dst, dst, insn->imm); in gen_imm_insn()
489 emit_instr(ctx, xori, dst, dst, insn->imm); in gen_imm_insn()
492 emit_instr(ctx, daddiu, dst, dst, insn->imm); in gen_imm_insn()
495 emit_instr(ctx, daddiu, dst, dst, -insn->imm); in gen_imm_insn()
498 emit_instr(ctx, dsrl_safe, dst, dst, insn->imm & 0x3f); in gen_imm_insn()
501 emit_instr(ctx, srl, dst, dst, insn->imm & 0x1f); in gen_imm_insn()
504 emit_instr(ctx, dsll_safe, dst, dst, insn->imm & 0x3f); in gen_imm_insn()
507 emit_instr(ctx, sll, dst, dst, insn->imm & 0x1f); in gen_imm_insn()
510 emit_instr(ctx, dsra_safe, dst, dst, insn->imm & 0x3f); in gen_imm_insn()
513 emit_instr(ctx, sra, dst, dst, insn->imm & 0x1f); in gen_imm_insn()
516 emit_instr(ctx, addiu, dst, MIPS_R_ZERO, insn->imm); in gen_imm_insn()
519 emit_instr(ctx, addiu, dst, dst, insn->imm); in gen_imm_insn()
522 emit_instr(ctx, addiu, dst, dst, -insn->imm); in gen_imm_insn()
530 gen_imm_to_reg(insn, dst, ctx); in gen_imm_insn()
532 gen_imm_to_reg(insn, MIPS_R_AT, ctx); in gen_imm_insn()
536 emit_instr(ctx, and, dst, dst, MIPS_R_AT); in gen_imm_insn()
540 emit_instr(ctx, or, dst, dst, MIPS_R_AT); in gen_imm_insn()
544 emit_instr(ctx, xor, dst, dst, MIPS_R_AT); in gen_imm_insn()
547 emit_instr(ctx, daddu, dst, dst, MIPS_R_AT); in gen_imm_insn()
550 emit_instr(ctx, dsubu, dst, dst, MIPS_R_AT); in gen_imm_insn()
553 emit_instr(ctx, addu, dst, dst, MIPS_R_AT); in gen_imm_insn()
556 emit_instr(ctx, subu, dst, dst, MIPS_R_AT); in gen_imm_insn()
567 static void emit_const_to_reg(struct jit_ctx *ctx, int dst, u64 value) in emit_const_to_reg() argument
570 emit_instr(ctx, daddiu, dst, MIPS_R_ZERO, (int)value); in emit_const_to_reg()
573 emit_instr(ctx, lui, dst, (s32)(s16)(value >> 16)); in emit_const_to_reg()
574 emit_instr(ctx, ori, dst, dst, (unsigned int)(value & 0xffff)); in emit_const_to_reg()
584 emit_instr(ctx, dsll_safe, dst, dst, needed_shift); in emit_const_to_reg()
589 emit_instr(ctx, lui, dst, (s32)(s16)part); in emit_const_to_reg()
592 emit_instr(ctx, ori, dst, in emit_const_to_reg()
604 static int emit_bpf_tail_call(struct jit_ctx *ctx, int this_idx) in emit_bpf_tail_call() argument
609 ctx->flags |= EBPF_SEEN_TC; in emit_bpf_tail_call()
615 emit_instr(ctx, lwu, MIPS_R_T5, off, MIPS_R_A1); in emit_bpf_tail_call()
616 emit_instr(ctx, sltu, MIPS_R_AT, MIPS_R_T5, MIPS_R_A2); in emit_bpf_tail_call()
617 b_off = b_imm(this_idx + 1, ctx); in emit_bpf_tail_call()
618 emit_instr(ctx, bne, MIPS_R_AT, MIPS_R_ZERO, b_off); in emit_bpf_tail_call()
624 tcc_reg = (ctx->flags & EBPF_TCC_IN_V1) ? MIPS_R_V1 : MIPS_R_S4; in emit_bpf_tail_call()
625 emit_instr(ctx, daddiu, MIPS_R_T5, tcc_reg, -1); in emit_bpf_tail_call()
626 b_off = b_imm(this_idx + 1, ctx); in emit_bpf_tail_call()
627 emit_instr(ctx, bltz, tcc_reg, b_off); in emit_bpf_tail_call()
634 emit_instr(ctx, dsll, MIPS_R_T8, MIPS_R_A2, 3); in emit_bpf_tail_call()
635 emit_instr(ctx, daddu, MIPS_R_T8, MIPS_R_T8, MIPS_R_A1); in emit_bpf_tail_call()
637 emit_instr(ctx, ld, MIPS_R_AT, off, MIPS_R_T8); in emit_bpf_tail_call()
638 b_off = b_imm(this_idx + 1, ctx); in emit_bpf_tail_call()
639 emit_instr(ctx, beq, MIPS_R_AT, MIPS_R_ZERO, b_off); in emit_bpf_tail_call()
641 emit_instr(ctx, nop); in emit_bpf_tail_call()
645 emit_instr(ctx, ld, MIPS_R_T9, off, MIPS_R_AT); in emit_bpf_tail_call()
647 emit_instr(ctx, daddu, MIPS_R_V1, MIPS_R_T5, MIPS_R_ZERO); in emit_bpf_tail_call()
649 emit_instr(ctx, daddiu, MIPS_R_T9, MIPS_R_T9, 4); in emit_bpf_tail_call()
650 return build_int_epilogue(ctx, MIPS_R_T9); in emit_bpf_tail_call()
659 static int build_one_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, in build_one_insn() argument
692 r = gen_imm_insn(insn, ctx, this_idx); in build_one_insn()
697 dst = ebpf_to_mips_reg(ctx, insn, dst_reg); in build_one_insn()
700 if (get_reg_val_type(ctx, this_idx, insn->dst_reg) == REG_32BIT) in build_one_insn()
701 emit_instr(ctx, dinsu, dst, MIPS_R_ZERO, 32, 32); in build_one_insn()
704 gen_imm_to_reg(insn, MIPS_R_AT, ctx); in build_one_insn()
706 emit_instr(ctx, dmulu, dst, dst, MIPS_R_AT); in build_one_insn()
708 emit_instr(ctx, dmultu, MIPS_R_AT, dst); in build_one_insn()
709 emit_instr(ctx, mflo, dst); in build_one_insn()
713 dst = ebpf_to_mips_reg(ctx, insn, dst_reg); in build_one_insn()
716 if (get_reg_val_type(ctx, this_idx, insn->dst_reg) == REG_32BIT) in build_one_insn()
717 emit_instr(ctx, dinsu, dst, MIPS_R_ZERO, 32, 32); in build_one_insn()
718 emit_instr(ctx, dsubu, dst, MIPS_R_ZERO, dst); in build_one_insn()
721 dst = ebpf_to_mips_reg(ctx, insn, dst_reg); in build_one_insn()
724 td = get_reg_val_type(ctx, this_idx, insn->dst_reg); in build_one_insn()
727 emit_instr(ctx, sll, dst, dst, 0); in build_one_insn()
731 gen_imm_to_reg(insn, MIPS_R_AT, ctx); in build_one_insn()
733 emit_instr(ctx, mulu, dst, dst, MIPS_R_AT); in build_one_insn()
735 emit_instr(ctx, multu, dst, MIPS_R_AT); in build_one_insn()
736 emit_instr(ctx, mflo, dst); in build_one_insn()
740 dst = ebpf_to_mips_reg(ctx, insn, dst_reg); in build_one_insn()
743 td = get_reg_val_type(ctx, this_idx, insn->dst_reg); in build_one_insn()
746 emit_instr(ctx, sll, dst, dst, 0); in build_one_insn()
748 emit_instr(ctx, subu, dst, MIPS_R_ZERO, dst); in build_one_insn()
754 dst = ebpf_to_mips_reg(ctx, insn, dst_reg); in build_one_insn()
757 td = get_reg_val_type(ctx, this_idx, insn->dst_reg); in build_one_insn()
760 emit_instr(ctx, sll, dst, dst, 0); in build_one_insn()
764 emit_instr(ctx, addu, dst, MIPS_R_ZERO, MIPS_R_ZERO); in build_one_insn()
767 gen_imm_to_reg(insn, MIPS_R_AT, ctx); in build_one_insn()
770 emit_instr(ctx, divu_r6, dst, dst, MIPS_R_AT); in build_one_insn()
772 emit_instr(ctx, modu, dst, dst, MIPS_R_AT); in build_one_insn()
775 emit_instr(ctx, divu, dst, MIPS_R_AT); in build_one_insn()
777 emit_instr(ctx, mflo, dst); in build_one_insn()
779 emit_instr(ctx, mfhi, dst); in build_one_insn()
785 dst = ebpf_to_mips_reg(ctx, insn, dst_reg); in build_one_insn()
788 if (get_reg_val_type(ctx, this_idx, insn->dst_reg) == REG_32BIT) in build_one_insn()
789 emit_instr(ctx, dinsu, dst, MIPS_R_ZERO, 32, 32); in build_one_insn()
793 emit_instr(ctx, addu, dst, MIPS_R_ZERO, MIPS_R_ZERO); in build_one_insn()
796 gen_imm_to_reg(insn, MIPS_R_AT, ctx); in build_one_insn()
799 emit_instr(ctx, ddivu_r6, dst, dst, MIPS_R_AT); in build_one_insn()
801 emit_instr(ctx, modu, dst, dst, MIPS_R_AT); in build_one_insn()
804 emit_instr(ctx, ddivu, dst, MIPS_R_AT); in build_one_insn()
806 emit_instr(ctx, mflo, dst); in build_one_insn()
808 emit_instr(ctx, mfhi, dst); in build_one_insn()
822 src = ebpf_to_mips_reg(ctx, insn, src_reg); in build_one_insn()
823 dst = ebpf_to_mips_reg(ctx, insn, dst_reg); in build_one_insn()
826 if (get_reg_val_type(ctx, this_idx, insn->dst_reg) == REG_32BIT) in build_one_insn()
827 emit_instr(ctx, dinsu, dst, MIPS_R_ZERO, 32, 32); in build_one_insn()
831 emit_instr(ctx, daddiu, dst, MIPS_R_SP, MAX_BPF_STACK); in build_one_insn()
834 emit_instr(ctx, daddiu, MIPS_R_AT, MIPS_R_SP, MAX_BPF_STACK); in build_one_insn()
837 } else if (get_reg_val_type(ctx, this_idx, insn->src_reg) == REG_32BIT) { in build_one_insn()
844 emit_instr(ctx, daddu, tmp_reg, src, MIPS_R_ZERO); in build_one_insn()
845 emit_instr(ctx, dinsu, tmp_reg, MIPS_R_ZERO, 32, 32); in build_one_insn()
851 emit_instr(ctx, daddu, dst, src, MIPS_R_ZERO); in build_one_insn()
854 emit_instr(ctx, daddu, dst, dst, src); in build_one_insn()
857 emit_instr(ctx, dsubu, dst, dst, src); in build_one_insn()
860 emit_instr(ctx, xor, dst, dst, src); in build_one_insn()
863 emit_instr(ctx, or, dst, dst, src); in build_one_insn()
866 emit_instr(ctx, and, dst, dst, src); in build_one_insn()
870 emit_instr(ctx, dmulu, dst, dst, src); in build_one_insn()
872 emit_instr(ctx, dmultu, dst, src); in build_one_insn()
873 emit_instr(ctx, mflo, dst); in build_one_insn()
880 emit_instr(ctx, ddivu_r6, in build_one_insn()
883 emit_instr(ctx, modu, dst, dst, src); in build_one_insn()
886 emit_instr(ctx, ddivu, dst, src); in build_one_insn()
888 emit_instr(ctx, mflo, dst); in build_one_insn()
890 emit_instr(ctx, mfhi, dst); in build_one_insn()
893 emit_instr(ctx, dsllv, dst, dst, src); in build_one_insn()
896 emit_instr(ctx, dsrlv, dst, dst, src); in build_one_insn()
899 emit_instr(ctx, dsrav, dst, dst, src); in build_one_insn()
918 src = ebpf_to_mips_reg(ctx, insn, src_reg_no_fp); in build_one_insn()
919 dst = ebpf_to_mips_reg(ctx, insn, dst_reg); in build_one_insn()
922 td = get_reg_val_type(ctx, this_idx, insn->dst_reg); in build_one_insn()
925 emit_instr(ctx, sll, dst, dst, 0); in build_one_insn()
928 ts = get_reg_val_type(ctx, this_idx, insn->src_reg); in build_one_insn()
937 emit_instr(ctx, sll, tmp_reg, src, 0); in build_one_insn()
943 emit_instr(ctx, addu, dst, src, MIPS_R_ZERO); in build_one_insn()
946 emit_instr(ctx, addu, dst, dst, src); in build_one_insn()
949 emit_instr(ctx, subu, dst, dst, src); in build_one_insn()
952 emit_instr(ctx, xor, dst, dst, src); in build_one_insn()
955 emit_instr(ctx, or, dst, dst, src); in build_one_insn()
958 emit_instr(ctx, and, dst, dst, src); in build_one_insn()
961 emit_instr(ctx, mul, dst, dst, src); in build_one_insn()
967 emit_instr(ctx, divu_r6, dst, dst, src); in build_one_insn()
969 emit_instr(ctx, modu, dst, dst, src); in build_one_insn()
972 emit_instr(ctx, divu, dst, src); in build_one_insn()
974 emit_instr(ctx, mflo, dst); in build_one_insn()
976 emit_instr(ctx, mfhi, dst); in build_one_insn()
979 emit_instr(ctx, sllv, dst, dst, src); in build_one_insn()
982 emit_instr(ctx, srlv, dst, dst, src); in build_one_insn()
985 emit_instr(ctx, srav, dst, dst, src); in build_one_insn()
994 b_off = b_imm(exit_idx, ctx); in build_one_insn()
997 emit_instr(ctx, beq, MIPS_R_ZERO, MIPS_R_ZERO, b_off); in build_one_insn()
998 emit_instr(ctx, nop); in build_one_insn()
1004 dst = ebpf_to_mips_reg(ctx, insn, dst_reg_fp_ok); in build_one_insn()
1010 gen_imm_to_reg(insn, MIPS_R_AT, ctx); in build_one_insn()
1025 src = ebpf_to_mips_reg(ctx, insn, src_reg_no_fp); in build_one_insn()
1026 dst = ebpf_to_mips_reg(ctx, insn, dst_reg); in build_one_insn()
1029 td = get_reg_val_type(ctx, this_idx, insn->dst_reg); in build_one_insn()
1030 ts = get_reg_val_type(ctx, this_idx, insn->src_reg); in build_one_insn()
1032 emit_instr(ctx, sll, MIPS_R_AT, src, 0); in build_one_insn()
1035 emit_instr(ctx, sll, MIPS_R_AT, dst, 0); in build_one_insn()
1039 emit_instr(ctx, and, MIPS_R_AT, dst, src); in build_one_insn()
1044 emit_instr(ctx, dsubu, MIPS_R_AT, dst, src); in build_one_insn()
1046 b_off = b_imm(exit_idx, ctx); in build_one_insn()
1050 emit_instr(ctx, blez, MIPS_R_AT, b_off); in build_one_insn()
1052 emit_instr(ctx, bgtz, MIPS_R_AT, b_off); in build_one_insn()
1053 emit_instr(ctx, nop); in build_one_insn()
1056 b_off = b_imm(this_idx + insn->off + 1, ctx); in build_one_insn()
1060 emit_instr(ctx, bgtz, MIPS_R_AT, b_off); in build_one_insn()
1062 emit_instr(ctx, blez, MIPS_R_AT, b_off); in build_one_insn()
1063 emit_instr(ctx, nop); in build_one_insn()
1066 emit_instr(ctx, slt, MIPS_R_AT, dst, src); in build_one_insn()
1072 emit_instr(ctx, dsubu, MIPS_R_T8, dst, src); in build_one_insn()
1073 emit_instr(ctx, sltu, MIPS_R_AT, dst, src); in build_one_insn()
1076 emit_instr(ctx, seleqz, MIPS_R_T9, in build_one_insn()
1079 emit_instr(ctx, movz, MIPS_R_T9, in build_one_insn()
1081 emit_instr(ctx, movn, MIPS_R_T9, in build_one_insn()
1084 emit_instr(ctx, or, MIPS_R_AT, MIPS_R_T9, MIPS_R_AT); in build_one_insn()
1089 emit_instr(ctx, sltu, MIPS_R_AT, dst, src); in build_one_insn()
1104 b_off = b_imm(exit_idx, ctx); in build_one_insn()
1106 target = j_target(ctx, exit_idx); in build_one_insn()
1111 if (!(ctx->offsets[this_idx] & OFFSETS_B_CONV)) { in build_one_insn()
1112 ctx->offsets[this_idx] |= OFFSETS_B_CONV; in build_one_insn()
1113 ctx->long_b_conversion = 1; in build_one_insn()
1118 emit_instr(ctx, bne, dst, src, b_off); in build_one_insn()
1120 emit_instr(ctx, beq, dst, src, b_off); in build_one_insn()
1121 emit_instr(ctx, nop); in build_one_insn()
1122 if (ctx->offsets[this_idx] & OFFSETS_B_CONV) { in build_one_insn()
1123 emit_instr(ctx, j, target); in build_one_insn()
1124 emit_instr(ctx, nop); in build_one_insn()
1128 b_off = b_imm(this_idx + insn->off + 1, ctx); in build_one_insn()
1130 target = j_target(ctx, this_idx + insn->off + 1); in build_one_insn()
1135 if (!(ctx->offsets[this_idx] & OFFSETS_B_CONV)) { in build_one_insn()
1136 ctx->offsets[this_idx] |= OFFSETS_B_CONV; in build_one_insn()
1137 ctx->long_b_conversion = 1; in build_one_insn()
1142 emit_instr(ctx, beq, dst, src, b_off); in build_one_insn()
1144 emit_instr(ctx, bne, dst, src, b_off); in build_one_insn()
1145 emit_instr(ctx, nop); in build_one_insn()
1146 if (ctx->offsets[this_idx] & OFFSETS_B_CONV) { in build_one_insn()
1147 emit_instr(ctx, j, target); in build_one_insn()
1148 emit_instr(ctx, nop); in build_one_insn()
1156 dst = ebpf_to_mips_reg(ctx, insn, dst_reg_fp_ok); in build_one_insn()
1162 b_off = b_imm(exit_idx, ctx); in build_one_insn()
1167 emit_instr(ctx, blez, dst, b_off); in build_one_insn()
1170 emit_instr(ctx, bltz, dst, b_off); in build_one_insn()
1173 emit_instr(ctx, bgez, dst, b_off); in build_one_insn()
1176 emit_instr(ctx, bgtz, dst, b_off); in build_one_insn()
1179 emit_instr(ctx, nop); in build_one_insn()
1182 b_off = b_imm(this_idx + insn->off + 1, ctx); in build_one_insn()
1187 emit_instr(ctx, bgtz, dst, b_off); in build_one_insn()
1190 emit_instr(ctx, bgez, dst, b_off); in build_one_insn()
1193 emit_instr(ctx, bltz, dst, b_off); in build_one_insn()
1196 emit_instr(ctx, blez, dst, b_off); in build_one_insn()
1199 emit_instr(ctx, nop); in build_one_insn()
1215 emit_instr(ctx, slti, MIPS_R_AT, dst, (int)t64s); in build_one_insn()
1220 emit_const_to_reg(ctx, MIPS_R_AT, (u64)t64s); in build_one_insn()
1221 emit_instr(ctx, slt, MIPS_R_AT, dst, MIPS_R_AT); in build_one_insn()
1231 dst = ebpf_to_mips_reg(ctx, insn, dst_reg_fp_ok); in build_one_insn()
1247 emit_const_to_reg(ctx, MIPS_R_AT, (u64)t64s); in build_one_insn()
1248 emit_instr(ctx, sltu, MIPS_R_AT, dst, MIPS_R_AT); in build_one_insn()
1254 dst = ebpf_to_mips_reg(ctx, insn, dst_reg_fp_ok); in build_one_insn()
1258 if (ctx->use_bbit_insns && hweight32((u32)insn->imm) == 1) { in build_one_insn()
1260 b_off = b_imm(exit_idx, ctx); in build_one_insn()
1263 emit_instr(ctx, bbit0, dst, ffs((u32)insn->imm) - 1, b_off); in build_one_insn()
1264 emit_instr(ctx, nop); in build_one_insn()
1267 b_off = b_imm(this_idx + insn->off + 1, ctx); in build_one_insn()
1270 emit_instr(ctx, bbit1, dst, ffs((u32)insn->imm) - 1, b_off); in build_one_insn()
1271 emit_instr(ctx, nop); in build_one_insn()
1275 emit_const_to_reg(ctx, MIPS_R_AT, t64); in build_one_insn()
1276 emit_instr(ctx, and, MIPS_R_AT, dst, MIPS_R_AT); in build_one_insn()
1287 b_off = b_imm(this_idx + insn->off + 1, ctx); in build_one_insn()
1289 target = j_target(ctx, this_idx + insn->off + 1); in build_one_insn()
1292 emit_instr(ctx, j, target); in build_one_insn()
1294 emit_instr(ctx, b, b_off); in build_one_insn()
1296 emit_instr(ctx, nop); in build_one_insn()
1301 dst = ebpf_to_mips_reg(ctx, insn, dst_reg); in build_one_insn()
1305 emit_const_to_reg(ctx, dst, t64); in build_one_insn()
1309 ctx->flags |= EBPF_SAVE_RA; in build_one_insn()
1311 emit_const_to_reg(ctx, MIPS_R_T9, (u64)t64s); in build_one_insn()
1312 emit_instr(ctx, jalr, MIPS_R_RA, MIPS_R_T9); in build_one_insn()
1314 emit_instr(ctx, nop); in build_one_insn()
1318 if (emit_bpf_tail_call(ctx, this_idx)) in build_one_insn()
1324 dst = ebpf_to_mips_reg(ctx, insn, dst_reg); in build_one_insn()
1327 td = get_reg_val_type(ctx, this_idx, insn->dst_reg); in build_one_insn()
1329 emit_instr(ctx, dinsu, dst, MIPS_R_ZERO, 32, 32); in build_one_insn()
1333 emit_instr(ctx, sll, dst, dst, 0); in build_one_insn()
1343 emit_instr(ctx, wsbh, dst, dst); in build_one_insn()
1344 emit_instr(ctx, andi, dst, dst, 0xffff); in build_one_insn()
1347 emit_instr(ctx, wsbh, dst, dst); in build_one_insn()
1348 emit_instr(ctx, rotr, dst, dst, 16); in build_one_insn()
1352 emit_instr(ctx, dsbh, dst, dst); in build_one_insn()
1353 emit_instr(ctx, dshd, dst, dst); in build_one_insn()
1366 ctx->flags |= EBPF_SEEN_FP; in build_one_insn()
1370 dst = ebpf_to_mips_reg(ctx, insn, dst_reg); in build_one_insn()
1375 gen_imm_to_reg(insn, MIPS_R_AT, ctx); in build_one_insn()
1378 emit_instr(ctx, sb, MIPS_R_AT, mem_off, dst); in build_one_insn()
1381 emit_instr(ctx, sh, MIPS_R_AT, mem_off, dst); in build_one_insn()
1384 emit_instr(ctx, sw, MIPS_R_AT, mem_off, dst); in build_one_insn()
1387 emit_instr(ctx, sd, MIPS_R_AT, mem_off, dst); in build_one_insn()
1397 ctx->flags |= EBPF_SEEN_FP; in build_one_insn()
1401 src = ebpf_to_mips_reg(ctx, insn, src_reg_no_fp); in build_one_insn()
1406 dst = ebpf_to_mips_reg(ctx, insn, dst_reg); in build_one_insn()
1411 emit_instr(ctx, lbu, dst, mem_off, src); in build_one_insn()
1414 emit_instr(ctx, lhu, dst, mem_off, src); in build_one_insn()
1417 emit_instr(ctx, lw, dst, mem_off, src); in build_one_insn()
1420 emit_instr(ctx, ld, dst, mem_off, src); in build_one_insn()
1432 ctx->flags |= EBPF_SEEN_FP; in build_one_insn()
1436 dst = ebpf_to_mips_reg(ctx, insn, dst_reg); in build_one_insn()
1441 src = ebpf_to_mips_reg(ctx, insn, src_reg_no_fp); in build_one_insn()
1456 emit_instr(ctx, daddiu, MIPS_R_T6, in build_one_insn()
1463 if (get_reg_val_type(ctx, this_idx, insn->src_reg) == REG_32BIT) { in build_one_insn()
1464 emit_instr(ctx, sll, MIPS_R_AT, src, 0); in build_one_insn()
1467 emit_instr(ctx, ll, MIPS_R_T8, mem_off, dst); in build_one_insn()
1468 emit_instr(ctx, addu, MIPS_R_T8, MIPS_R_T8, src); in build_one_insn()
1469 emit_instr(ctx, sc, MIPS_R_T8, mem_off, dst); in build_one_insn()
1474 emit_instr(ctx, beq, MIPS_R_T8, MIPS_R_ZERO, -4 * 4); in build_one_insn()
1475 emit_instr(ctx, nop); in build_one_insn()
1478 if (get_reg_val_type(ctx, this_idx, insn->src_reg) == REG_32BIT) { in build_one_insn()
1479 emit_instr(ctx, daddu, MIPS_R_AT, src, MIPS_R_ZERO); in build_one_insn()
1480 emit_instr(ctx, dinsu, MIPS_R_AT, MIPS_R_ZERO, 32, 32); in build_one_insn()
1483 emit_instr(ctx, lld, MIPS_R_T8, mem_off, dst); in build_one_insn()
1484 emit_instr(ctx, daddu, MIPS_R_T8, MIPS_R_T8, src); in build_one_insn()
1485 emit_instr(ctx, scd, MIPS_R_T8, mem_off, dst); in build_one_insn()
1486 emit_instr(ctx, beq, MIPS_R_T8, MIPS_R_ZERO, -4 * 4); in build_one_insn()
1487 emit_instr(ctx, nop); in build_one_insn()
1493 emit_instr(ctx, sb, src, mem_off, dst); in build_one_insn()
1496 emit_instr(ctx, sh, src, mem_off, dst); in build_one_insn()
1499 emit_instr(ctx, sw, src, mem_off, dst); in build_one_insn()
1502 if (get_reg_val_type(ctx, this_idx, insn->src_reg) == REG_32BIT) { in build_one_insn()
1503 emit_instr(ctx, daddu, MIPS_R_AT, src, MIPS_R_ZERO); in build_one_insn()
1504 emit_instr(ctx, dinsu, MIPS_R_AT, MIPS_R_ZERO, 32, 32); in build_one_insn()
1507 emit_instr(ctx, sd, src, mem_off, dst); in build_one_insn()
1526 static int build_int_body(struct jit_ctx *ctx) in build_int_body() argument
1528 const struct bpf_prog *prog = ctx->skf; in build_int_body()
1534 if ((ctx->reg_val_types[i] & RVT_VISITED_MASK) == 0) { in build_int_body()
1540 if (ctx->target == NULL) in build_int_body()
1541 ctx->offsets[i] = (ctx->offsets[i] & OFFSETS_B_CONV) | (ctx->idx * 4); in build_int_body()
1543 r = build_one_insn(insn, ctx, i, prog->len); in build_int_body()
1549 if (ctx->target == NULL) in build_int_body()
1550 ctx->offsets[i] = ctx->idx * 4; in build_int_body()
1557 if (ctx->target == NULL) in build_int_body()
1561 ctx->offsets[i] = ctx->idx * 4; in build_int_body()
1567 static int reg_val_propagate_range(struct jit_ctx *ctx, u64 initial_rvt, in reg_val_propagate_range() argument
1570 const struct bpf_prog *prog = ctx->skf; in reg_val_propagate_range()
1573 u64 *rvt = ctx->reg_val_types; in reg_val_propagate_range()
1744 static int reg_val_propagate(struct jit_ctx *ctx) in reg_val_propagate() argument
1746 const struct bpf_prog *prog = ctx->skf; in reg_val_propagate()
1765 reg_val_propagate_range(ctx, exit_rvt, 0, false); in reg_val_propagate()
1774 u64 rvt = ctx->reg_val_types[i]; in reg_val_propagate()
1780 reg_val_propagate_range(ctx, rvt & ~RVT_VISITED_MASK, i, true); in reg_val_propagate()
1783 reg_val_propagate_range(ctx, rvt & ~RVT_VISITED_MASK, i, false); in reg_val_propagate()
1811 struct jit_ctx ctx; in bpf_int_jit_compile() local
1829 memset(&ctx, 0, sizeof(ctx)); in bpf_int_jit_compile()
1837 ctx.use_bbit_insns = 1; in bpf_int_jit_compile()
1840 ctx.use_bbit_insns = 0; in bpf_int_jit_compile()
1844 ctx.offsets = kcalloc(prog->len + 1, sizeof(*ctx.offsets), GFP_KERNEL); in bpf_int_jit_compile()
1845 if (ctx.offsets == NULL) in bpf_int_jit_compile()
1848 ctx.reg_val_types = kcalloc(prog->len + 1, sizeof(*ctx.reg_val_types), GFP_KERNEL); in bpf_int_jit_compile()
1849 if (ctx.reg_val_types == NULL) in bpf_int_jit_compile()
1852 ctx.skf = prog; in bpf_int_jit_compile()
1854 if (reg_val_propagate(&ctx)) in bpf_int_jit_compile()
1861 if (build_int_body(&ctx)) in bpf_int_jit_compile()
1868 if (ctx.flags & EBPF_SEEN_TC) { in bpf_int_jit_compile()
1869 if (ctx.flags & EBPF_SAVE_RA) in bpf_int_jit_compile()
1870 ctx.flags |= EBPF_SAVE_S4; in bpf_int_jit_compile()
1872 ctx.flags |= EBPF_TCC_IN_V1; in bpf_int_jit_compile()
1883 ctx.idx = 0; in bpf_int_jit_compile()
1884 ctx.gen_b_offsets = 1; in bpf_int_jit_compile()
1885 ctx.long_b_conversion = 0; in bpf_int_jit_compile()
1886 if (gen_int_prologue(&ctx)) in bpf_int_jit_compile()
1888 if (build_int_body(&ctx)) in bpf_int_jit_compile()
1890 if (build_int_epilogue(&ctx, MIPS_R_RA)) in bpf_int_jit_compile()
1892 } while (ctx.long_b_conversion); in bpf_int_jit_compile()
1894 image_size = 4 * ctx.idx; in bpf_int_jit_compile()
1901 ctx.target = (u32 *)image_ptr; in bpf_int_jit_compile()
1904 ctx.idx = 0; in bpf_int_jit_compile()
1905 if (gen_int_prologue(&ctx)) in bpf_int_jit_compile()
1907 if (build_int_body(&ctx)) in bpf_int_jit_compile()
1909 if (build_int_epilogue(&ctx, MIPS_R_RA)) in bpf_int_jit_compile()
1913 flush_icache_range((unsigned long)ctx.target, in bpf_int_jit_compile()
1914 (unsigned long)&ctx.target[ctx.idx]); in bpf_int_jit_compile()
1918 bpf_jit_dump(prog->len, image_size, 2, ctx.target); in bpf_int_jit_compile()
1921 prog->bpf_func = (void *)ctx.target; in bpf_int_jit_compile()
1928 kfree(ctx.offsets); in bpf_int_jit_compile()
1929 kfree(ctx.reg_val_types); in bpf_int_jit_compile()