Lines Matching refs:ctx
118 static inline void emit_jit_reg_move(ptr dst, ptr src, struct jit_ctx *ctx);
121 #define emit_instr(ctx, func, ...) \ argument
123 if ((ctx)->target != NULL) { \
124 u32 *p = &(ctx)->target[ctx->idx]; \
127 (ctx)->idx++; \
134 #define emit_long_instr(ctx, func, ...) \ argument
136 if ((ctx)->target != NULL) { \
137 u32 *p = &(ctx)->target[ctx->idx]; \
140 (ctx)->idx++; \
150 unsigned int src2, struct jit_ctx *ctx) in emit_addu() argument
152 emit_instr(ctx, addu, dst, src1, src2); in emit_addu()
155 static inline void emit_nop(struct jit_ctx *ctx) in emit_nop() argument
157 emit_instr(ctx, nop); in emit_nop()
161 static inline void emit_load_imm(unsigned int dst, u32 imm, struct jit_ctx *ctx) in emit_load_imm() argument
163 if (ctx->target != NULL) { in emit_load_imm()
166 u32 *p = &ctx->target[ctx->idx]; in emit_load_imm()
168 p = &ctx->target[ctx->idx + 1]; in emit_load_imm()
171 u32 *p = &ctx->target[ctx->idx]; in emit_load_imm()
175 ctx->idx++; in emit_load_imm()
178 ctx->idx++; in emit_load_imm()
182 unsigned int src2, struct jit_ctx *ctx) in emit_or() argument
184 emit_instr(ctx, or, dst, src1, src2); in emit_or()
188 struct jit_ctx *ctx) in emit_ori() argument
191 emit_load_imm(r_tmp, imm, ctx); in emit_ori()
192 emit_or(dst, src, r_tmp, ctx); in emit_ori()
194 emit_instr(ctx, ori, dst, src, imm); in emit_ori()
199 int imm, struct jit_ctx *ctx) in emit_daddiu() argument
205 emit_instr(ctx, daddiu, dst, src, imm); in emit_daddiu()
209 u32 imm, struct jit_ctx *ctx) in emit_addiu() argument
212 emit_load_imm(r_tmp, imm, ctx); in emit_addiu()
213 emit_addu(dst, r_tmp, src, ctx); in emit_addiu()
215 emit_instr(ctx, addiu, dst, src, imm); in emit_addiu()
220 unsigned int src2, struct jit_ctx *ctx) in emit_and() argument
222 emit_instr(ctx, and, dst, src1, src2); in emit_and()
226 u32 imm, struct jit_ctx *ctx) in emit_andi() argument
230 emit_load_imm(r_tmp, imm, ctx); in emit_andi()
231 emit_and(dst, src, r_tmp, ctx); in emit_andi()
233 emit_instr(ctx, andi, dst, src, imm); in emit_andi()
238 unsigned int src2, struct jit_ctx *ctx) in emit_xor() argument
240 emit_instr(ctx, xor, dst, src1, src2); in emit_xor()
243 static inline void emit_xori(ptr dst, ptr src, u32 imm, struct jit_ctx *ctx) in emit_xori() argument
247 emit_load_imm(r_tmp, imm, ctx); in emit_xori()
248 emit_xor(dst, src, r_tmp, ctx); in emit_xori()
250 emit_instr(ctx, xori, dst, src, imm); in emit_xori()
254 static inline void emit_stack_offset(int offset, struct jit_ctx *ctx) in emit_stack_offset() argument
256 emit_long_instr(ctx, ADDIU, r_sp, r_sp, offset); in emit_stack_offset()
260 unsigned int src2, struct jit_ctx *ctx) in emit_subu() argument
262 emit_instr(ctx, subu, dst, src1, src2); in emit_subu()
265 static inline void emit_neg(unsigned int reg, struct jit_ctx *ctx) in emit_neg() argument
267 emit_subu(reg, r_zero, reg, ctx); in emit_neg()
271 unsigned int sa, struct jit_ctx *ctx) in emit_sllv() argument
273 emit_instr(ctx, sllv, dst, src, sa); in emit_sllv()
277 unsigned int sa, struct jit_ctx *ctx) in emit_sll() argument
282 emit_jit_reg_move(dst, r_zero, ctx); in emit_sll()
284 emit_instr(ctx, sll, dst, src, sa); in emit_sll()
288 unsigned int sa, struct jit_ctx *ctx) in emit_srlv() argument
290 emit_instr(ctx, srlv, dst, src, sa); in emit_srlv()
294 unsigned int sa, struct jit_ctx *ctx) in emit_srl() argument
299 emit_jit_reg_move(dst, r_zero, ctx); in emit_srl()
301 emit_instr(ctx, srl, dst, src, sa); in emit_srl()
305 unsigned int src2, struct jit_ctx *ctx) in emit_slt() argument
307 emit_instr(ctx, slt, dst, src1, src2); in emit_slt()
311 unsigned int src2, struct jit_ctx *ctx) in emit_sltu() argument
313 emit_instr(ctx, sltu, dst, src1, src2); in emit_sltu()
317 unsigned int imm, struct jit_ctx *ctx) in emit_sltiu() argument
321 emit_load_imm(r_tmp, imm, ctx); in emit_sltiu()
322 emit_sltu(dst, src, r_tmp, ctx); in emit_sltiu()
324 emit_instr(ctx, sltiu, dst, src, imm); in emit_sltiu()
332 struct jit_ctx *ctx) in emit_store_stack_reg() argument
334 emit_long_instr(ctx, SW, reg, offset, base); in emit_store_stack_reg()
338 struct jit_ctx *ctx) in emit_store() argument
340 emit_instr(ctx, sw, reg, offset, base); in emit_store()
345 struct jit_ctx *ctx) in emit_load_stack_reg() argument
347 emit_long_instr(ctx, LW, reg, offset, base); in emit_load_stack_reg()
351 unsigned int offset, struct jit_ctx *ctx) in emit_load() argument
353 emit_instr(ctx, lw, reg, offset, base); in emit_load()
357 unsigned int offset, struct jit_ctx *ctx) in emit_load_byte() argument
359 emit_instr(ctx, lb, reg, offset, base); in emit_load_byte()
363 unsigned int offset, struct jit_ctx *ctx) in emit_half_load() argument
365 emit_instr(ctx, lh, reg, offset, base); in emit_half_load()
369 unsigned int offset, struct jit_ctx *ctx) in emit_half_load_unsigned() argument
371 emit_instr(ctx, lhu, reg, offset, base); in emit_half_load_unsigned()
375 unsigned int src2, struct jit_ctx *ctx) in emit_mul() argument
377 emit_instr(ctx, mul, dst, src1, src2); in emit_mul()
381 struct jit_ctx *ctx) in emit_div() argument
383 if (ctx->target != NULL) { in emit_div()
384 u32 *p = &ctx->target[ctx->idx]; in emit_div()
386 p = &ctx->target[ctx->idx + 1]; in emit_div()
389 ctx->idx += 2; /* 2 insts */ in emit_div()
393 struct jit_ctx *ctx) in emit_mod() argument
395 if (ctx->target != NULL) { in emit_mod()
396 u32 *p = &ctx->target[ctx->idx]; in emit_mod()
398 p = &ctx->target[ctx->idx + 1]; in emit_mod()
401 ctx->idx += 2; /* 2 insts */ in emit_mod()
405 unsigned int sa, struct jit_ctx *ctx) in emit_dsll() argument
407 emit_instr(ctx, dsll, dst, src, sa); in emit_dsll()
411 unsigned int sa, struct jit_ctx *ctx) in emit_dsrl32() argument
413 emit_instr(ctx, dsrl32, dst, src, sa); in emit_dsrl32()
417 struct jit_ctx *ctx) in emit_wsbh() argument
419 emit_instr(ctx, wsbh, dst, src); in emit_wsbh()
424 int imm, struct jit_ctx *ctx) in emit_load_ptr() argument
427 emit_long_instr(ctx, LW, dst, imm, src); in emit_load_ptr()
432 struct jit_ctx *ctx) in emit_load_func() argument
436 emit_load_imm(r_tmp, (u64)imm >> 32, ctx); in emit_load_func()
437 emit_dsll(r_tmp_imm, r_tmp, 16, ctx); /* left shift by 16 */ in emit_load_func()
438 emit_ori(r_tmp, r_tmp_imm, (imm >> 16) & 0xffff, ctx); in emit_load_func()
439 emit_dsll(r_tmp_imm, r_tmp, 16, ctx); /* left shift by 16 */ in emit_load_func()
440 emit_ori(reg, r_tmp_imm, imm & 0xffff, ctx); in emit_load_func()
442 emit_load_imm(reg, imm, ctx); in emit_load_func()
447 static inline void emit_reg_move(ptr dst, ptr src, struct jit_ctx *ctx) in emit_reg_move() argument
449 emit_long_instr(ctx, ADDU, dst, src, r_zero); in emit_reg_move()
453 static inline void emit_jit_reg_move(ptr dst, ptr src, struct jit_ctx *ctx) in emit_jit_reg_move() argument
455 emit_addu(dst, src, r_zero, ctx); in emit_jit_reg_move()
459 static inline u32 b_imm(unsigned int tgt, struct jit_ctx *ctx) in b_imm() argument
461 if (ctx->target == NULL) in b_imm()
477 return ctx->offsets[tgt] - in b_imm()
478 (ctx->idx * 4 - ctx->prologue_bytes) - 4; in b_imm()
482 unsigned int imm, struct jit_ctx *ctx) in emit_bcond() argument
484 if (ctx->target != NULL) { in emit_bcond()
485 u32 *p = &ctx->target[ctx->idx]; in emit_bcond()
502 ctx->idx++; in emit_bcond()
505 static inline void emit_b(unsigned int imm, struct jit_ctx *ctx) in emit_b() argument
507 emit_bcond(MIPS_COND_ALL, r_zero, r_zero, imm, ctx); in emit_b()
511 struct jit_ctx *ctx) in emit_jalr() argument
513 emit_instr(ctx, jalr, link, reg); in emit_jalr()
516 static inline void emit_jr(unsigned int reg, struct jit_ctx *ctx) in emit_jr() argument
518 emit_instr(ctx, jr, reg); in emit_jr()
529 static void save_bpf_jit_regs(struct jit_ctx *ctx, unsigned offset) in save_bpf_jit_regs() argument
536 emit_stack_offset(-align_sp(offset), ctx); in save_bpf_jit_regs()
538 tmp_flags = sflags = ctx->flags >> SEEN_SREG_SFT; in save_bpf_jit_regs()
543 ctx); in save_bpf_jit_regs()
551 if (ctx->flags & SEEN_CALL) { in save_bpf_jit_regs()
552 emit_store_stack_reg(r_ra, r_sp, real_off, ctx); in save_bpf_jit_regs()
557 if (ctx->flags & SEEN_MEM) { in save_bpf_jit_regs()
560 emit_long_instr(ctx, ADDIU, r_M, r_sp, real_off); in save_bpf_jit_regs()
564 static void restore_bpf_jit_regs(struct jit_ctx *ctx, in restore_bpf_jit_regs() argument
570 tmp_flags = sflags = ctx->flags >> SEEN_SREG_SFT; in restore_bpf_jit_regs()
576 ctx); in restore_bpf_jit_regs()
584 if (ctx->flags & SEEN_CALL) in restore_bpf_jit_regs()
585 emit_load_stack_reg(r_ra, r_sp, real_off, ctx); in restore_bpf_jit_regs()
589 emit_stack_offset(align_sp(offset), ctx); in restore_bpf_jit_regs()
592 static unsigned int get_stack_depth(struct jit_ctx *ctx) in get_stack_depth() argument
598 sp_off += hweight32(ctx->flags >> SEEN_SREG_SFT) * SZREG; in get_stack_depth()
600 if (ctx->flags & SEEN_MEM) in get_stack_depth()
603 if (ctx->flags & SEEN_CALL) in get_stack_depth()
609 static void build_prologue(struct jit_ctx *ctx) in build_prologue() argument
614 sp_off = get_stack_depth(ctx); in build_prologue()
615 save_bpf_jit_regs(ctx, sp_off); in build_prologue()
617 if (ctx->flags & SEEN_SKB) in build_prologue()
618 emit_reg_move(r_skb, MIPS_R_A0, ctx); in build_prologue()
620 if (ctx->flags & SEEN_SKB_DATA) { in build_prologue()
623 ctx); in build_prologue()
625 ctx); in build_prologue()
628 offsetof(struct sk_buff, data), ctx); in build_prologue()
630 emit_subu(r_skb_hl, r_skb_len, r_tmp, ctx); in build_prologue()
633 if (ctx->flags & SEEN_X) in build_prologue()
634 emit_jit_reg_move(r_X, r_zero, ctx); in build_prologue()
642 if (bpf_needs_clear_a(&ctx->skf->insns[0]) && in build_prologue()
643 (ctx->flags & SEEN_A)) in build_prologue()
644 emit_jit_reg_move(r_A, r_zero, ctx); in build_prologue()
647 static void build_epilogue(struct jit_ctx *ctx) in build_epilogue() argument
653 sp_off = get_stack_depth(ctx); in build_epilogue()
654 restore_bpf_jit_regs(ctx, sp_off); in build_epilogue()
657 emit_jr(r_ra, ctx); in build_epilogue()
658 emit_nop(ctx); in build_epilogue()
670 static int build_body(struct jit_ctx *ctx) in build_body() argument
672 const struct bpf_prog *prog = ctx->skf; in build_body()
687 if (ctx->target == NULL) in build_body()
688 ctx->offsets[i] = ctx->idx * 4; in build_body()
693 ctx->flags |= SEEN_A; in build_body()
694 emit_load_imm(r_A, k, ctx); in build_body()
699 ctx->flags |= SEEN_SKB | SEEN_A; in build_body()
701 emit_load(r_A, r_skb, off, ctx); in build_body()
705 ctx->flags |= SEEN_MEM | SEEN_A; in build_body()
706 emit_load(r_A, r_M, SCRATCH_OFF(k), ctx); in build_body()
720 emit_load_imm(r_off, k, ctx); in build_body()
722 ctx->flags |= SEEN_CALL | SEEN_OFF | in build_body()
725 emit_load_func(r_s0, (ptr)sk_load_func, ctx); in build_body()
726 emit_reg_move(MIPS_R_A0, r_skb, ctx); in build_body()
727 emit_jalr(MIPS_R_RA, r_s0, ctx); in build_body()
729 emit_reg_move(MIPS_R_A1, r_off, ctx); in build_body()
731 emit_bcond(MIPS_COND_EQ, r_ret, 0, b_imm(i + 1, ctx), in build_body()
732 ctx); in build_body()
734 emit_reg_move(r_ret, r_zero, ctx); in build_body()
736 b_off = b_imm(prog->len, ctx); in build_body()
739 emit_b(b_off, ctx); in build_body()
740 emit_nop(ctx); in build_body()
754 ctx->flags |= SEEN_OFF | SEEN_X; in build_body()
755 emit_addiu(r_off, r_X, k, ctx); in build_body()
759 ctx->flags |= SEEN_X; in build_body()
760 emit_load_imm(r_X, k, ctx); in build_body()
764 ctx->flags |= SEEN_X | SEEN_MEM; in build_body()
765 emit_load(r_X, r_M, SCRATCH_OFF(k), ctx); in build_body()
769 ctx->flags |= SEEN_X | SEEN_SKB; in build_body()
771 emit_load(r_X, r_skb, off, ctx); in build_body()
775 ctx->flags |= SEEN_X | SEEN_CALL | SEEN_SKB; in build_body()
777 emit_load_func(r_s0, (ptr)sk_load_byte, ctx); in build_body()
782 emit_load_imm(MIPS_R_A1, k, ctx); in build_body()
783 emit_jalr(MIPS_R_RA, r_s0, ctx); in build_body()
784 emit_reg_move(MIPS_R_A0, r_skb, ctx); /* delay slot */ in build_body()
786 b_off = b_imm(prog->len, ctx); in build_body()
789 emit_bcond(MIPS_COND_NE, r_ret, 0, b_off, ctx); in build_body()
790 emit_reg_move(r_ret, r_zero, ctx); in build_body()
793 emit_andi(r_X, r_A, 0xf, ctx); in build_body()
795 emit_b(b_imm(i + 1, ctx), ctx); in build_body()
796 emit_sll(r_X, r_X, 2, ctx); /* delay slot */ in build_body()
800 ctx->flags |= SEEN_MEM | SEEN_A; in build_body()
801 emit_store(r_A, r_M, SCRATCH_OFF(k), ctx); in build_body()
805 ctx->flags |= SEEN_MEM | SEEN_X; in build_body()
806 emit_store(r_X, r_M, SCRATCH_OFF(k), ctx); in build_body()
810 ctx->flags |= SEEN_A; in build_body()
811 emit_addiu(r_A, r_A, k, ctx); in build_body()
815 ctx->flags |= SEEN_A | SEEN_X; in build_body()
816 emit_addu(r_A, r_A, r_X, ctx); in build_body()
820 ctx->flags |= SEEN_A; in build_body()
821 emit_addiu(r_A, r_A, -k, ctx); in build_body()
825 ctx->flags |= SEEN_A | SEEN_X; in build_body()
826 emit_subu(r_A, r_A, r_X, ctx); in build_body()
831 ctx->flags |= SEEN_A; in build_body()
832 emit_load_imm(r_s0, k, ctx); in build_body()
833 emit_mul(r_A, r_A, r_s0, ctx); in build_body()
837 ctx->flags |= SEEN_A | SEEN_X; in build_body()
838 emit_mul(r_A, r_A, r_X, ctx); in build_body()
845 ctx->flags |= SEEN_A; in build_body()
846 emit_srl(r_A, r_A, k, ctx); in build_body()
849 ctx->flags |= SEEN_A; in build_body()
850 emit_load_imm(r_s0, k, ctx); in build_body()
851 emit_div(r_A, r_s0, ctx); in build_body()
856 ctx->flags |= SEEN_A; in build_body()
857 emit_jit_reg_move(r_A, r_zero, ctx); in build_body()
859 ctx->flags |= SEEN_A; in build_body()
860 emit_load_imm(r_s0, k, ctx); in build_body()
861 emit_mod(r_A, r_s0, ctx); in build_body()
866 ctx->flags |= SEEN_X | SEEN_A; in build_body()
868 b_off = b_imm(prog->len, ctx); in build_body()
871 emit_bcond(MIPS_COND_EQ, r_X, r_zero, b_off, ctx); in build_body()
872 emit_load_imm(r_ret, 0, ctx); /* delay slot */ in build_body()
873 emit_div(r_A, r_X, ctx); in build_body()
877 ctx->flags |= SEEN_X | SEEN_A; in build_body()
879 b_off = b_imm(prog->len, ctx); in build_body()
882 emit_bcond(MIPS_COND_EQ, r_X, r_zero, b_off, ctx); in build_body()
883 emit_load_imm(r_ret, 0, ctx); /* delay slot */ in build_body()
884 emit_mod(r_A, r_X, ctx); in build_body()
888 ctx->flags |= SEEN_A; in build_body()
889 emit_ori(r_A, r_A, k, ctx); in build_body()
893 ctx->flags |= SEEN_A; in build_body()
894 emit_ori(r_A, r_A, r_X, ctx); in build_body()
898 ctx->flags |= SEEN_A; in build_body()
899 emit_xori(r_A, r_A, k, ctx); in build_body()
904 ctx->flags |= SEEN_A; in build_body()
905 emit_xor(r_A, r_A, r_X, ctx); in build_body()
909 ctx->flags |= SEEN_A; in build_body()
910 emit_andi(r_A, r_A, k, ctx); in build_body()
914 ctx->flags |= SEEN_A | SEEN_X; in build_body()
915 emit_and(r_A, r_A, r_X, ctx); in build_body()
919 ctx->flags |= SEEN_A; in build_body()
920 emit_sll(r_A, r_A, k, ctx); in build_body()
924 ctx->flags |= SEEN_A | SEEN_X; in build_body()
925 emit_sllv(r_A, r_A, r_X, ctx); in build_body()
929 ctx->flags |= SEEN_A; in build_body()
930 emit_srl(r_A, r_A, k, ctx); in build_body()
933 ctx->flags |= SEEN_A | SEEN_X; in build_body()
934 emit_srlv(r_A, r_A, r_X, ctx); in build_body()
938 ctx->flags |= SEEN_A; in build_body()
939 emit_neg(r_A, ctx); in build_body()
943 b_off = b_imm(i + k + 1, ctx); in build_body()
946 emit_b(b_off, ctx); in build_body()
947 emit_nop(ctx); in build_body()
954 ctx->flags |= SEEN_X; in build_body()
963 ctx->flags |= SEEN_X; in build_body()
972 ctx->flags |= SEEN_X; in build_body()
980 ctx->flags |= SEEN_A; in build_body()
981 emit_sltiu(r_s0, r_A, k, ctx); in build_body()
983 ctx->flags |= SEEN_A | in build_body()
985 emit_sltu(r_s0, r_A, r_X, ctx); in build_body()
988 b_off = b_imm(i + inst->jf + 1, ctx); in build_body()
990 ctx); in build_body()
991 emit_nop(ctx); in build_body()
995 ctx->flags |= SEEN_A | SEEN_X; in build_body()
997 emit_load_imm(r_s0, k, ctx); in build_body()
1000 ctx); in build_body()
1001 b_off = b_imm(i + inst->jf + 1, ctx); in build_body()
1003 b_off, ctx); in build_body()
1004 emit_nop(ctx); in build_body()
1006 b_off = b_imm(i + inst->jt + 1, ctx); in build_body()
1007 emit_b(b_off, ctx); in build_body()
1008 emit_nop(ctx); in build_body()
1011 b_off = b_imm(i + inst->jt + 1, ctx); in build_body()
1012 emit_b(b_off, ctx); in build_body()
1013 emit_nop(ctx); in build_body()
1018 ctx->flags |= SEEN_A; in build_body()
1019 emit_load_imm(r_s0, k, ctx); in build_body()
1021 b_off = b_imm(i + inst->jt + 1, ctx); in build_body()
1023 b_off, ctx); in build_body()
1024 emit_nop(ctx); in build_body()
1027 ctx); in build_body()
1029 b_off, ctx); in build_body()
1030 emit_nop(ctx); in build_body()
1033 ctx->flags |= SEEN_A | SEEN_X; in build_body()
1035 ctx); in build_body()
1037 b_off, ctx); in build_body()
1038 emit_nop(ctx); in build_body()
1040 b_off = b_imm(i + inst->jf + 1, ctx); in build_body()
1042 b_off, ctx); in build_body()
1043 emit_nop(ctx); in build_body()
1048 ctx->flags |= SEEN_A; in build_body()
1050 emit_load_imm(r_s1, k, ctx); in build_body()
1051 emit_and(r_s0, r_A, r_s1, ctx); in build_body()
1053 b_off = b_imm(i + inst->jt + 1, ctx); in build_body()
1054 emit_bcond(MIPS_COND_NE, r_s0, r_zero, b_off, ctx); in build_body()
1055 emit_nop(ctx); in build_body()
1057 b_off = b_imm(i + inst->jf + 1, ctx); in build_body()
1058 emit_b(b_off, ctx); in build_body()
1059 emit_nop(ctx); in build_body()
1062 ctx->flags |= SEEN_X | SEEN_A; in build_body()
1064 emit_and(r_s0, r_A, r_X, ctx); in build_body()
1066 b_off = b_imm(i + inst->jt + 1, ctx); in build_body()
1067 emit_bcond(MIPS_COND_NE, r_s0, r_zero, b_off, ctx); in build_body()
1068 emit_nop(ctx); in build_body()
1070 b_off = b_imm(i + inst->jf + 1, ctx); in build_body()
1071 emit_b(b_off, ctx); in build_body()
1072 emit_nop(ctx); in build_body()
1075 ctx->flags |= SEEN_A; in build_body()
1081 b_off = b_imm(prog->len, ctx); in build_body()
1084 emit_b(b_off, ctx); in build_body()
1086 emit_reg_move(r_ret, r_A, ctx); /* delay slot */ in build_body()
1093 emit_load_imm(r_ret, k, ctx); in build_body()
1099 b_off = b_imm(prog->len, ctx); in build_body()
1102 emit_b(b_off, ctx); in build_body()
1103 emit_nop(ctx); in build_body()
1108 ctx->flags |= SEEN_X | SEEN_A; in build_body()
1109 emit_jit_reg_move(r_X, r_A, ctx); in build_body()
1113 ctx->flags |= SEEN_A | SEEN_X; in build_body()
1114 emit_jit_reg_move(r_A, r_X, ctx); in build_body()
1119 ctx->flags |= SEEN_SKB | SEEN_OFF | SEEN_A; in build_body()
1123 emit_half_load(r_A, r_skb, off, ctx); in build_body()
1128 emit_wsbh(r_A, r_A, ctx); in build_body()
1131 emit_andi(r_tmp_imm, r_A, 0xff, ctx); in build_body()
1133 emit_sll(r_tmp, r_tmp_imm, 8, ctx); in build_body()
1135 emit_srl(r_tmp_imm, r_A, 8, ctx); in build_body()
1136 emit_andi(r_tmp_imm, r_tmp_imm, 0xff, ctx); in build_body()
1138 emit_or(r_A, r_tmp, r_tmp_imm, ctx); in build_body()
1143 ctx->flags |= SEEN_A | SEEN_OFF; in build_body()
1149 emit_load(r_A, 28, off, ctx); in build_body()
1155 ctx->flags |= SEEN_SKB | SEEN_A; in build_body()
1158 emit_load_ptr(r_s0, r_skb, off, ctx); in build_body()
1160 b_off = b_imm(prog->len, ctx); in build_body()
1163 emit_bcond(MIPS_COND_EQ, r_s0, r_zero, b_off, ctx); in build_body()
1164 emit_reg_move(r_ret, r_zero, ctx); in build_body()
1168 emit_load(r_A, r_s0, off, ctx); in build_body()
1172 emit_half_load_unsigned(r_A, r_s0, off, ctx); in build_body()
1176 ctx->flags |= SEEN_SKB | SEEN_A; in build_body()
1179 emit_load(r_A, r_skb, off, ctx); in build_body()
1182 ctx->flags |= SEEN_SKB | SEEN_A; in build_body()
1185 emit_load(r_A, r_skb, off, ctx); in build_body()
1188 ctx->flags |= SEEN_SKB | SEEN_A; in build_body()
1192 emit_half_load_unsigned(r_A, r_skb, off, ctx); in build_body()
1195 ctx->flags |= SEEN_SKB | SEEN_A; in build_body()
1196 emit_load_byte(r_A, r_skb, PKT_VLAN_PRESENT_OFFSET(), ctx); in build_body()
1198 emit_srl(r_A, r_A, PKT_VLAN_PRESENT_BIT, ctx); in build_body()
1200 emit_andi(r_A, r_A, 1, ctx); in build_body()
1203 ctx->flags |= SEEN_SKB; in build_body()
1205 emit_load_byte(r_tmp, r_skb, PKT_TYPE_OFFSET(), ctx); in build_body()
1207 emit_andi(r_A, r_tmp, PKT_TYPE_MAX, ctx); in build_body()
1210 emit_srl(r_A, r_A, 5, ctx); in build_body()
1214 ctx->flags |= SEEN_SKB | SEEN_A; in build_body()
1220 emit_half_load_unsigned(r_A, r_skb, off, ctx); in build_body()
1230 if (ctx->target == NULL) in build_body()
1231 ctx->offsets[i] = ctx->idx * 4; in build_body()
1238 struct jit_ctx ctx; in bpf_jit_compile() local
1244 memset(&ctx, 0, sizeof(ctx)); in bpf_jit_compile()
1246 ctx.offsets = kcalloc(fp->len + 1, sizeof(*ctx.offsets), GFP_KERNEL); in bpf_jit_compile()
1247 if (ctx.offsets == NULL) in bpf_jit_compile()
1250 ctx.skf = fp; in bpf_jit_compile()
1252 if (build_body(&ctx)) in bpf_jit_compile()
1255 tmp_idx = ctx.idx; in bpf_jit_compile()
1256 build_prologue(&ctx); in bpf_jit_compile()
1257 ctx.prologue_bytes = (ctx.idx - tmp_idx) * 4; in bpf_jit_compile()
1259 build_epilogue(&ctx); in bpf_jit_compile()
1261 alloc_size = 4 * ctx.idx; in bpf_jit_compile()
1262 ctx.target = module_alloc(alloc_size); in bpf_jit_compile()
1263 if (ctx.target == NULL) in bpf_jit_compile()
1267 memset(ctx.target, 0, alloc_size); in bpf_jit_compile()
1269 ctx.idx = 0; in bpf_jit_compile()
1272 build_prologue(&ctx); in bpf_jit_compile()
1273 if (build_body(&ctx)) { in bpf_jit_compile()
1274 module_memfree(ctx.target); in bpf_jit_compile()
1277 build_epilogue(&ctx); in bpf_jit_compile()
1280 flush_icache_range((ptr)ctx.target, (ptr)(ctx.target + ctx.idx)); in bpf_jit_compile()
1284 bpf_jit_dump(fp->len, alloc_size, 2, ctx.target); in bpf_jit_compile()
1286 fp->bpf_func = (void *)ctx.target; in bpf_jit_compile()
1290 kfree(ctx.offsets); in bpf_jit_compile()