Lines Matching refs:ctx
41 static u8 bpf_to_rv_reg(int bpf_reg, struct rv_jit_context *ctx) in bpf_to_rv_reg() argument
52 __set_bit(reg, &ctx->flags); in bpf_to_rv_reg()
57 static bool seen_reg(int reg, struct rv_jit_context *ctx) in seen_reg() argument
67 return test_bit(reg, &ctx->flags); in seen_reg()
72 static void mark_fp(struct rv_jit_context *ctx) in mark_fp() argument
74 __set_bit(RV_CTX_F_SEEN_S5, &ctx->flags); in mark_fp()
77 static void mark_call(struct rv_jit_context *ctx) in mark_call() argument
79 __set_bit(RV_CTX_F_SEEN_CALL, &ctx->flags); in mark_call()
82 static bool seen_call(struct rv_jit_context *ctx) in seen_call() argument
84 return test_bit(RV_CTX_F_SEEN_CALL, &ctx->flags); in seen_call()
87 static void mark_tail_call(struct rv_jit_context *ctx) in mark_tail_call() argument
89 __set_bit(RV_CTX_F_SEEN_TAIL_CALL, &ctx->flags); in mark_tail_call()
92 static bool seen_tail_call(struct rv_jit_context *ctx) in seen_tail_call() argument
94 return test_bit(RV_CTX_F_SEEN_TAIL_CALL, &ctx->flags); in seen_tail_call()
97 static u8 rv_tail_call_reg(struct rv_jit_context *ctx) in rv_tail_call_reg() argument
99 mark_tail_call(ctx); in rv_tail_call_reg()
101 if (seen_call(ctx)) { in rv_tail_call_reg()
102 __set_bit(RV_CTX_F_SEEN_S6, &ctx->flags); in rv_tail_call_reg()
123 static void emit_imm(u8 rd, s64 val, struct rv_jit_context *ctx) in emit_imm() argument
144 emit_lui(rd, upper, ctx); in emit_imm()
147 emit_li(rd, lower, ctx); in emit_imm()
151 emit_addiw(rd, rd, lower, ctx); in emit_imm()
159 emit_imm(rd, upper, ctx); in emit_imm()
161 emit_slli(rd, rd, shift, ctx); in emit_imm()
163 emit_addi(rd, rd, lower, ctx); in emit_imm()
166 static void __build_epilogue(bool is_tail_call, struct rv_jit_context *ctx) in __build_epilogue() argument
168 int stack_adjust = ctx->stack_size, store_offset = stack_adjust - 8; in __build_epilogue()
170 if (seen_reg(RV_REG_RA, ctx)) { in __build_epilogue()
171 emit_ld(RV_REG_RA, store_offset, RV_REG_SP, ctx); in __build_epilogue()
174 emit_ld(RV_REG_FP, store_offset, RV_REG_SP, ctx); in __build_epilogue()
176 if (seen_reg(RV_REG_S1, ctx)) { in __build_epilogue()
177 emit_ld(RV_REG_S1, store_offset, RV_REG_SP, ctx); in __build_epilogue()
180 if (seen_reg(RV_REG_S2, ctx)) { in __build_epilogue()
181 emit_ld(RV_REG_S2, store_offset, RV_REG_SP, ctx); in __build_epilogue()
184 if (seen_reg(RV_REG_S3, ctx)) { in __build_epilogue()
185 emit_ld(RV_REG_S3, store_offset, RV_REG_SP, ctx); in __build_epilogue()
188 if (seen_reg(RV_REG_S4, ctx)) { in __build_epilogue()
189 emit_ld(RV_REG_S4, store_offset, RV_REG_SP, ctx); in __build_epilogue()
192 if (seen_reg(RV_REG_S5, ctx)) { in __build_epilogue()
193 emit_ld(RV_REG_S5, store_offset, RV_REG_SP, ctx); in __build_epilogue()
196 if (seen_reg(RV_REG_S6, ctx)) { in __build_epilogue()
197 emit_ld(RV_REG_S6, store_offset, RV_REG_SP, ctx); in __build_epilogue()
201 emit_addi(RV_REG_SP, RV_REG_SP, stack_adjust, ctx); in __build_epilogue()
204 emit_mv(RV_REG_A0, RV_REG_A5, ctx); in __build_epilogue()
207 ctx); in __build_epilogue()
211 struct rv_jit_context *ctx) in emit_bcc() argument
215 emit(rv_beq(rd, rs, rvoff >> 1), ctx); in emit_bcc()
218 emit(rv_bltu(rs, rd, rvoff >> 1), ctx); in emit_bcc()
221 emit(rv_bltu(rd, rs, rvoff >> 1), ctx); in emit_bcc()
224 emit(rv_bgeu(rd, rs, rvoff >> 1), ctx); in emit_bcc()
227 emit(rv_bgeu(rs, rd, rvoff >> 1), ctx); in emit_bcc()
230 emit(rv_bne(rd, rs, rvoff >> 1), ctx); in emit_bcc()
233 emit(rv_blt(rs, rd, rvoff >> 1), ctx); in emit_bcc()
236 emit(rv_blt(rd, rs, rvoff >> 1), ctx); in emit_bcc()
239 emit(rv_bge(rd, rs, rvoff >> 1), ctx); in emit_bcc()
242 emit(rv_bge(rs, rd, rvoff >> 1), ctx); in emit_bcc()
247 struct rv_jit_context *ctx) in emit_branch() argument
252 emit_bcc(cond, rd, rs, rvoff, ctx); in emit_branch()
269 emit_bcc(cond, rd, rs, 8, ctx); in emit_branch()
270 emit(rv_jal(RV_REG_ZERO, rvoff >> 1), ctx); in emit_branch()
280 emit_bcc(cond, rd, rs, 12, ctx); in emit_branch()
281 emit(rv_auipc(RV_REG_T1, upper), ctx); in emit_branch()
282 emit(rv_jalr(RV_REG_ZERO, RV_REG_T1, lower), ctx); in emit_branch()
285 static void emit_zext_32(u8 reg, struct rv_jit_context *ctx) in emit_zext_32() argument
287 emit_slli(reg, reg, 32, ctx); in emit_zext_32()
288 emit_srli(reg, reg, 32, ctx); in emit_zext_32()
291 static int emit_bpf_tail_call(int insn, struct rv_jit_context *ctx) in emit_bpf_tail_call() argument
293 int tc_ninsn, off, start_insn = ctx->ninsns; in emit_bpf_tail_call()
294 u8 tcc = rv_tail_call_reg(ctx); in emit_bpf_tail_call()
303 tc_ninsn = insn ? ctx->offset[insn] - ctx->offset[insn - 1] : in emit_bpf_tail_call()
304 ctx->offset[0]; in emit_bpf_tail_call()
305 emit_zext_32(RV_REG_A2, ctx); in emit_bpf_tail_call()
310 emit(rv_lwu(RV_REG_T1, off, RV_REG_A1), ctx); in emit_bpf_tail_call()
311 off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); in emit_bpf_tail_call()
312 emit_branch(BPF_JGE, RV_REG_A2, RV_REG_T1, off, ctx); in emit_bpf_tail_call()
317 emit_addi(RV_REG_T1, tcc, -1, ctx); in emit_bpf_tail_call()
318 off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); in emit_bpf_tail_call()
319 emit_branch(BPF_JSLT, tcc, RV_REG_ZERO, off, ctx); in emit_bpf_tail_call()
325 emit_slli(RV_REG_T2, RV_REG_A2, 3, ctx); in emit_bpf_tail_call()
326 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_A1, ctx); in emit_bpf_tail_call()
330 emit_ld(RV_REG_T2, off, RV_REG_T2, ctx); in emit_bpf_tail_call()
331 off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); in emit_bpf_tail_call()
332 emit_branch(BPF_JEQ, RV_REG_T2, RV_REG_ZERO, off, ctx); in emit_bpf_tail_call()
338 emit_ld(RV_REG_T3, off, RV_REG_T2, ctx); in emit_bpf_tail_call()
339 emit_mv(RV_REG_TCC, RV_REG_T1, ctx); in emit_bpf_tail_call()
340 __build_epilogue(true, ctx); in emit_bpf_tail_call()
345 struct rv_jit_context *ctx) in init_regs() argument
356 *rd = bpf_to_rv_reg(insn->dst_reg, ctx); in init_regs()
362 *rs = bpf_to_rv_reg(insn->src_reg, ctx); in init_regs()
365 static void emit_zext_32_rd_rs(u8 *rd, u8 *rs, struct rv_jit_context *ctx) in emit_zext_32_rd_rs() argument
367 emit_mv(RV_REG_T2, *rd, ctx); in emit_zext_32_rd_rs()
368 emit_zext_32(RV_REG_T2, ctx); in emit_zext_32_rd_rs()
369 emit_mv(RV_REG_T1, *rs, ctx); in emit_zext_32_rd_rs()
370 emit_zext_32(RV_REG_T1, ctx); in emit_zext_32_rd_rs()
375 static void emit_sext_32_rd_rs(u8 *rd, u8 *rs, struct rv_jit_context *ctx) in emit_sext_32_rd_rs() argument
377 emit_addiw(RV_REG_T2, *rd, 0, ctx); in emit_sext_32_rd_rs()
378 emit_addiw(RV_REG_T1, *rs, 0, ctx); in emit_sext_32_rd_rs()
383 static void emit_zext_32_rd_t1(u8 *rd, struct rv_jit_context *ctx) in emit_zext_32_rd_t1() argument
385 emit_mv(RV_REG_T2, *rd, ctx); in emit_zext_32_rd_t1()
386 emit_zext_32(RV_REG_T2, ctx); in emit_zext_32_rd_t1()
387 emit_zext_32(RV_REG_T1, ctx); in emit_zext_32_rd_t1()
391 static void emit_sext_32_rd(u8 *rd, struct rv_jit_context *ctx) in emit_sext_32_rd() argument
393 emit_addiw(RV_REG_T2, *rd, 0, ctx); in emit_sext_32_rd()
398 struct rv_jit_context *ctx) in emit_jump_and_link() argument
403 emit(rv_jal(rd, rvoff >> 1), ctx); in emit_jump_and_link()
408 emit(rv_auipc(RV_REG_T1, upper), ctx); in emit_jump_and_link()
409 emit(rv_jalr(rd, RV_REG_T1, lower), ctx); in emit_jump_and_link()
423 static int emit_call(bool fixed, u64 addr, struct rv_jit_context *ctx) in emit_call() argument
430 if (addr && ctx->insns) { in emit_call()
431 ip = (u64)(long)(ctx->insns + ctx->ninsns); in emit_call()
435 ret = emit_jump_and_link(RV_REG_RA, off, !fixed, ctx); in emit_call()
438 rd = bpf_to_rv_reg(BPF_REG_0, ctx); in emit_call()
439 emit_mv(rd, RV_REG_A0, ctx); in emit_call()
443 int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, in bpf_jit_emit_insn() argument
448 int s, e, rvoff, ret, i = insn - ctx->prog->insnsi; in bpf_jit_emit_insn()
449 struct bpf_prog_aux *aux = ctx->prog->aux; in bpf_jit_emit_insn()
454 init_regs(&rd, &rs, insn, ctx); in bpf_jit_emit_insn()
462 emit_zext_32(rd, ctx); in bpf_jit_emit_insn()
465 emit_mv(rd, rs, ctx); in bpf_jit_emit_insn()
467 emit_zext_32(rd, ctx); in bpf_jit_emit_insn()
473 emit_add(rd, rd, rs, ctx); in bpf_jit_emit_insn()
475 emit_zext_32(rd, ctx); in bpf_jit_emit_insn()
480 emit_sub(rd, rd, rs, ctx); in bpf_jit_emit_insn()
482 emit_subw(rd, rd, rs, ctx); in bpf_jit_emit_insn()
485 emit_zext_32(rd, ctx); in bpf_jit_emit_insn()
489 emit_and(rd, rd, rs, ctx); in bpf_jit_emit_insn()
491 emit_zext_32(rd, ctx); in bpf_jit_emit_insn()
495 emit_or(rd, rd, rs, ctx); in bpf_jit_emit_insn()
497 emit_zext_32(rd, ctx); in bpf_jit_emit_insn()
501 emit_xor(rd, rd, rs, ctx); in bpf_jit_emit_insn()
503 emit_zext_32(rd, ctx); in bpf_jit_emit_insn()
507 emit(is64 ? rv_mul(rd, rd, rs) : rv_mulw(rd, rd, rs), ctx); in bpf_jit_emit_insn()
509 emit_zext_32(rd, ctx); in bpf_jit_emit_insn()
513 emit(is64 ? rv_divu(rd, rd, rs) : rv_divuw(rd, rd, rs), ctx); in bpf_jit_emit_insn()
515 emit_zext_32(rd, ctx); in bpf_jit_emit_insn()
519 emit(is64 ? rv_remu(rd, rd, rs) : rv_remuw(rd, rd, rs), ctx); in bpf_jit_emit_insn()
521 emit_zext_32(rd, ctx); in bpf_jit_emit_insn()
525 emit(is64 ? rv_sll(rd, rd, rs) : rv_sllw(rd, rd, rs), ctx); in bpf_jit_emit_insn()
527 emit_zext_32(rd, ctx); in bpf_jit_emit_insn()
531 emit(is64 ? rv_srl(rd, rd, rs) : rv_srlw(rd, rd, rs), ctx); in bpf_jit_emit_insn()
533 emit_zext_32(rd, ctx); in bpf_jit_emit_insn()
537 emit(is64 ? rv_sra(rd, rd, rs) : rv_sraw(rd, rd, rs), ctx); in bpf_jit_emit_insn()
539 emit_zext_32(rd, ctx); in bpf_jit_emit_insn()
545 emit_sub(rd, RV_REG_ZERO, rd, ctx); in bpf_jit_emit_insn()
547 emit_zext_32(rd, ctx); in bpf_jit_emit_insn()
554 emit_slli(rd, rd, 48, ctx); in bpf_jit_emit_insn()
555 emit_srli(rd, rd, 48, ctx); in bpf_jit_emit_insn()
559 emit_zext_32(rd, ctx); in bpf_jit_emit_insn()
568 emit_li(RV_REG_T2, 0, ctx); in bpf_jit_emit_insn()
570 emit_andi(RV_REG_T1, rd, 0xff, ctx); in bpf_jit_emit_insn()
571 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); in bpf_jit_emit_insn()
572 emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); in bpf_jit_emit_insn()
573 emit_srli(rd, rd, 8, ctx); in bpf_jit_emit_insn()
577 emit_andi(RV_REG_T1, rd, 0xff, ctx); in bpf_jit_emit_insn()
578 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); in bpf_jit_emit_insn()
579 emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); in bpf_jit_emit_insn()
580 emit_srli(rd, rd, 8, ctx); in bpf_jit_emit_insn()
582 emit_andi(RV_REG_T1, rd, 0xff, ctx); in bpf_jit_emit_insn()
583 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); in bpf_jit_emit_insn()
584 emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); in bpf_jit_emit_insn()
585 emit_srli(rd, rd, 8, ctx); in bpf_jit_emit_insn()
589 emit_andi(RV_REG_T1, rd, 0xff, ctx); in bpf_jit_emit_insn()
590 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); in bpf_jit_emit_insn()
591 emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); in bpf_jit_emit_insn()
592 emit_srli(rd, rd, 8, ctx); in bpf_jit_emit_insn()
594 emit_andi(RV_REG_T1, rd, 0xff, ctx); in bpf_jit_emit_insn()
595 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); in bpf_jit_emit_insn()
596 emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); in bpf_jit_emit_insn()
597 emit_srli(rd, rd, 8, ctx); in bpf_jit_emit_insn()
599 emit_andi(RV_REG_T1, rd, 0xff, ctx); in bpf_jit_emit_insn()
600 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); in bpf_jit_emit_insn()
601 emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); in bpf_jit_emit_insn()
602 emit_srli(rd, rd, 8, ctx); in bpf_jit_emit_insn()
604 emit_andi(RV_REG_T1, rd, 0xff, ctx); in bpf_jit_emit_insn()
605 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); in bpf_jit_emit_insn()
606 emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); in bpf_jit_emit_insn()
607 emit_srli(rd, rd, 8, ctx); in bpf_jit_emit_insn()
609 emit_andi(RV_REG_T1, rd, 0xff, ctx); in bpf_jit_emit_insn()
610 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); in bpf_jit_emit_insn()
612 emit_mv(rd, RV_REG_T2, ctx); in bpf_jit_emit_insn()
618 emit_imm(rd, imm, ctx); in bpf_jit_emit_insn()
620 emit_zext_32(rd, ctx); in bpf_jit_emit_insn()
627 emit_addi(rd, rd, imm, ctx); in bpf_jit_emit_insn()
629 emit_imm(RV_REG_T1, imm, ctx); in bpf_jit_emit_insn()
630 emit_add(rd, rd, RV_REG_T1, ctx); in bpf_jit_emit_insn()
633 emit_zext_32(rd, ctx); in bpf_jit_emit_insn()
638 emit_addi(rd, rd, -imm, ctx); in bpf_jit_emit_insn()
640 emit_imm(RV_REG_T1, imm, ctx); in bpf_jit_emit_insn()
641 emit_sub(rd, rd, RV_REG_T1, ctx); in bpf_jit_emit_insn()
644 emit_zext_32(rd, ctx); in bpf_jit_emit_insn()
649 emit_andi(rd, rd, imm, ctx); in bpf_jit_emit_insn()
651 emit_imm(RV_REG_T1, imm, ctx); in bpf_jit_emit_insn()
652 emit_and(rd, rd, RV_REG_T1, ctx); in bpf_jit_emit_insn()
655 emit_zext_32(rd, ctx); in bpf_jit_emit_insn()
660 emit(rv_ori(rd, rd, imm), ctx); in bpf_jit_emit_insn()
662 emit_imm(RV_REG_T1, imm, ctx); in bpf_jit_emit_insn()
663 emit_or(rd, rd, RV_REG_T1, ctx); in bpf_jit_emit_insn()
666 emit_zext_32(rd, ctx); in bpf_jit_emit_insn()
671 emit(rv_xori(rd, rd, imm), ctx); in bpf_jit_emit_insn()
673 emit_imm(RV_REG_T1, imm, ctx); in bpf_jit_emit_insn()
674 emit_xor(rd, rd, RV_REG_T1, ctx); in bpf_jit_emit_insn()
677 emit_zext_32(rd, ctx); in bpf_jit_emit_insn()
681 emit_imm(RV_REG_T1, imm, ctx); in bpf_jit_emit_insn()
683 rv_mulw(rd, rd, RV_REG_T1), ctx); in bpf_jit_emit_insn()
685 emit_zext_32(rd, ctx); in bpf_jit_emit_insn()
689 emit_imm(RV_REG_T1, imm, ctx); in bpf_jit_emit_insn()
691 rv_divuw(rd, rd, RV_REG_T1), ctx); in bpf_jit_emit_insn()
693 emit_zext_32(rd, ctx); in bpf_jit_emit_insn()
697 emit_imm(RV_REG_T1, imm, ctx); in bpf_jit_emit_insn()
699 rv_remuw(rd, rd, RV_REG_T1), ctx); in bpf_jit_emit_insn()
701 emit_zext_32(rd, ctx); in bpf_jit_emit_insn()
705 emit_slli(rd, rd, imm, ctx); in bpf_jit_emit_insn()
708 emit_zext_32(rd, ctx); in bpf_jit_emit_insn()
713 emit_srli(rd, rd, imm, ctx); in bpf_jit_emit_insn()
715 emit(rv_srliw(rd, rd, imm), ctx); in bpf_jit_emit_insn()
718 emit_zext_32(rd, ctx); in bpf_jit_emit_insn()
723 emit_srai(rd, rd, imm, ctx); in bpf_jit_emit_insn()
725 emit(rv_sraiw(rd, rd, imm), ctx); in bpf_jit_emit_insn()
728 emit_zext_32(rd, ctx); in bpf_jit_emit_insn()
733 rvoff = rv_offset(i, off, ctx); in bpf_jit_emit_insn()
734 ret = emit_jump_and_link(RV_REG_ZERO, rvoff, false, ctx); in bpf_jit_emit_insn()
762 rvoff = rv_offset(i, off, ctx); in bpf_jit_emit_insn()
764 s = ctx->ninsns; in bpf_jit_emit_insn()
766 emit_sext_32_rd_rs(&rd, &rs, ctx); in bpf_jit_emit_insn()
768 emit_zext_32_rd_rs(&rd, &rs, ctx); in bpf_jit_emit_insn()
769 e = ctx->ninsns; in bpf_jit_emit_insn()
778 emit_and(RV_REG_T1, rd, rs, ctx); in bpf_jit_emit_insn()
780 ctx); in bpf_jit_emit_insn()
782 emit_branch(BPF_OP(code), rd, rs, rvoff, ctx); in bpf_jit_emit_insn()
807 rvoff = rv_offset(i, off, ctx); in bpf_jit_emit_insn()
808 s = ctx->ninsns; in bpf_jit_emit_insn()
810 emit_imm(RV_REG_T1, imm, ctx); in bpf_jit_emit_insn()
818 emit_sext_32_rd(&rd, ctx); in bpf_jit_emit_insn()
820 emit_zext_32_rd_t1(&rd, ctx); in bpf_jit_emit_insn()
822 e = ctx->ninsns; in bpf_jit_emit_insn()
826 emit_branch(BPF_OP(code), rd, rs, rvoff, ctx); in bpf_jit_emit_insn()
831 rvoff = rv_offset(i, off, ctx); in bpf_jit_emit_insn()
832 s = ctx->ninsns; in bpf_jit_emit_insn()
834 emit_andi(RV_REG_T1, rd, imm, ctx); in bpf_jit_emit_insn()
836 emit_imm(RV_REG_T1, imm, ctx); in bpf_jit_emit_insn()
837 emit_and(RV_REG_T1, rd, RV_REG_T1, ctx); in bpf_jit_emit_insn()
844 emit_addiw(RV_REG_T1, RV_REG_T1, 0, ctx); in bpf_jit_emit_insn()
845 e = ctx->ninsns; in bpf_jit_emit_insn()
847 emit_branch(BPF_JNE, RV_REG_T1, RV_REG_ZERO, rvoff, ctx); in bpf_jit_emit_insn()
856 mark_call(ctx); in bpf_jit_emit_insn()
857 ret = bpf_jit_get_func_addr(ctx->prog, insn, extra_pass, &addr, in bpf_jit_emit_insn()
861 ret = emit_call(fixed, addr, ctx); in bpf_jit_emit_insn()
868 if (emit_bpf_tail_call(i, ctx)) in bpf_jit_emit_insn()
874 if (i == ctx->prog->len - 1) in bpf_jit_emit_insn()
877 rvoff = epilogue_offset(ctx); in bpf_jit_emit_insn()
878 ret = emit_jump_and_link(RV_REG_ZERO, rvoff, false, ctx); in bpf_jit_emit_insn()
890 emit_imm(rd, imm64, ctx); in bpf_jit_emit_insn()
897 emit(rv_lbu(rd, off, rs), ctx); in bpf_jit_emit_insn()
901 emit_imm(RV_REG_T1, off, ctx); in bpf_jit_emit_insn()
902 emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); in bpf_jit_emit_insn()
903 emit(rv_lbu(rd, 0, RV_REG_T1), ctx); in bpf_jit_emit_insn()
909 emit(rv_lhu(rd, off, rs), ctx); in bpf_jit_emit_insn()
913 emit_imm(RV_REG_T1, off, ctx); in bpf_jit_emit_insn()
914 emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); in bpf_jit_emit_insn()
915 emit(rv_lhu(rd, 0, RV_REG_T1), ctx); in bpf_jit_emit_insn()
921 emit(rv_lwu(rd, off, rs), ctx); in bpf_jit_emit_insn()
925 emit_imm(RV_REG_T1, off, ctx); in bpf_jit_emit_insn()
926 emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); in bpf_jit_emit_insn()
927 emit(rv_lwu(rd, 0, RV_REG_T1), ctx); in bpf_jit_emit_insn()
933 emit_ld(rd, off, rs, ctx); in bpf_jit_emit_insn()
937 emit_imm(RV_REG_T1, off, ctx); in bpf_jit_emit_insn()
938 emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); in bpf_jit_emit_insn()
939 emit_ld(rd, 0, RV_REG_T1, ctx); in bpf_jit_emit_insn()
948 emit_imm(RV_REG_T1, imm, ctx); in bpf_jit_emit_insn()
950 emit(rv_sb(rd, off, RV_REG_T1), ctx); in bpf_jit_emit_insn()
954 emit_imm(RV_REG_T2, off, ctx); in bpf_jit_emit_insn()
955 emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); in bpf_jit_emit_insn()
956 emit(rv_sb(RV_REG_T2, 0, RV_REG_T1), ctx); in bpf_jit_emit_insn()
960 emit_imm(RV_REG_T1, imm, ctx); in bpf_jit_emit_insn()
962 emit(rv_sh(rd, off, RV_REG_T1), ctx); in bpf_jit_emit_insn()
966 emit_imm(RV_REG_T2, off, ctx); in bpf_jit_emit_insn()
967 emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); in bpf_jit_emit_insn()
968 emit(rv_sh(RV_REG_T2, 0, RV_REG_T1), ctx); in bpf_jit_emit_insn()
971 emit_imm(RV_REG_T1, imm, ctx); in bpf_jit_emit_insn()
973 emit_sw(rd, off, RV_REG_T1, ctx); in bpf_jit_emit_insn()
977 emit_imm(RV_REG_T2, off, ctx); in bpf_jit_emit_insn()
978 emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); in bpf_jit_emit_insn()
979 emit_sw(RV_REG_T2, 0, RV_REG_T1, ctx); in bpf_jit_emit_insn()
982 emit_imm(RV_REG_T1, imm, ctx); in bpf_jit_emit_insn()
984 emit_sd(rd, off, RV_REG_T1, ctx); in bpf_jit_emit_insn()
988 emit_imm(RV_REG_T2, off, ctx); in bpf_jit_emit_insn()
989 emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); in bpf_jit_emit_insn()
990 emit_sd(RV_REG_T2, 0, RV_REG_T1, ctx); in bpf_jit_emit_insn()
996 emit(rv_sb(rd, off, rs), ctx); in bpf_jit_emit_insn()
1000 emit_imm(RV_REG_T1, off, ctx); in bpf_jit_emit_insn()
1001 emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); in bpf_jit_emit_insn()
1002 emit(rv_sb(RV_REG_T1, 0, rs), ctx); in bpf_jit_emit_insn()
1006 emit(rv_sh(rd, off, rs), ctx); in bpf_jit_emit_insn()
1010 emit_imm(RV_REG_T1, off, ctx); in bpf_jit_emit_insn()
1011 emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); in bpf_jit_emit_insn()
1012 emit(rv_sh(RV_REG_T1, 0, rs), ctx); in bpf_jit_emit_insn()
1016 emit_sw(rd, off, rs, ctx); in bpf_jit_emit_insn()
1020 emit_imm(RV_REG_T1, off, ctx); in bpf_jit_emit_insn()
1021 emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); in bpf_jit_emit_insn()
1022 emit_sw(RV_REG_T1, 0, rs, ctx); in bpf_jit_emit_insn()
1026 emit_sd(rd, off, rs, ctx); in bpf_jit_emit_insn()
1030 emit_imm(RV_REG_T1, off, ctx); in bpf_jit_emit_insn()
1031 emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); in bpf_jit_emit_insn()
1032 emit_sd(RV_REG_T1, 0, rs, ctx); in bpf_jit_emit_insn()
1040 emit_addi(RV_REG_T1, rd, off, ctx); in bpf_jit_emit_insn()
1042 emit_imm(RV_REG_T1, off, ctx); in bpf_jit_emit_insn()
1043 emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); in bpf_jit_emit_insn()
1051 rv_amoadd_d(RV_REG_ZERO, rs, rd, 0, 0), ctx); in bpf_jit_emit_insn()
1061 void bpf_jit_build_prologue(struct rv_jit_context *ctx) in bpf_jit_build_prologue() argument
1065 bpf_stack_adjust = round_up(ctx->prog->aux->stack_depth, 16); in bpf_jit_build_prologue()
1067 mark_fp(ctx); in bpf_jit_build_prologue()
1069 if (seen_reg(RV_REG_RA, ctx)) in bpf_jit_build_prologue()
1072 if (seen_reg(RV_REG_S1, ctx)) in bpf_jit_build_prologue()
1074 if (seen_reg(RV_REG_S2, ctx)) in bpf_jit_build_prologue()
1076 if (seen_reg(RV_REG_S3, ctx)) in bpf_jit_build_prologue()
1078 if (seen_reg(RV_REG_S4, ctx)) in bpf_jit_build_prologue()
1080 if (seen_reg(RV_REG_S5, ctx)) in bpf_jit_build_prologue()
1082 if (seen_reg(RV_REG_S6, ctx)) in bpf_jit_build_prologue()
1094 emit(rv_addi(RV_REG_TCC, RV_REG_ZERO, MAX_TAIL_CALL_CNT), ctx); in bpf_jit_build_prologue()
1096 emit_addi(RV_REG_SP, RV_REG_SP, -stack_adjust, ctx); in bpf_jit_build_prologue()
1098 if (seen_reg(RV_REG_RA, ctx)) { in bpf_jit_build_prologue()
1099 emit_sd(RV_REG_SP, store_offset, RV_REG_RA, ctx); in bpf_jit_build_prologue()
1102 emit_sd(RV_REG_SP, store_offset, RV_REG_FP, ctx); in bpf_jit_build_prologue()
1104 if (seen_reg(RV_REG_S1, ctx)) { in bpf_jit_build_prologue()
1105 emit_sd(RV_REG_SP, store_offset, RV_REG_S1, ctx); in bpf_jit_build_prologue()
1108 if (seen_reg(RV_REG_S2, ctx)) { in bpf_jit_build_prologue()
1109 emit_sd(RV_REG_SP, store_offset, RV_REG_S2, ctx); in bpf_jit_build_prologue()
1112 if (seen_reg(RV_REG_S3, ctx)) { in bpf_jit_build_prologue()
1113 emit_sd(RV_REG_SP, store_offset, RV_REG_S3, ctx); in bpf_jit_build_prologue()
1116 if (seen_reg(RV_REG_S4, ctx)) { in bpf_jit_build_prologue()
1117 emit_sd(RV_REG_SP, store_offset, RV_REG_S4, ctx); in bpf_jit_build_prologue()
1120 if (seen_reg(RV_REG_S5, ctx)) { in bpf_jit_build_prologue()
1121 emit_sd(RV_REG_SP, store_offset, RV_REG_S5, ctx); in bpf_jit_build_prologue()
1124 if (seen_reg(RV_REG_S6, ctx)) { in bpf_jit_build_prologue()
1125 emit_sd(RV_REG_SP, store_offset, RV_REG_S6, ctx); in bpf_jit_build_prologue()
1129 emit_addi(RV_REG_FP, RV_REG_SP, stack_adjust, ctx); in bpf_jit_build_prologue()
1132 emit_addi(RV_REG_S5, RV_REG_SP, bpf_stack_adjust, ctx); in bpf_jit_build_prologue()
1137 if (seen_tail_call(ctx) && seen_call(ctx)) in bpf_jit_build_prologue()
1138 emit_mv(RV_REG_TCC_SAVED, RV_REG_TCC, ctx); in bpf_jit_build_prologue()
1140 ctx->stack_size = stack_adjust; in bpf_jit_build_prologue()
1143 void bpf_jit_build_epilogue(struct rv_jit_context *ctx) in bpf_jit_build_epilogue() argument
1145 __build_epilogue(false, ctx); in bpf_jit_build_epilogue()