1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * BPF JIT compiler for LoongArch
4 *
5 * Copyright (C) 2022 Loongson Technology Corporation Limited
6 */
7 #include "bpf_jit.h"
8
9 #define REG_TCC LOONGARCH_GPR_A6
10 #define TCC_SAVED LOONGARCH_GPR_S5
11
12 #define SAVE_RA BIT(0)
13 #define SAVE_TCC BIT(1)
14
15 static const int regmap[] = {
16 /* return value from in-kernel function, and exit value for eBPF program */
17 [BPF_REG_0] = LOONGARCH_GPR_A5,
18 /* arguments from eBPF program to in-kernel function */
19 [BPF_REG_1] = LOONGARCH_GPR_A0,
20 [BPF_REG_2] = LOONGARCH_GPR_A1,
21 [BPF_REG_3] = LOONGARCH_GPR_A2,
22 [BPF_REG_4] = LOONGARCH_GPR_A3,
23 [BPF_REG_5] = LOONGARCH_GPR_A4,
24 /* callee saved registers that in-kernel function will preserve */
25 [BPF_REG_6] = LOONGARCH_GPR_S0,
26 [BPF_REG_7] = LOONGARCH_GPR_S1,
27 [BPF_REG_8] = LOONGARCH_GPR_S2,
28 [BPF_REG_9] = LOONGARCH_GPR_S3,
29 /* read-only frame pointer to access stack */
30 [BPF_REG_FP] = LOONGARCH_GPR_S4,
31 /* temporary register for blinding constants */
32 [BPF_REG_AX] = LOONGARCH_GPR_T0,
33 };
34
mark_call(struct jit_ctx * ctx)35 static void mark_call(struct jit_ctx *ctx)
36 {
37 ctx->flags |= SAVE_RA;
38 }
39
mark_tail_call(struct jit_ctx * ctx)40 static void mark_tail_call(struct jit_ctx *ctx)
41 {
42 ctx->flags |= SAVE_TCC;
43 }
44
seen_call(struct jit_ctx * ctx)45 static bool seen_call(struct jit_ctx *ctx)
46 {
47 return (ctx->flags & SAVE_RA);
48 }
49
seen_tail_call(struct jit_ctx * ctx)50 static bool seen_tail_call(struct jit_ctx *ctx)
51 {
52 return (ctx->flags & SAVE_TCC);
53 }
54
tail_call_reg(struct jit_ctx * ctx)55 static u8 tail_call_reg(struct jit_ctx *ctx)
56 {
57 if (seen_call(ctx))
58 return TCC_SAVED;
59
60 return REG_TCC;
61 }
62
63 /*
64 * eBPF prog stack layout:
65 *
66 * high
67 * original $sp ------------> +-------------------------+ <--LOONGARCH_GPR_FP
68 * | $ra |
69 * +-------------------------+
70 * | $fp |
71 * +-------------------------+
72 * | $s0 |
73 * +-------------------------+
74 * | $s1 |
75 * +-------------------------+
76 * | $s2 |
77 * +-------------------------+
78 * | $s3 |
79 * +-------------------------+
80 * | $s4 |
81 * +-------------------------+
82 * | $s5 |
83 * +-------------------------+ <--BPF_REG_FP
84 * | prog->aux->stack_depth |
85 * | (optional) |
86 * current $sp -------------> +-------------------------+
87 * low
88 */
build_prologue(struct jit_ctx * ctx)89 static void build_prologue(struct jit_ctx *ctx)
90 {
91 int stack_adjust = 0, store_offset, bpf_stack_adjust;
92
93 bpf_stack_adjust = round_up(ctx->prog->aux->stack_depth, 16);
94
95 /* To store ra, fp, s0, s1, s2, s3, s4 and s5. */
96 stack_adjust += sizeof(long) * 8;
97
98 stack_adjust = round_up(stack_adjust, 16);
99 stack_adjust += bpf_stack_adjust;
100
101 /*
102 * First instruction initializes the tail call count (TCC).
103 * On tail call we skip this instruction, and the TCC is
104 * passed in REG_TCC from the caller.
105 */
106 emit_insn(ctx, addid, REG_TCC, LOONGARCH_GPR_ZERO, MAX_TAIL_CALL_CNT);
107
108 emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, -stack_adjust);
109
110 store_offset = stack_adjust - sizeof(long);
111 emit_insn(ctx, std, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, store_offset);
112
113 store_offset -= sizeof(long);
114 emit_insn(ctx, std, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, store_offset);
115
116 store_offset -= sizeof(long);
117 emit_insn(ctx, std, LOONGARCH_GPR_S0, LOONGARCH_GPR_SP, store_offset);
118
119 store_offset -= sizeof(long);
120 emit_insn(ctx, std, LOONGARCH_GPR_S1, LOONGARCH_GPR_SP, store_offset);
121
122 store_offset -= sizeof(long);
123 emit_insn(ctx, std, LOONGARCH_GPR_S2, LOONGARCH_GPR_SP, store_offset);
124
125 store_offset -= sizeof(long);
126 emit_insn(ctx, std, LOONGARCH_GPR_S3, LOONGARCH_GPR_SP, store_offset);
127
128 store_offset -= sizeof(long);
129 emit_insn(ctx, std, LOONGARCH_GPR_S4, LOONGARCH_GPR_SP, store_offset);
130
131 store_offset -= sizeof(long);
132 emit_insn(ctx, std, LOONGARCH_GPR_S5, LOONGARCH_GPR_SP, store_offset);
133
134 emit_insn(ctx, addid, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_adjust);
135
136 if (bpf_stack_adjust)
137 emit_insn(ctx, addid, regmap[BPF_REG_FP], LOONGARCH_GPR_SP, bpf_stack_adjust);
138
139 /*
140 * Program contains calls and tail calls, so REG_TCC need
141 * to be saved across calls.
142 */
143 if (seen_tail_call(ctx) && seen_call(ctx))
144 move_reg(ctx, TCC_SAVED, REG_TCC);
145 else
146 emit_insn(ctx, nop);
147
148 ctx->stack_size = stack_adjust;
149 }
150
__build_epilogue(struct jit_ctx * ctx,bool is_tail_call)151 static void __build_epilogue(struct jit_ctx *ctx, bool is_tail_call)
152 {
153 int stack_adjust = ctx->stack_size;
154 int load_offset;
155
156 load_offset = stack_adjust - sizeof(long);
157 emit_insn(ctx, ldd, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, load_offset);
158
159 load_offset -= sizeof(long);
160 emit_insn(ctx, ldd, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, load_offset);
161
162 load_offset -= sizeof(long);
163 emit_insn(ctx, ldd, LOONGARCH_GPR_S0, LOONGARCH_GPR_SP, load_offset);
164
165 load_offset -= sizeof(long);
166 emit_insn(ctx, ldd, LOONGARCH_GPR_S1, LOONGARCH_GPR_SP, load_offset);
167
168 load_offset -= sizeof(long);
169 emit_insn(ctx, ldd, LOONGARCH_GPR_S2, LOONGARCH_GPR_SP, load_offset);
170
171 load_offset -= sizeof(long);
172 emit_insn(ctx, ldd, LOONGARCH_GPR_S3, LOONGARCH_GPR_SP, load_offset);
173
174 load_offset -= sizeof(long);
175 emit_insn(ctx, ldd, LOONGARCH_GPR_S4, LOONGARCH_GPR_SP, load_offset);
176
177 load_offset -= sizeof(long);
178 emit_insn(ctx, ldd, LOONGARCH_GPR_S5, LOONGARCH_GPR_SP, load_offset);
179
180 emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, stack_adjust);
181
182 if (!is_tail_call) {
183 /* Set return value */
184 emit_insn(ctx, addiw, LOONGARCH_GPR_A0, regmap[BPF_REG_0], 0);
185 /* Return to the caller */
186 emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_RA, 0);
187 } else {
188 /*
189 * Call the next bpf prog and skip the first instruction
190 * of TCC initialization.
191 */
192 emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_T3, 1);
193 }
194 }
195
build_epilogue(struct jit_ctx * ctx)196 static void build_epilogue(struct jit_ctx *ctx)
197 {
198 __build_epilogue(ctx, false);
199 }
200
bpf_jit_supports_kfunc_call(void)201 bool bpf_jit_supports_kfunc_call(void)
202 {
203 return true;
204 }
205
bpf_jit_supports_far_kfunc_call(void)206 bool bpf_jit_supports_far_kfunc_call(void)
207 {
208 return true;
209 }
210
emit_bpf_tail_call(struct jit_ctx * ctx,int insn)211 static int emit_bpf_tail_call(struct jit_ctx *ctx, int insn)
212 {
213 int off, tc_ninsn = 0;
214 u8 tcc = tail_call_reg(ctx);
215 u8 a1 = LOONGARCH_GPR_A1;
216 u8 a2 = LOONGARCH_GPR_A2;
217 u8 t1 = LOONGARCH_GPR_T1;
218 u8 t2 = LOONGARCH_GPR_T2;
219 u8 t3 = LOONGARCH_GPR_T3;
220 const int idx0 = ctx->idx;
221
222 #define cur_offset (ctx->idx - idx0)
223 #define jmp_offset (tc_ninsn - (cur_offset))
224
225 /*
226 * a0: &ctx
227 * a1: &array
228 * a2: index
229 *
230 * if (index >= array->map.max_entries)
231 * goto out;
232 */
233 tc_ninsn = insn ? ctx->offset[insn+1] - ctx->offset[insn] : ctx->offset[0];
234 off = offsetof(struct bpf_array, map.max_entries);
235 emit_insn(ctx, ldwu, t1, a1, off);
236 /* bgeu $a2, $t1, jmp_offset */
237 if (emit_tailcall_jmp(ctx, BPF_JGE, a2, t1, jmp_offset) < 0)
238 goto toofar;
239
240 /*
241 * if (--TCC < 0)
242 * goto out;
243 */
244 emit_insn(ctx, addid, REG_TCC, tcc, -1);
245 if (emit_tailcall_jmp(ctx, BPF_JSLT, REG_TCC, LOONGARCH_GPR_ZERO, jmp_offset) < 0)
246 goto toofar;
247
248 /*
249 * prog = array->ptrs[index];
250 * if (!prog)
251 * goto out;
252 */
253 emit_insn(ctx, alsld, t2, a2, a1, 2);
254 off = offsetof(struct bpf_array, ptrs);
255 emit_insn(ctx, ldd, t2, t2, off);
256 /* beq $t2, $zero, jmp_offset */
257 if (emit_tailcall_jmp(ctx, BPF_JEQ, t2, LOONGARCH_GPR_ZERO, jmp_offset) < 0)
258 goto toofar;
259
260 /* goto *(prog->bpf_func + 4); */
261 off = offsetof(struct bpf_prog, bpf_func);
262 emit_insn(ctx, ldd, t3, t2, off);
263 __build_epilogue(ctx, true);
264
265 return 0;
266
267 toofar:
268 pr_info_once("tail_call: jump too far\n");
269 return -1;
270 #undef cur_offset
271 #undef jmp_offset
272 }
273
emit_atomic(const struct bpf_insn * insn,struct jit_ctx * ctx)274 static void emit_atomic(const struct bpf_insn *insn, struct jit_ctx *ctx)
275 {
276 const u8 t1 = LOONGARCH_GPR_T1;
277 const u8 t2 = LOONGARCH_GPR_T2;
278 const u8 t3 = LOONGARCH_GPR_T3;
279 const u8 r0 = regmap[BPF_REG_0];
280 const u8 src = regmap[insn->src_reg];
281 const u8 dst = regmap[insn->dst_reg];
282 const s16 off = insn->off;
283 const s32 imm = insn->imm;
284 const bool isdw = BPF_SIZE(insn->code) == BPF_DW;
285
286 move_imm(ctx, t1, off, false);
287 emit_insn(ctx, addd, t1, dst, t1);
288 move_reg(ctx, t3, src);
289
290 switch (imm) {
291 /* lock *(size *)(dst + off) <op>= src */
292 case BPF_ADD:
293 if (isdw)
294 emit_insn(ctx, amaddd, t2, t1, src);
295 else
296 emit_insn(ctx, amaddw, t2, t1, src);
297 break;
298 case BPF_AND:
299 if (isdw)
300 emit_insn(ctx, amandd, t2, t1, src);
301 else
302 emit_insn(ctx, amandw, t2, t1, src);
303 break;
304 case BPF_OR:
305 if (isdw)
306 emit_insn(ctx, amord, t2, t1, src);
307 else
308 emit_insn(ctx, amorw, t2, t1, src);
309 break;
310 case BPF_XOR:
311 if (isdw)
312 emit_insn(ctx, amxord, t2, t1, src);
313 else
314 emit_insn(ctx, amxorw, t2, t1, src);
315 break;
316 /* src = atomic_fetch_<op>(dst + off, src) */
317 case BPF_ADD | BPF_FETCH:
318 if (isdw) {
319 emit_insn(ctx, amaddd, src, t1, t3);
320 } else {
321 emit_insn(ctx, amaddw, src, t1, t3);
322 emit_zext_32(ctx, src, true);
323 }
324 break;
325 case BPF_AND | BPF_FETCH:
326 if (isdw) {
327 emit_insn(ctx, amandd, src, t1, t3);
328 } else {
329 emit_insn(ctx, amandw, src, t1, t3);
330 emit_zext_32(ctx, src, true);
331 }
332 break;
333 case BPF_OR | BPF_FETCH:
334 if (isdw) {
335 emit_insn(ctx, amord, src, t1, t3);
336 } else {
337 emit_insn(ctx, amorw, src, t1, t3);
338 emit_zext_32(ctx, src, true);
339 }
340 break;
341 case BPF_XOR | BPF_FETCH:
342 if (isdw) {
343 emit_insn(ctx, amxord, src, t1, t3);
344 } else {
345 emit_insn(ctx, amxorw, src, t1, t3);
346 emit_zext_32(ctx, src, true);
347 }
348 break;
349 /* src = atomic_xchg(dst + off, src); */
350 case BPF_XCHG:
351 if (isdw) {
352 emit_insn(ctx, amswapd, src, t1, t3);
353 } else {
354 emit_insn(ctx, amswapw, src, t1, t3);
355 emit_zext_32(ctx, src, true);
356 }
357 break;
358 /* r0 = atomic_cmpxchg(dst + off, r0, src); */
359 case BPF_CMPXCHG:
360 move_reg(ctx, t2, r0);
361 if (isdw) {
362 emit_insn(ctx, lld, r0, t1, 0);
363 emit_insn(ctx, bne, t2, r0, 4);
364 move_reg(ctx, t3, src);
365 emit_insn(ctx, scd, t3, t1, 0);
366 emit_insn(ctx, beq, t3, LOONGARCH_GPR_ZERO, -4);
367 } else {
368 emit_insn(ctx, llw, r0, t1, 0);
369 emit_zext_32(ctx, t2, true);
370 emit_zext_32(ctx, r0, true);
371 emit_insn(ctx, bne, t2, r0, 4);
372 move_reg(ctx, t3, src);
373 emit_insn(ctx, scw, t3, t1, 0);
374 emit_insn(ctx, beq, t3, LOONGARCH_GPR_ZERO, -6);
375 emit_zext_32(ctx, r0, true);
376 }
377 break;
378 }
379 }
380
is_signed_bpf_cond(u8 cond)381 static bool is_signed_bpf_cond(u8 cond)
382 {
383 return cond == BPF_JSGT || cond == BPF_JSLT ||
384 cond == BPF_JSGE || cond == BPF_JSLE;
385 }
386
387 #define BPF_FIXUP_REG_MASK GENMASK(31, 27)
388 #define BPF_FIXUP_OFFSET_MASK GENMASK(26, 0)
389
ex_handler_bpf(const struct exception_table_entry * ex,struct pt_regs * regs)390 bool ex_handler_bpf(const struct exception_table_entry *ex,
391 struct pt_regs *regs)
392 {
393 int dst_reg = FIELD_GET(BPF_FIXUP_REG_MASK, ex->fixup);
394 off_t offset = FIELD_GET(BPF_FIXUP_OFFSET_MASK, ex->fixup);
395
396 regs->regs[dst_reg] = 0;
397 regs->csr_era = (unsigned long)&ex->fixup - offset;
398
399 return true;
400 }
401
402 /* For accesses to BTF pointers, add an entry to the exception table */
add_exception_handler(const struct bpf_insn * insn,struct jit_ctx * ctx,int dst_reg)403 static int add_exception_handler(const struct bpf_insn *insn,
404 struct jit_ctx *ctx,
405 int dst_reg)
406 {
407 unsigned long pc;
408 off_t offset;
409 struct exception_table_entry *ex;
410
411 if (!ctx->image || !ctx->prog->aux->extable)
412 return 0;
413
414 if (BPF_MODE(insn->code) != BPF_PROBE_MEM &&
415 BPF_MODE(insn->code) != BPF_PROBE_MEMSX)
416 return 0;
417
418 if (WARN_ON_ONCE(ctx->num_exentries >= ctx->prog->aux->num_exentries))
419 return -EINVAL;
420
421 ex = &ctx->prog->aux->extable[ctx->num_exentries];
422 pc = (unsigned long)&ctx->image[ctx->idx - 1];
423
424 offset = pc - (long)&ex->insn;
425 if (WARN_ON_ONCE(offset >= 0 || offset < INT_MIN))
426 return -ERANGE;
427
428 ex->insn = offset;
429
430 /*
431 * Since the extable follows the program, the fixup offset is always
432 * negative and limited to BPF_JIT_REGION_SIZE. Store a positive value
433 * to keep things simple, and put the destination register in the upper
434 * bits. We don't need to worry about buildtime or runtime sort
435 * modifying the upper bits because the table is already sorted, and
436 * isn't part of the main exception table.
437 */
438 offset = (long)&ex->fixup - (pc + LOONGARCH_INSN_SIZE);
439 if (!FIELD_FIT(BPF_FIXUP_OFFSET_MASK, offset))
440 return -ERANGE;
441
442 ex->type = EX_TYPE_BPF;
443 ex->fixup = FIELD_PREP(BPF_FIXUP_OFFSET_MASK, offset) | FIELD_PREP(BPF_FIXUP_REG_MASK, dst_reg);
444
445 ctx->num_exentries++;
446
447 return 0;
448 }
449
build_insn(const struct bpf_insn * insn,struct jit_ctx * ctx,bool extra_pass)450 static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool extra_pass)
451 {
452 u8 tm = -1;
453 u64 func_addr;
454 bool func_addr_fixed, sign_extend;
455 int i = insn - ctx->prog->insnsi;
456 int ret, jmp_offset;
457 const u8 code = insn->code;
458 const u8 cond = BPF_OP(code);
459 const u8 t1 = LOONGARCH_GPR_T1;
460 const u8 t2 = LOONGARCH_GPR_T2;
461 const u8 src = regmap[insn->src_reg];
462 const u8 dst = regmap[insn->dst_reg];
463 const s16 off = insn->off;
464 const s32 imm = insn->imm;
465 const bool is32 = BPF_CLASS(insn->code) == BPF_ALU || BPF_CLASS(insn->code) == BPF_JMP32;
466
467 switch (code) {
468 /* dst = src */
469 case BPF_ALU | BPF_MOV | BPF_X:
470 case BPF_ALU64 | BPF_MOV | BPF_X:
471 switch (off) {
472 case 0:
473 move_reg(ctx, dst, src);
474 emit_zext_32(ctx, dst, is32);
475 break;
476 case 8:
477 move_reg(ctx, t1, src);
478 emit_insn(ctx, extwb, dst, t1);
479 emit_zext_32(ctx, dst, is32);
480 break;
481 case 16:
482 move_reg(ctx, t1, src);
483 emit_insn(ctx, extwh, dst, t1);
484 emit_zext_32(ctx, dst, is32);
485 break;
486 case 32:
487 emit_insn(ctx, addw, dst, src, LOONGARCH_GPR_ZERO);
488 break;
489 }
490 break;
491
492 /* dst = imm */
493 case BPF_ALU | BPF_MOV | BPF_K:
494 case BPF_ALU64 | BPF_MOV | BPF_K:
495 move_imm(ctx, dst, imm, is32);
496 break;
497
498 /* dst = dst + src */
499 case BPF_ALU | BPF_ADD | BPF_X:
500 case BPF_ALU64 | BPF_ADD | BPF_X:
501 emit_insn(ctx, addd, dst, dst, src);
502 emit_zext_32(ctx, dst, is32);
503 break;
504
505 /* dst = dst + imm */
506 case BPF_ALU | BPF_ADD | BPF_K:
507 case BPF_ALU64 | BPF_ADD | BPF_K:
508 if (is_signed_imm12(imm)) {
509 emit_insn(ctx, addid, dst, dst, imm);
510 } else {
511 move_imm(ctx, t1, imm, is32);
512 emit_insn(ctx, addd, dst, dst, t1);
513 }
514 emit_zext_32(ctx, dst, is32);
515 break;
516
517 /* dst = dst - src */
518 case BPF_ALU | BPF_SUB | BPF_X:
519 case BPF_ALU64 | BPF_SUB | BPF_X:
520 emit_insn(ctx, subd, dst, dst, src);
521 emit_zext_32(ctx, dst, is32);
522 break;
523
524 /* dst = dst - imm */
525 case BPF_ALU | BPF_SUB | BPF_K:
526 case BPF_ALU64 | BPF_SUB | BPF_K:
527 if (is_signed_imm12(-imm)) {
528 emit_insn(ctx, addid, dst, dst, -imm);
529 } else {
530 move_imm(ctx, t1, imm, is32);
531 emit_insn(ctx, subd, dst, dst, t1);
532 }
533 emit_zext_32(ctx, dst, is32);
534 break;
535
536 /* dst = dst * src */
537 case BPF_ALU | BPF_MUL | BPF_X:
538 case BPF_ALU64 | BPF_MUL | BPF_X:
539 emit_insn(ctx, muld, dst, dst, src);
540 emit_zext_32(ctx, dst, is32);
541 break;
542
543 /* dst = dst * imm */
544 case BPF_ALU | BPF_MUL | BPF_K:
545 case BPF_ALU64 | BPF_MUL | BPF_K:
546 move_imm(ctx, t1, imm, is32);
547 emit_insn(ctx, muld, dst, dst, t1);
548 emit_zext_32(ctx, dst, is32);
549 break;
550
551 /* dst = dst / src */
552 case BPF_ALU | BPF_DIV | BPF_X:
553 case BPF_ALU64 | BPF_DIV | BPF_X:
554 if (!off) {
555 emit_zext_32(ctx, dst, is32);
556 move_reg(ctx, t1, src);
557 emit_zext_32(ctx, t1, is32);
558 emit_insn(ctx, divdu, dst, dst, t1);
559 emit_zext_32(ctx, dst, is32);
560 } else {
561 emit_sext_32(ctx, dst, is32);
562 move_reg(ctx, t1, src);
563 emit_sext_32(ctx, t1, is32);
564 emit_insn(ctx, divd, dst, dst, t1);
565 emit_sext_32(ctx, dst, is32);
566 }
567 break;
568
569 /* dst = dst / imm */
570 case BPF_ALU | BPF_DIV | BPF_K:
571 case BPF_ALU64 | BPF_DIV | BPF_K:
572 if (!off) {
573 move_imm(ctx, t1, imm, is32);
574 emit_zext_32(ctx, dst, is32);
575 emit_insn(ctx, divdu, dst, dst, t1);
576 emit_zext_32(ctx, dst, is32);
577 } else {
578 move_imm(ctx, t1, imm, false);
579 emit_sext_32(ctx, t1, is32);
580 emit_sext_32(ctx, dst, is32);
581 emit_insn(ctx, divd, dst, dst, t1);
582 emit_sext_32(ctx, dst, is32);
583 }
584 break;
585
586 /* dst = dst % src */
587 case BPF_ALU | BPF_MOD | BPF_X:
588 case BPF_ALU64 | BPF_MOD | BPF_X:
589 if (!off) {
590 emit_zext_32(ctx, dst, is32);
591 move_reg(ctx, t1, src);
592 emit_zext_32(ctx, t1, is32);
593 emit_insn(ctx, moddu, dst, dst, t1);
594 emit_zext_32(ctx, dst, is32);
595 } else {
596 emit_sext_32(ctx, dst, is32);
597 move_reg(ctx, t1, src);
598 emit_sext_32(ctx, t1, is32);
599 emit_insn(ctx, modd, dst, dst, t1);
600 emit_sext_32(ctx, dst, is32);
601 }
602 break;
603
604 /* dst = dst % imm */
605 case BPF_ALU | BPF_MOD | BPF_K:
606 case BPF_ALU64 | BPF_MOD | BPF_K:
607 if (!off) {
608 move_imm(ctx, t1, imm, is32);
609 emit_zext_32(ctx, dst, is32);
610 emit_insn(ctx, moddu, dst, dst, t1);
611 emit_zext_32(ctx, dst, is32);
612 } else {
613 move_imm(ctx, t1, imm, false);
614 emit_sext_32(ctx, t1, is32);
615 emit_sext_32(ctx, dst, is32);
616 emit_insn(ctx, modd, dst, dst, t1);
617 emit_sext_32(ctx, dst, is32);
618 }
619 break;
620
621 /* dst = -dst */
622 case BPF_ALU | BPF_NEG:
623 case BPF_ALU64 | BPF_NEG:
624 move_imm(ctx, t1, imm, is32);
625 emit_insn(ctx, subd, dst, LOONGARCH_GPR_ZERO, dst);
626 emit_zext_32(ctx, dst, is32);
627 break;
628
629 /* dst = dst & src */
630 case BPF_ALU | BPF_AND | BPF_X:
631 case BPF_ALU64 | BPF_AND | BPF_X:
632 emit_insn(ctx, and, dst, dst, src);
633 emit_zext_32(ctx, dst, is32);
634 break;
635
636 /* dst = dst & imm */
637 case BPF_ALU | BPF_AND | BPF_K:
638 case BPF_ALU64 | BPF_AND | BPF_K:
639 if (is_unsigned_imm12(imm)) {
640 emit_insn(ctx, andi, dst, dst, imm);
641 } else {
642 move_imm(ctx, t1, imm, is32);
643 emit_insn(ctx, and, dst, dst, t1);
644 }
645 emit_zext_32(ctx, dst, is32);
646 break;
647
648 /* dst = dst | src */
649 case BPF_ALU | BPF_OR | BPF_X:
650 case BPF_ALU64 | BPF_OR | BPF_X:
651 emit_insn(ctx, or, dst, dst, src);
652 emit_zext_32(ctx, dst, is32);
653 break;
654
655 /* dst = dst | imm */
656 case BPF_ALU | BPF_OR | BPF_K:
657 case BPF_ALU64 | BPF_OR | BPF_K:
658 if (is_unsigned_imm12(imm)) {
659 emit_insn(ctx, ori, dst, dst, imm);
660 } else {
661 move_imm(ctx, t1, imm, is32);
662 emit_insn(ctx, or, dst, dst, t1);
663 }
664 emit_zext_32(ctx, dst, is32);
665 break;
666
667 /* dst = dst ^ src */
668 case BPF_ALU | BPF_XOR | BPF_X:
669 case BPF_ALU64 | BPF_XOR | BPF_X:
670 emit_insn(ctx, xor, dst, dst, src);
671 emit_zext_32(ctx, dst, is32);
672 break;
673
674 /* dst = dst ^ imm */
675 case BPF_ALU | BPF_XOR | BPF_K:
676 case BPF_ALU64 | BPF_XOR | BPF_K:
677 if (is_unsigned_imm12(imm)) {
678 emit_insn(ctx, xori, dst, dst, imm);
679 } else {
680 move_imm(ctx, t1, imm, is32);
681 emit_insn(ctx, xor, dst, dst, t1);
682 }
683 emit_zext_32(ctx, dst, is32);
684 break;
685
686 /* dst = dst << src (logical) */
687 case BPF_ALU | BPF_LSH | BPF_X:
688 emit_insn(ctx, sllw, dst, dst, src);
689 emit_zext_32(ctx, dst, is32);
690 break;
691
692 case BPF_ALU64 | BPF_LSH | BPF_X:
693 emit_insn(ctx, slld, dst, dst, src);
694 break;
695
696 /* dst = dst << imm (logical) */
697 case BPF_ALU | BPF_LSH | BPF_K:
698 emit_insn(ctx, slliw, dst, dst, imm);
699 emit_zext_32(ctx, dst, is32);
700 break;
701
702 case BPF_ALU64 | BPF_LSH | BPF_K:
703 emit_insn(ctx, sllid, dst, dst, imm);
704 break;
705
706 /* dst = dst >> src (logical) */
707 case BPF_ALU | BPF_RSH | BPF_X:
708 emit_insn(ctx, srlw, dst, dst, src);
709 emit_zext_32(ctx, dst, is32);
710 break;
711
712 case BPF_ALU64 | BPF_RSH | BPF_X:
713 emit_insn(ctx, srld, dst, dst, src);
714 break;
715
716 /* dst = dst >> imm (logical) */
717 case BPF_ALU | BPF_RSH | BPF_K:
718 emit_insn(ctx, srliw, dst, dst, imm);
719 emit_zext_32(ctx, dst, is32);
720 break;
721
722 case BPF_ALU64 | BPF_RSH | BPF_K:
723 emit_insn(ctx, srlid, dst, dst, imm);
724 break;
725
726 /* dst = dst >> src (arithmetic) */
727 case BPF_ALU | BPF_ARSH | BPF_X:
728 emit_insn(ctx, sraw, dst, dst, src);
729 emit_zext_32(ctx, dst, is32);
730 break;
731
732 case BPF_ALU64 | BPF_ARSH | BPF_X:
733 emit_insn(ctx, srad, dst, dst, src);
734 break;
735
736 /* dst = dst >> imm (arithmetic) */
737 case BPF_ALU | BPF_ARSH | BPF_K:
738 emit_insn(ctx, sraiw, dst, dst, imm);
739 emit_zext_32(ctx, dst, is32);
740 break;
741
742 case BPF_ALU64 | BPF_ARSH | BPF_K:
743 emit_insn(ctx, sraid, dst, dst, imm);
744 break;
745
746 /* dst = BSWAP##imm(dst) */
747 case BPF_ALU | BPF_END | BPF_FROM_LE:
748 switch (imm) {
749 case 16:
750 /* zero-extend 16 bits into 64 bits */
751 emit_insn(ctx, bstrpickd, dst, dst, 15, 0);
752 break;
753 case 32:
754 /* zero-extend 32 bits into 64 bits */
755 emit_zext_32(ctx, dst, is32);
756 break;
757 case 64:
758 /* do nothing */
759 break;
760 }
761 break;
762
763 case BPF_ALU | BPF_END | BPF_FROM_BE:
764 case BPF_ALU64 | BPF_END | BPF_FROM_LE:
765 switch (imm) {
766 case 16:
767 emit_insn(ctx, revb2h, dst, dst);
768 /* zero-extend 16 bits into 64 bits */
769 emit_insn(ctx, bstrpickd, dst, dst, 15, 0);
770 break;
771 case 32:
772 emit_insn(ctx, revb2w, dst, dst);
773 /* clear the upper 32 bits */
774 emit_zext_32(ctx, dst, true);
775 break;
776 case 64:
777 emit_insn(ctx, revbd, dst, dst);
778 break;
779 }
780 break;
781
782 /* PC += off if dst cond src */
783 case BPF_JMP | BPF_JEQ | BPF_X:
784 case BPF_JMP | BPF_JNE | BPF_X:
785 case BPF_JMP | BPF_JGT | BPF_X:
786 case BPF_JMP | BPF_JGE | BPF_X:
787 case BPF_JMP | BPF_JLT | BPF_X:
788 case BPF_JMP | BPF_JLE | BPF_X:
789 case BPF_JMP | BPF_JSGT | BPF_X:
790 case BPF_JMP | BPF_JSGE | BPF_X:
791 case BPF_JMP | BPF_JSLT | BPF_X:
792 case BPF_JMP | BPF_JSLE | BPF_X:
793 case BPF_JMP32 | BPF_JEQ | BPF_X:
794 case BPF_JMP32 | BPF_JNE | BPF_X:
795 case BPF_JMP32 | BPF_JGT | BPF_X:
796 case BPF_JMP32 | BPF_JGE | BPF_X:
797 case BPF_JMP32 | BPF_JLT | BPF_X:
798 case BPF_JMP32 | BPF_JLE | BPF_X:
799 case BPF_JMP32 | BPF_JSGT | BPF_X:
800 case BPF_JMP32 | BPF_JSGE | BPF_X:
801 case BPF_JMP32 | BPF_JSLT | BPF_X:
802 case BPF_JMP32 | BPF_JSLE | BPF_X:
803 jmp_offset = bpf2la_offset(i, off, ctx);
804 move_reg(ctx, t1, dst);
805 move_reg(ctx, t2, src);
806 if (is_signed_bpf_cond(BPF_OP(code))) {
807 emit_sext_32(ctx, t1, is32);
808 emit_sext_32(ctx, t2, is32);
809 } else {
810 emit_zext_32(ctx, t1, is32);
811 emit_zext_32(ctx, t2, is32);
812 }
813 if (emit_cond_jmp(ctx, cond, t1, t2, jmp_offset) < 0)
814 goto toofar;
815 break;
816
817 /* PC += off if dst cond imm */
818 case BPF_JMP | BPF_JEQ | BPF_K:
819 case BPF_JMP | BPF_JNE | BPF_K:
820 case BPF_JMP | BPF_JGT | BPF_K:
821 case BPF_JMP | BPF_JGE | BPF_K:
822 case BPF_JMP | BPF_JLT | BPF_K:
823 case BPF_JMP | BPF_JLE | BPF_K:
824 case BPF_JMP | BPF_JSGT | BPF_K:
825 case BPF_JMP | BPF_JSGE | BPF_K:
826 case BPF_JMP | BPF_JSLT | BPF_K:
827 case BPF_JMP | BPF_JSLE | BPF_K:
828 case BPF_JMP32 | BPF_JEQ | BPF_K:
829 case BPF_JMP32 | BPF_JNE | BPF_K:
830 case BPF_JMP32 | BPF_JGT | BPF_K:
831 case BPF_JMP32 | BPF_JGE | BPF_K:
832 case BPF_JMP32 | BPF_JLT | BPF_K:
833 case BPF_JMP32 | BPF_JLE | BPF_K:
834 case BPF_JMP32 | BPF_JSGT | BPF_K:
835 case BPF_JMP32 | BPF_JSGE | BPF_K:
836 case BPF_JMP32 | BPF_JSLT | BPF_K:
837 case BPF_JMP32 | BPF_JSLE | BPF_K:
838 jmp_offset = bpf2la_offset(i, off, ctx);
839 if (imm) {
840 move_imm(ctx, t1, imm, false);
841 tm = t1;
842 } else {
843 /* If imm is 0, simply use zero register. */
844 tm = LOONGARCH_GPR_ZERO;
845 }
846 move_reg(ctx, t2, dst);
847 if (is_signed_bpf_cond(BPF_OP(code))) {
848 emit_sext_32(ctx, tm, is32);
849 emit_sext_32(ctx, t2, is32);
850 } else {
851 emit_zext_32(ctx, tm, is32);
852 emit_zext_32(ctx, t2, is32);
853 }
854 if (emit_cond_jmp(ctx, cond, t2, tm, jmp_offset) < 0)
855 goto toofar;
856 break;
857
858 /* PC += off if dst & src */
859 case BPF_JMP | BPF_JSET | BPF_X:
860 case BPF_JMP32 | BPF_JSET | BPF_X:
861 jmp_offset = bpf2la_offset(i, off, ctx);
862 emit_insn(ctx, and, t1, dst, src);
863 emit_zext_32(ctx, t1, is32);
864 if (emit_cond_jmp(ctx, cond, t1, LOONGARCH_GPR_ZERO, jmp_offset) < 0)
865 goto toofar;
866 break;
867
868 /* PC += off if dst & imm */
869 case BPF_JMP | BPF_JSET | BPF_K:
870 case BPF_JMP32 | BPF_JSET | BPF_K:
871 jmp_offset = bpf2la_offset(i, off, ctx);
872 move_imm(ctx, t1, imm, is32);
873 emit_insn(ctx, and, t1, dst, t1);
874 emit_zext_32(ctx, t1, is32);
875 if (emit_cond_jmp(ctx, cond, t1, LOONGARCH_GPR_ZERO, jmp_offset) < 0)
876 goto toofar;
877 break;
878
879 /* PC += off */
880 case BPF_JMP | BPF_JA:
881 case BPF_JMP32 | BPF_JA:
882 if (BPF_CLASS(code) == BPF_JMP)
883 jmp_offset = bpf2la_offset(i, off, ctx);
884 else
885 jmp_offset = bpf2la_offset(i, imm, ctx);
886 if (emit_uncond_jmp(ctx, jmp_offset) < 0)
887 goto toofar;
888 break;
889
890 /* function call */
891 case BPF_JMP | BPF_CALL:
892 mark_call(ctx);
893 ret = bpf_jit_get_func_addr(ctx->prog, insn, extra_pass,
894 &func_addr, &func_addr_fixed);
895 if (ret < 0)
896 return ret;
897
898 move_addr(ctx, t1, func_addr);
899 emit_insn(ctx, jirl, LOONGARCH_GPR_RA, t1, 0);
900
901 if (insn->src_reg != BPF_PSEUDO_CALL)
902 move_reg(ctx, regmap[BPF_REG_0], LOONGARCH_GPR_A0);
903
904 break;
905
906 /* tail call */
907 case BPF_JMP | BPF_TAIL_CALL:
908 mark_tail_call(ctx);
909 if (emit_bpf_tail_call(ctx, i) < 0)
910 return -EINVAL;
911 break;
912
913 /* function return */
914 case BPF_JMP | BPF_EXIT:
915 if (i == ctx->prog->len - 1)
916 break;
917
918 jmp_offset = epilogue_offset(ctx);
919 if (emit_uncond_jmp(ctx, jmp_offset) < 0)
920 goto toofar;
921 break;
922
923 /* dst = imm64 */
924 case BPF_LD | BPF_IMM | BPF_DW:
925 {
926 const u64 imm64 = (u64)(insn + 1)->imm << 32 | (u32)insn->imm;
927
928 if (bpf_pseudo_func(insn))
929 move_addr(ctx, dst, imm64);
930 else
931 move_imm(ctx, dst, imm64, is32);
932 return 1;
933 }
934
935 /* dst = *(size *)(src + off) */
936 case BPF_LDX | BPF_MEM | BPF_B:
937 case BPF_LDX | BPF_MEM | BPF_H:
938 case BPF_LDX | BPF_MEM | BPF_W:
939 case BPF_LDX | BPF_MEM | BPF_DW:
940 case BPF_LDX | BPF_PROBE_MEM | BPF_DW:
941 case BPF_LDX | BPF_PROBE_MEM | BPF_W:
942 case BPF_LDX | BPF_PROBE_MEM | BPF_H:
943 case BPF_LDX | BPF_PROBE_MEM | BPF_B:
944 /* dst_reg = (s64)*(signed size *)(src_reg + off) */
945 case BPF_LDX | BPF_MEMSX | BPF_B:
946 case BPF_LDX | BPF_MEMSX | BPF_H:
947 case BPF_LDX | BPF_MEMSX | BPF_W:
948 case BPF_LDX | BPF_PROBE_MEMSX | BPF_B:
949 case BPF_LDX | BPF_PROBE_MEMSX | BPF_H:
950 case BPF_LDX | BPF_PROBE_MEMSX | BPF_W:
951 sign_extend = BPF_MODE(insn->code) == BPF_MEMSX ||
952 BPF_MODE(insn->code) == BPF_PROBE_MEMSX;
953 switch (BPF_SIZE(code)) {
954 case BPF_B:
955 if (is_signed_imm12(off)) {
956 if (sign_extend)
957 emit_insn(ctx, ldb, dst, src, off);
958 else
959 emit_insn(ctx, ldbu, dst, src, off);
960 } else {
961 move_imm(ctx, t1, off, is32);
962 if (sign_extend)
963 emit_insn(ctx, ldxb, dst, src, t1);
964 else
965 emit_insn(ctx, ldxbu, dst, src, t1);
966 }
967 break;
968 case BPF_H:
969 if (is_signed_imm12(off)) {
970 if (sign_extend)
971 emit_insn(ctx, ldh, dst, src, off);
972 else
973 emit_insn(ctx, ldhu, dst, src, off);
974 } else {
975 move_imm(ctx, t1, off, is32);
976 if (sign_extend)
977 emit_insn(ctx, ldxh, dst, src, t1);
978 else
979 emit_insn(ctx, ldxhu, dst, src, t1);
980 }
981 break;
982 case BPF_W:
983 if (is_signed_imm12(off)) {
984 if (sign_extend)
985 emit_insn(ctx, ldw, dst, src, off);
986 else
987 emit_insn(ctx, ldwu, dst, src, off);
988 } else {
989 move_imm(ctx, t1, off, is32);
990 if (sign_extend)
991 emit_insn(ctx, ldxw, dst, src, t1);
992 else
993 emit_insn(ctx, ldxwu, dst, src, t1);
994 }
995 break;
996 case BPF_DW:
997 move_imm(ctx, t1, off, is32);
998 emit_insn(ctx, ldxd, dst, src, t1);
999 break;
1000 }
1001
1002 ret = add_exception_handler(insn, ctx, dst);
1003 if (ret)
1004 return ret;
1005 break;
1006
1007 /* *(size *)(dst + off) = imm */
1008 case BPF_ST | BPF_MEM | BPF_B:
1009 case BPF_ST | BPF_MEM | BPF_H:
1010 case BPF_ST | BPF_MEM | BPF_W:
1011 case BPF_ST | BPF_MEM | BPF_DW:
1012 switch (BPF_SIZE(code)) {
1013 case BPF_B:
1014 move_imm(ctx, t1, imm, is32);
1015 if (is_signed_imm12(off)) {
1016 emit_insn(ctx, stb, t1, dst, off);
1017 } else {
1018 move_imm(ctx, t2, off, is32);
1019 emit_insn(ctx, stxb, t1, dst, t2);
1020 }
1021 break;
1022 case BPF_H:
1023 move_imm(ctx, t1, imm, is32);
1024 if (is_signed_imm12(off)) {
1025 emit_insn(ctx, sth, t1, dst, off);
1026 } else {
1027 move_imm(ctx, t2, off, is32);
1028 emit_insn(ctx, stxh, t1, dst, t2);
1029 }
1030 break;
1031 case BPF_W:
1032 move_imm(ctx, t1, imm, is32);
1033 if (is_signed_imm12(off)) {
1034 emit_insn(ctx, stw, t1, dst, off);
1035 } else if (is_signed_imm14(off)) {
1036 emit_insn(ctx, stptrw, t1, dst, off);
1037 } else {
1038 move_imm(ctx, t2, off, is32);
1039 emit_insn(ctx, stxw, t1, dst, t2);
1040 }
1041 break;
1042 case BPF_DW:
1043 move_imm(ctx, t1, imm, is32);
1044 if (is_signed_imm12(off)) {
1045 emit_insn(ctx, std, t1, dst, off);
1046 } else if (is_signed_imm14(off)) {
1047 emit_insn(ctx, stptrd, t1, dst, off);
1048 } else {
1049 move_imm(ctx, t2, off, is32);
1050 emit_insn(ctx, stxd, t1, dst, t2);
1051 }
1052 break;
1053 }
1054 break;
1055
1056 /* *(size *)(dst + off) = src */
1057 case BPF_STX | BPF_MEM | BPF_B:
1058 case BPF_STX | BPF_MEM | BPF_H:
1059 case BPF_STX | BPF_MEM | BPF_W:
1060 case BPF_STX | BPF_MEM | BPF_DW:
1061 switch (BPF_SIZE(code)) {
1062 case BPF_B:
1063 if (is_signed_imm12(off)) {
1064 emit_insn(ctx, stb, src, dst, off);
1065 } else {
1066 move_imm(ctx, t1, off, is32);
1067 emit_insn(ctx, stxb, src, dst, t1);
1068 }
1069 break;
1070 case BPF_H:
1071 if (is_signed_imm12(off)) {
1072 emit_insn(ctx, sth, src, dst, off);
1073 } else {
1074 move_imm(ctx, t1, off, is32);
1075 emit_insn(ctx, stxh, src, dst, t1);
1076 }
1077 break;
1078 case BPF_W:
1079 if (is_signed_imm12(off)) {
1080 emit_insn(ctx, stw, src, dst, off);
1081 } else if (is_signed_imm14(off)) {
1082 emit_insn(ctx, stptrw, src, dst, off);
1083 } else {
1084 move_imm(ctx, t1, off, is32);
1085 emit_insn(ctx, stxw, src, dst, t1);
1086 }
1087 break;
1088 case BPF_DW:
1089 if (is_signed_imm12(off)) {
1090 emit_insn(ctx, std, src, dst, off);
1091 } else if (is_signed_imm14(off)) {
1092 emit_insn(ctx, stptrd, src, dst, off);
1093 } else {
1094 move_imm(ctx, t1, off, is32);
1095 emit_insn(ctx, stxd, src, dst, t1);
1096 }
1097 break;
1098 }
1099 break;
1100
1101 case BPF_STX | BPF_ATOMIC | BPF_W:
1102 case BPF_STX | BPF_ATOMIC | BPF_DW:
1103 emit_atomic(insn, ctx);
1104 break;
1105
1106 /* Speculation barrier */
1107 case BPF_ST | BPF_NOSPEC:
1108 break;
1109
1110 default:
1111 pr_err("bpf_jit: unknown opcode %02x\n", code);
1112 return -EINVAL;
1113 }
1114
1115 return 0;
1116
1117 toofar:
1118 pr_info_once("bpf_jit: opcode %02x, jump too far\n", code);
1119 return -E2BIG;
1120 }
1121
build_body(struct jit_ctx * ctx,bool extra_pass)1122 static int build_body(struct jit_ctx *ctx, bool extra_pass)
1123 {
1124 int i;
1125 const struct bpf_prog *prog = ctx->prog;
1126
1127 for (i = 0; i < prog->len; i++) {
1128 const struct bpf_insn *insn = &prog->insnsi[i];
1129 int ret;
1130
1131 if (ctx->image == NULL)
1132 ctx->offset[i] = ctx->idx;
1133
1134 ret = build_insn(insn, ctx, extra_pass);
1135 if (ret > 0) {
1136 i++;
1137 if (ctx->image == NULL)
1138 ctx->offset[i] = ctx->idx;
1139 continue;
1140 }
1141 if (ret)
1142 return ret;
1143 }
1144
1145 if (ctx->image == NULL)
1146 ctx->offset[i] = ctx->idx;
1147
1148 return 0;
1149 }
1150
1151 /* Fill space with break instructions */
jit_fill_hole(void * area,unsigned int size)1152 static void jit_fill_hole(void *area, unsigned int size)
1153 {
1154 u32 *ptr;
1155
1156 /* We are guaranteed to have aligned memory */
1157 for (ptr = area; size >= sizeof(u32); size -= sizeof(u32))
1158 *ptr++ = INSN_BREAK;
1159 }
1160
validate_code(struct jit_ctx * ctx)1161 static int validate_code(struct jit_ctx *ctx)
1162 {
1163 int i;
1164 union loongarch_instruction insn;
1165
1166 for (i = 0; i < ctx->idx; i++) {
1167 insn = ctx->image[i];
1168 /* Check INSN_BREAK */
1169 if (insn.word == INSN_BREAK)
1170 return -1;
1171 }
1172
1173 if (WARN_ON_ONCE(ctx->num_exentries != ctx->prog->aux->num_exentries))
1174 return -1;
1175
1176 return 0;
1177 }
1178
bpf_int_jit_compile(struct bpf_prog * prog)1179 struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
1180 {
1181 bool tmp_blinded = false, extra_pass = false;
1182 u8 *image_ptr;
1183 int image_size, prog_size, extable_size;
1184 struct jit_ctx ctx;
1185 struct jit_data *jit_data;
1186 struct bpf_binary_header *header;
1187 struct bpf_prog *tmp, *orig_prog = prog;
1188
1189 /*
1190 * If BPF JIT was not enabled then we must fall back to
1191 * the interpreter.
1192 */
1193 if (!prog->jit_requested)
1194 return orig_prog;
1195
1196 tmp = bpf_jit_blind_constants(prog);
1197 /*
1198 * If blinding was requested and we failed during blinding,
1199 * we must fall back to the interpreter. Otherwise, we save
1200 * the new JITed code.
1201 */
1202 if (IS_ERR(tmp))
1203 return orig_prog;
1204
1205 if (tmp != prog) {
1206 tmp_blinded = true;
1207 prog = tmp;
1208 }
1209
1210 jit_data = prog->aux->jit_data;
1211 if (!jit_data) {
1212 jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL);
1213 if (!jit_data) {
1214 prog = orig_prog;
1215 goto out;
1216 }
1217 prog->aux->jit_data = jit_data;
1218 }
1219 if (jit_data->ctx.offset) {
1220 ctx = jit_data->ctx;
1221 image_ptr = jit_data->image;
1222 header = jit_data->header;
1223 extra_pass = true;
1224 prog_size = sizeof(u32) * ctx.idx;
1225 goto skip_init_ctx;
1226 }
1227
1228 memset(&ctx, 0, sizeof(ctx));
1229 ctx.prog = prog;
1230
1231 ctx.offset = kvcalloc(prog->len + 1, sizeof(u32), GFP_KERNEL);
1232 if (ctx.offset == NULL) {
1233 prog = orig_prog;
1234 goto out_offset;
1235 }
1236
1237 /* 1. Initial fake pass to compute ctx->idx and set ctx->flags */
1238 build_prologue(&ctx);
1239 if (build_body(&ctx, extra_pass)) {
1240 prog = orig_prog;
1241 goto out_offset;
1242 }
1243 ctx.epilogue_offset = ctx.idx;
1244 build_epilogue(&ctx);
1245
1246 extable_size = prog->aux->num_exentries * sizeof(struct exception_table_entry);
1247
1248 /* Now we know the actual image size.
1249 * As each LoongArch instruction is of length 32bit,
1250 * we are translating number of JITed intructions into
1251 * the size required to store these JITed code.
1252 */
1253 prog_size = sizeof(u32) * ctx.idx;
1254 image_size = prog_size + extable_size;
1255 /* Now we know the size of the structure to make */
1256 header = bpf_jit_binary_alloc(image_size, &image_ptr,
1257 sizeof(u32), jit_fill_hole);
1258 if (header == NULL) {
1259 prog = orig_prog;
1260 goto out_offset;
1261 }
1262
1263 /* 2. Now, the actual pass to generate final JIT code */
1264 ctx.image = (union loongarch_instruction *)image_ptr;
1265 if (extable_size)
1266 prog->aux->extable = (void *)image_ptr + prog_size;
1267
1268 skip_init_ctx:
1269 ctx.idx = 0;
1270 ctx.num_exentries = 0;
1271
1272 build_prologue(&ctx);
1273 if (build_body(&ctx, extra_pass)) {
1274 bpf_jit_binary_free(header);
1275 prog = orig_prog;
1276 goto out_offset;
1277 }
1278 build_epilogue(&ctx);
1279
1280 /* 3. Extra pass to validate JITed code */
1281 if (validate_code(&ctx)) {
1282 bpf_jit_binary_free(header);
1283 prog = orig_prog;
1284 goto out_offset;
1285 }
1286
1287 /* And we're done */
1288 if (bpf_jit_enable > 1)
1289 bpf_jit_dump(prog->len, prog_size, 2, ctx.image);
1290
1291 /* Update the icache */
1292 flush_icache_range((unsigned long)header, (unsigned long)(ctx.image + ctx.idx));
1293
1294 if (!prog->is_func || extra_pass) {
1295 int err;
1296
1297 if (extra_pass && ctx.idx != jit_data->ctx.idx) {
1298 pr_err_once("multi-func JIT bug %d != %d\n",
1299 ctx.idx, jit_data->ctx.idx);
1300 goto out_free;
1301 }
1302 err = bpf_jit_binary_lock_ro(header);
1303 if (err) {
1304 pr_err_once("bpf_jit_binary_lock_ro() returned %d\n",
1305 err);
1306 goto out_free;
1307 }
1308 } else {
1309 jit_data->ctx = ctx;
1310 jit_data->image = image_ptr;
1311 jit_data->header = header;
1312 }
1313 prog->jited = 1;
1314 prog->jited_len = prog_size;
1315 prog->bpf_func = (void *)ctx.image;
1316
1317 if (!prog->is_func || extra_pass) {
1318 int i;
1319
1320 /* offset[prog->len] is the size of program */
1321 for (i = 0; i <= prog->len; i++)
1322 ctx.offset[i] *= LOONGARCH_INSN_SIZE;
1323 bpf_prog_fill_jited_linfo(prog, ctx.offset + 1);
1324
1325 out_offset:
1326 kvfree(ctx.offset);
1327 kfree(jit_data);
1328 prog->aux->jit_data = NULL;
1329 }
1330
1331 out:
1332 if (tmp_blinded)
1333 bpf_jit_prog_release_other(prog, prog == orig_prog ? tmp : orig_prog);
1334
1335
1336 return prog;
1337
1338 out_free:
1339 bpf_jit_binary_free(header);
1340 prog->bpf_func = NULL;
1341 prog->jited = 0;
1342 prog->jited_len = 0;
1343 goto out_offset;
1344 }
1345
1346 /* Indicate the JIT backend supports mixing bpf2bpf and tailcalls. */
bpf_jit_supports_subprog_tailcalls(void)1347 bool bpf_jit_supports_subprog_tailcalls(void)
1348 {
1349 return true;
1350 }
1351