1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * eBPF JIT compiler
4 *
5 * Copyright 2016 Naveen N. Rao <naveen.n.rao@linux.vnet.ibm.com>
6 * IBM Corporation
7 *
8 * Based on the powerpc classic BPF JIT compiler by Matt Evans
9 */
10 #include <linux/moduleloader.h>
11 #include <asm/cacheflush.h>
12 #include <asm/asm-compat.h>
13 #include <linux/netdevice.h>
14 #include <linux/filter.h>
15 #include <linux/if_vlan.h>
16 #include <asm/kprobes.h>
17 #include <linux/bpf.h>
18
19 #include "bpf_jit.h"
20
bpf_jit_fill_ill_insns(void * area,unsigned int size)21 static void bpf_jit_fill_ill_insns(void *area, unsigned int size)
22 {
23 memset32(area, BREAKPOINT_INSTRUCTION, size / 4);
24 }
25
26 /* Fix updated addresses (for subprog calls, ldimm64, et al) during extra pass */
bpf_jit_fixup_addresses(struct bpf_prog * fp,u32 * image,struct codegen_context * ctx,u32 * addrs)27 static int bpf_jit_fixup_addresses(struct bpf_prog *fp, u32 *image,
28 struct codegen_context *ctx, u32 *addrs)
29 {
30 const struct bpf_insn *insn = fp->insnsi;
31 bool func_addr_fixed;
32 u64 func_addr;
33 u32 tmp_idx;
34 int i, j, ret;
35
36 for (i = 0; i < fp->len; i++) {
37 /*
38 * During the extra pass, only the branch target addresses for
39 * the subprog calls need to be fixed. All other instructions
40 * can left untouched.
41 *
42 * The JITed image length does not change because we already
43 * ensure that the JITed instruction sequence for these calls
44 * are of fixed length by padding them with NOPs.
45 */
46 if (insn[i].code == (BPF_JMP | BPF_CALL) &&
47 insn[i].src_reg == BPF_PSEUDO_CALL) {
48 ret = bpf_jit_get_func_addr(fp, &insn[i], true,
49 &func_addr,
50 &func_addr_fixed);
51 if (ret < 0)
52 return ret;
53
54 /*
55 * Save ctx->idx as this would currently point to the
56 * end of the JITed image and set it to the offset of
57 * the instruction sequence corresponding to the
58 * subprog call temporarily.
59 */
60 tmp_idx = ctx->idx;
61 ctx->idx = addrs[i] / 4;
62 bpf_jit_emit_func_call_rel(image, ctx, func_addr);
63
64 /*
65 * Restore ctx->idx here. This is safe as the length
66 * of the JITed sequence remains unchanged.
67 */
68 ctx->idx = tmp_idx;
69 } else if (insn[i].code == (BPF_LD | BPF_IMM | BPF_DW)) {
70 tmp_idx = ctx->idx;
71 ctx->idx = addrs[i] / 4;
72 #ifdef CONFIG_PPC32
73 PPC_LI32(ctx->b2p[insn[i].dst_reg] - 1, (u32)insn[i + 1].imm);
74 PPC_LI32(ctx->b2p[insn[i].dst_reg], (u32)insn[i].imm);
75 for (j = ctx->idx - addrs[i] / 4; j < 4; j++)
76 EMIT(PPC_RAW_NOP());
77 #else
78 func_addr = ((u64)(u32)insn[i].imm) | (((u64)(u32)insn[i + 1].imm) << 32);
79 PPC_LI64(b2p[insn[i].dst_reg], func_addr);
80 /* overwrite rest with nops */
81 for (j = ctx->idx - addrs[i] / 4; j < 5; j++)
82 EMIT(PPC_RAW_NOP());
83 #endif
84 ctx->idx = tmp_idx;
85 i++;
86 }
87 }
88
89 return 0;
90 }
91
92 struct powerpc64_jit_data {
93 struct bpf_binary_header *header;
94 u32 *addrs;
95 u8 *image;
96 u32 proglen;
97 struct codegen_context ctx;
98 };
99
bpf_jit_needs_zext(void)100 bool bpf_jit_needs_zext(void)
101 {
102 return true;
103 }
104
bpf_int_jit_compile(struct bpf_prog * fp)105 struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
106 {
107 u32 proglen;
108 u32 alloclen;
109 u8 *image = NULL;
110 u32 *code_base;
111 u32 *addrs;
112 struct powerpc64_jit_data *jit_data;
113 struct codegen_context cgctx;
114 int pass;
115 int flen;
116 struct bpf_binary_header *bpf_hdr;
117 struct bpf_prog *org_fp = fp;
118 struct bpf_prog *tmp_fp;
119 bool bpf_blinded = false;
120 bool extra_pass = false;
121
122 if (!fp->jit_requested)
123 return org_fp;
124
125 tmp_fp = bpf_jit_blind_constants(org_fp);
126 if (IS_ERR(tmp_fp))
127 return org_fp;
128
129 if (tmp_fp != org_fp) {
130 bpf_blinded = true;
131 fp = tmp_fp;
132 }
133
134 jit_data = fp->aux->jit_data;
135 if (!jit_data) {
136 jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL);
137 if (!jit_data) {
138 fp = org_fp;
139 goto out;
140 }
141 fp->aux->jit_data = jit_data;
142 }
143
144 flen = fp->len;
145 addrs = jit_data->addrs;
146 if (addrs) {
147 cgctx = jit_data->ctx;
148 image = jit_data->image;
149 bpf_hdr = jit_data->header;
150 proglen = jit_data->proglen;
151 alloclen = proglen + FUNCTION_DESCR_SIZE;
152 extra_pass = true;
153 goto skip_init_ctx;
154 }
155
156 addrs = kcalloc(flen + 1, sizeof(*addrs), GFP_KERNEL);
157 if (addrs == NULL) {
158 fp = org_fp;
159 goto out_addrs;
160 }
161
162 memset(&cgctx, 0, sizeof(struct codegen_context));
163 memcpy(cgctx.b2p, b2p, sizeof(cgctx.b2p));
164
165 /* Make sure that the stack is quadword aligned. */
166 cgctx.stack_size = round_up(fp->aux->stack_depth, 16);
167
168 /* Scouting faux-generate pass 0 */
169 if (bpf_jit_build_body(fp, 0, &cgctx, addrs, false)) {
170 /* We hit something illegal or unsupported. */
171 fp = org_fp;
172 goto out_addrs;
173 }
174
175 /*
176 * If we have seen a tail call, we need a second pass.
177 * This is because bpf_jit_emit_common_epilogue() is called
178 * from bpf_jit_emit_tail_call() with a not yet stable ctx->seen.
179 */
180 if (cgctx.seen & SEEN_TAILCALL) {
181 cgctx.idx = 0;
182 if (bpf_jit_build_body(fp, 0, &cgctx, addrs, false)) {
183 fp = org_fp;
184 goto out_addrs;
185 }
186 }
187
188 bpf_jit_realloc_regs(&cgctx);
189 /*
190 * Pretend to build prologue, given the features we've seen. This will
191 * update ctgtx.idx as it pretends to output instructions, then we can
192 * calculate total size from idx.
193 */
194 bpf_jit_build_prologue(0, &cgctx);
195 bpf_jit_build_epilogue(0, &cgctx);
196
197 proglen = cgctx.idx * 4;
198 alloclen = proglen + FUNCTION_DESCR_SIZE;
199
200 bpf_hdr = bpf_jit_binary_alloc(alloclen, &image, 4, bpf_jit_fill_ill_insns);
201 if (!bpf_hdr) {
202 fp = org_fp;
203 goto out_addrs;
204 }
205
206 skip_init_ctx:
207 code_base = (u32 *)(image + FUNCTION_DESCR_SIZE);
208
209 if (extra_pass) {
210 /*
211 * Do not touch the prologue and epilogue as they will remain
212 * unchanged. Only fix the branch target address for subprog
213 * calls in the body, and ldimm64 instructions.
214 *
215 * This does not change the offsets and lengths of the subprog
216 * call instruction sequences and hence, the size of the JITed
217 * image as well.
218 */
219 bpf_jit_fixup_addresses(fp, code_base, &cgctx, addrs);
220
221 /* There is no need to perform the usual passes. */
222 goto skip_codegen_passes;
223 }
224
225 /* Code generation passes 1-2 */
226 for (pass = 1; pass < 3; pass++) {
227 /* Now build the prologue, body code & epilogue for real. */
228 cgctx.idx = 0;
229 bpf_jit_build_prologue(code_base, &cgctx);
230 if (bpf_jit_build_body(fp, code_base, &cgctx, addrs, extra_pass)) {
231 bpf_jit_binary_free(bpf_hdr);
232 fp = org_fp;
233 goto out_addrs;
234 }
235 bpf_jit_build_epilogue(code_base, &cgctx);
236
237 if (bpf_jit_enable > 1)
238 pr_info("Pass %d: shrink = %d, seen = 0x%x\n", pass,
239 proglen - (cgctx.idx * 4), cgctx.seen);
240 }
241
242 skip_codegen_passes:
243 if (bpf_jit_enable > 1)
244 /*
245 * Note that we output the base address of the code_base
246 * rather than image, since opcodes are in code_base.
247 */
248 bpf_jit_dump(flen, proglen, pass, code_base);
249
250 #ifdef PPC64_ELF_ABI_v1
251 /* Function descriptor nastiness: Address + TOC */
252 ((u64 *)image)[0] = (u64)code_base;
253 ((u64 *)image)[1] = local_paca->kernel_toc;
254 #endif
255
256 fp->bpf_func = (void *)image;
257 fp->jited = 1;
258 fp->jited_len = alloclen;
259
260 bpf_flush_icache(bpf_hdr, (u8 *)bpf_hdr + (bpf_hdr->pages * PAGE_SIZE));
261 if (!fp->is_func || extra_pass) {
262 bpf_jit_binary_lock_ro(bpf_hdr);
263 bpf_prog_fill_jited_linfo(fp, addrs);
264 out_addrs:
265 kfree(addrs);
266 kfree(jit_data);
267 fp->aux->jit_data = NULL;
268 } else {
269 jit_data->addrs = addrs;
270 jit_data->ctx = cgctx;
271 jit_data->proglen = proglen;
272 jit_data->image = image;
273 jit_data->header = bpf_hdr;
274 }
275
276 out:
277 if (bpf_blinded)
278 bpf_jit_prog_release_other(fp, fp == org_fp ? tmp_fp : org_fp);
279
280 return fp;
281 }
282