1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (c) 2017 Facebook
3 */
4 #include <linux/bpf.h>
5 #include <linux/slab.h>
6 #include <linux/vmalloc.h>
7 #include <linux/etherdevice.h>
8 #include <linux/filter.h>
9 #include <linux/sched/signal.h>
10 #include <net/bpf_sk_storage.h>
11 #include <net/sock.h>
12 #include <net/tcp.h>
13
14 #define CREATE_TRACE_POINTS
15 #include <trace/events/bpf_test_run.h>
16
bpf_test_run(struct bpf_prog * prog,void * ctx,u32 repeat,u32 * retval,u32 * time)17 static int bpf_test_run(struct bpf_prog *prog, void *ctx, u32 repeat,
18 u32 *retval, u32 *time)
19 {
20 struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE] = { NULL };
21 enum bpf_cgroup_storage_type stype;
22 u64 time_start, time_spent = 0;
23 int ret = 0;
24 u32 i;
25
26 for_each_cgroup_storage_type(stype) {
27 storage[stype] = bpf_cgroup_storage_alloc(prog, stype);
28 if (IS_ERR(storage[stype])) {
29 storage[stype] = NULL;
30 for_each_cgroup_storage_type(stype)
31 bpf_cgroup_storage_free(storage[stype]);
32 return -ENOMEM;
33 }
34 }
35
36 if (!repeat)
37 repeat = 1;
38
39 rcu_read_lock();
40 preempt_disable();
41 time_start = ktime_get_ns();
42 for (i = 0; i < repeat; i++) {
43 bpf_cgroup_storage_set(storage);
44 *retval = BPF_PROG_RUN(prog, ctx);
45
46 if (signal_pending(current)) {
47 ret = -EINTR;
48 break;
49 }
50
51 if (need_resched()) {
52 time_spent += ktime_get_ns() - time_start;
53 preempt_enable();
54 rcu_read_unlock();
55
56 cond_resched();
57
58 rcu_read_lock();
59 preempt_disable();
60 time_start = ktime_get_ns();
61 }
62 }
63 time_spent += ktime_get_ns() - time_start;
64 preempt_enable();
65 rcu_read_unlock();
66
67 do_div(time_spent, repeat);
68 *time = time_spent > U32_MAX ? U32_MAX : (u32)time_spent;
69
70 for_each_cgroup_storage_type(stype)
71 bpf_cgroup_storage_free(storage[stype]);
72
73 return ret;
74 }
75
bpf_test_finish(const union bpf_attr * kattr,union bpf_attr __user * uattr,const void * data,u32 size,u32 retval,u32 duration)76 static int bpf_test_finish(const union bpf_attr *kattr,
77 union bpf_attr __user *uattr, const void *data,
78 u32 size, u32 retval, u32 duration)
79 {
80 void __user *data_out = u64_to_user_ptr(kattr->test.data_out);
81 int err = -EFAULT;
82 u32 copy_size = size;
83
84 /* Clamp copy if the user has provided a size hint, but copy the full
85 * buffer if not to retain old behaviour.
86 */
87 if (kattr->test.data_size_out &&
88 copy_size > kattr->test.data_size_out) {
89 copy_size = kattr->test.data_size_out;
90 err = -ENOSPC;
91 }
92
93 if (data_out && copy_to_user(data_out, data, copy_size))
94 goto out;
95 if (copy_to_user(&uattr->test.data_size_out, &size, sizeof(size)))
96 goto out;
97 if (copy_to_user(&uattr->test.retval, &retval, sizeof(retval)))
98 goto out;
99 if (copy_to_user(&uattr->test.duration, &duration, sizeof(duration)))
100 goto out;
101 if (err != -ENOSPC)
102 err = 0;
103 out:
104 trace_bpf_test_finish(&err);
105 return err;
106 }
107
bpf_test_init(const union bpf_attr * kattr,u32 size,u32 headroom,u32 tailroom)108 static void *bpf_test_init(const union bpf_attr *kattr, u32 size,
109 u32 headroom, u32 tailroom)
110 {
111 void __user *data_in = u64_to_user_ptr(kattr->test.data_in);
112 void *data;
113
114 if (size < ETH_HLEN || size > PAGE_SIZE - headroom - tailroom)
115 return ERR_PTR(-EINVAL);
116
117 size = SKB_DATA_ALIGN(size);
118 data = kzalloc(size + headroom + tailroom, GFP_USER);
119 if (!data)
120 return ERR_PTR(-ENOMEM);
121
122 if (copy_from_user(data + headroom, data_in, size)) {
123 kfree(data);
124 return ERR_PTR(-EFAULT);
125 }
126 return data;
127 }
128
bpf_ctx_init(const union bpf_attr * kattr,u32 max_size)129 static void *bpf_ctx_init(const union bpf_attr *kattr, u32 max_size)
130 {
131 void __user *data_in = u64_to_user_ptr(kattr->test.ctx_in);
132 void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out);
133 u32 size = kattr->test.ctx_size_in;
134 void *data;
135 int err;
136
137 if (!data_in && !data_out)
138 return NULL;
139
140 data = kzalloc(max_size, GFP_USER);
141 if (!data)
142 return ERR_PTR(-ENOMEM);
143
144 if (data_in) {
145 err = bpf_check_uarg_tail_zero(data_in, max_size, size);
146 if (err) {
147 kfree(data);
148 return ERR_PTR(err);
149 }
150
151 size = min_t(u32, max_size, size);
152 if (copy_from_user(data, data_in, size)) {
153 kfree(data);
154 return ERR_PTR(-EFAULT);
155 }
156 }
157 return data;
158 }
159
bpf_ctx_finish(const union bpf_attr * kattr,union bpf_attr __user * uattr,const void * data,u32 size)160 static int bpf_ctx_finish(const union bpf_attr *kattr,
161 union bpf_attr __user *uattr, const void *data,
162 u32 size)
163 {
164 void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out);
165 int err = -EFAULT;
166 u32 copy_size = size;
167
168 if (!data || !data_out)
169 return 0;
170
171 if (copy_size > kattr->test.ctx_size_out) {
172 copy_size = kattr->test.ctx_size_out;
173 err = -ENOSPC;
174 }
175
176 if (copy_to_user(data_out, data, copy_size))
177 goto out;
178 if (copy_to_user(&uattr->test.ctx_size_out, &size, sizeof(size)))
179 goto out;
180 if (err != -ENOSPC)
181 err = 0;
182 out:
183 return err;
184 }
185
186 /**
187 * range_is_zero - test whether buffer is initialized
188 * @buf: buffer to check
189 * @from: check from this position
190 * @to: check up until (excluding) this position
191 *
192 * This function returns true if the there is a non-zero byte
193 * in the buf in the range [from,to).
194 */
range_is_zero(void * buf,size_t from,size_t to)195 static inline bool range_is_zero(void *buf, size_t from, size_t to)
196 {
197 return !memchr_inv((u8 *)buf + from, 0, to - from);
198 }
199
convert___skb_to_skb(struct sk_buff * skb,struct __sk_buff * __skb)200 static int convert___skb_to_skb(struct sk_buff *skb, struct __sk_buff *__skb)
201 {
202 struct qdisc_skb_cb *cb = (struct qdisc_skb_cb *)skb->cb;
203
204 if (!__skb)
205 return 0;
206
207 /* make sure the fields we don't use are zeroed */
208 if (!range_is_zero(__skb, 0, offsetof(struct __sk_buff, priority)))
209 return -EINVAL;
210
211 /* priority is allowed */
212
213 if (!range_is_zero(__skb, offsetof(struct __sk_buff, priority) +
214 FIELD_SIZEOF(struct __sk_buff, priority),
215 offsetof(struct __sk_buff, cb)))
216 return -EINVAL;
217
218 /* cb is allowed */
219
220 if (!range_is_zero(__skb, offsetof(struct __sk_buff, cb) +
221 FIELD_SIZEOF(struct __sk_buff, cb),
222 sizeof(struct __sk_buff)))
223 return -EINVAL;
224
225 skb->priority = __skb->priority;
226 memcpy(&cb->data, __skb->cb, QDISC_CB_PRIV_LEN);
227
228 return 0;
229 }
230
convert_skb_to___skb(struct sk_buff * skb,struct __sk_buff * __skb)231 static void convert_skb_to___skb(struct sk_buff *skb, struct __sk_buff *__skb)
232 {
233 struct qdisc_skb_cb *cb = (struct qdisc_skb_cb *)skb->cb;
234
235 if (!__skb)
236 return;
237
238 __skb->priority = skb->priority;
239 memcpy(__skb->cb, &cb->data, QDISC_CB_PRIV_LEN);
240 }
241
bpf_prog_test_run_skb(struct bpf_prog * prog,const union bpf_attr * kattr,union bpf_attr __user * uattr)242 int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr,
243 union bpf_attr __user *uattr)
244 {
245 bool is_l2 = false, is_direct_pkt_access = false;
246 u32 size = kattr->test.data_size_in;
247 u32 repeat = kattr->test.repeat;
248 struct __sk_buff *ctx = NULL;
249 u32 retval, duration;
250 int hh_len = ETH_HLEN;
251 struct sk_buff *skb;
252 struct sock *sk;
253 void *data;
254 int ret;
255
256 data = bpf_test_init(kattr, size, NET_SKB_PAD + NET_IP_ALIGN,
257 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)));
258 if (IS_ERR(data))
259 return PTR_ERR(data);
260
261 ctx = bpf_ctx_init(kattr, sizeof(struct __sk_buff));
262 if (IS_ERR(ctx)) {
263 kfree(data);
264 return PTR_ERR(ctx);
265 }
266
267 switch (prog->type) {
268 case BPF_PROG_TYPE_SCHED_CLS:
269 case BPF_PROG_TYPE_SCHED_ACT:
270 is_l2 = true;
271 /* fall through */
272 case BPF_PROG_TYPE_LWT_IN:
273 case BPF_PROG_TYPE_LWT_OUT:
274 case BPF_PROG_TYPE_LWT_XMIT:
275 is_direct_pkt_access = true;
276 break;
277 default:
278 break;
279 }
280
281 sk = kzalloc(sizeof(struct sock), GFP_USER);
282 if (!sk) {
283 kfree(data);
284 kfree(ctx);
285 return -ENOMEM;
286 }
287 sock_net_set(sk, current->nsproxy->net_ns);
288 sock_init_data(NULL, sk);
289
290 skb = build_skb(data, 0);
291 if (!skb) {
292 kfree(data);
293 kfree(ctx);
294 kfree(sk);
295 return -ENOMEM;
296 }
297 skb->sk = sk;
298
299 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN);
300 __skb_put(skb, size);
301 skb->protocol = eth_type_trans(skb, current->nsproxy->net_ns->loopback_dev);
302 skb_reset_network_header(skb);
303
304 if (is_l2)
305 __skb_push(skb, hh_len);
306 if (is_direct_pkt_access)
307 bpf_compute_data_pointers(skb);
308 ret = convert___skb_to_skb(skb, ctx);
309 if (ret)
310 goto out;
311 ret = bpf_test_run(prog, skb, repeat, &retval, &duration);
312 if (ret)
313 goto out;
314 if (!is_l2) {
315 if (skb_headroom(skb) < hh_len) {
316 int nhead = HH_DATA_ALIGN(hh_len - skb_headroom(skb));
317
318 if (pskb_expand_head(skb, nhead, 0, GFP_USER)) {
319 ret = -ENOMEM;
320 goto out;
321 }
322 }
323 memset(__skb_push(skb, hh_len), 0, hh_len);
324 }
325 convert_skb_to___skb(skb, ctx);
326
327 size = skb->len;
328 /* bpf program can never convert linear skb to non-linear */
329 if (WARN_ON_ONCE(skb_is_nonlinear(skb)))
330 size = skb_headlen(skb);
331 ret = bpf_test_finish(kattr, uattr, skb->data, size, retval, duration);
332 if (!ret)
333 ret = bpf_ctx_finish(kattr, uattr, ctx,
334 sizeof(struct __sk_buff));
335 out:
336 kfree_skb(skb);
337 bpf_sk_storage_free(sk);
338 kfree(sk);
339 kfree(ctx);
340 return ret;
341 }
342
bpf_prog_test_run_xdp(struct bpf_prog * prog,const union bpf_attr * kattr,union bpf_attr __user * uattr)343 int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr,
344 union bpf_attr __user *uattr)
345 {
346 u32 size = kattr->test.data_size_in;
347 u32 repeat = kattr->test.repeat;
348 struct netdev_rx_queue *rxqueue;
349 struct xdp_buff xdp = {};
350 u32 retval, duration;
351 void *data;
352 int ret;
353
354 if (kattr->test.ctx_in || kattr->test.ctx_out)
355 return -EINVAL;
356
357 data = bpf_test_init(kattr, size, XDP_PACKET_HEADROOM + NET_IP_ALIGN, 0);
358 if (IS_ERR(data))
359 return PTR_ERR(data);
360
361 xdp.data_hard_start = data;
362 xdp.data = data + XDP_PACKET_HEADROOM + NET_IP_ALIGN;
363 xdp.data_meta = xdp.data;
364 xdp.data_end = xdp.data + size;
365
366 rxqueue = __netif_get_rx_queue(current->nsproxy->net_ns->loopback_dev, 0);
367 xdp.rxq = &rxqueue->xdp_rxq;
368
369 ret = bpf_test_run(prog, &xdp, repeat, &retval, &duration);
370 if (ret)
371 goto out;
372 if (xdp.data != data + XDP_PACKET_HEADROOM + NET_IP_ALIGN ||
373 xdp.data_end != xdp.data + size)
374 size = xdp.data_end - xdp.data;
375 ret = bpf_test_finish(kattr, uattr, xdp.data, size, retval, duration);
376 out:
377 kfree(data);
378 return ret;
379 }
380
verify_user_bpf_flow_keys(struct bpf_flow_keys * ctx)381 static int verify_user_bpf_flow_keys(struct bpf_flow_keys *ctx)
382 {
383 /* make sure the fields we don't use are zeroed */
384 if (!range_is_zero(ctx, 0, offsetof(struct bpf_flow_keys, flags)))
385 return -EINVAL;
386
387 /* flags is allowed */
388
389 if (!range_is_zero(ctx, offsetof(struct bpf_flow_keys, flags) +
390 FIELD_SIZEOF(struct bpf_flow_keys, flags),
391 sizeof(struct bpf_flow_keys)))
392 return -EINVAL;
393
394 return 0;
395 }
396
bpf_prog_test_run_flow_dissector(struct bpf_prog * prog,const union bpf_attr * kattr,union bpf_attr __user * uattr)397 int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog,
398 const union bpf_attr *kattr,
399 union bpf_attr __user *uattr)
400 {
401 u32 size = kattr->test.data_size_in;
402 struct bpf_flow_dissector ctx = {};
403 u32 repeat = kattr->test.repeat;
404 struct bpf_flow_keys *user_ctx;
405 struct bpf_flow_keys flow_keys;
406 u64 time_start, time_spent = 0;
407 const struct ethhdr *eth;
408 unsigned int flags = 0;
409 u32 retval, duration;
410 void *data;
411 int ret;
412 u32 i;
413
414 if (prog->type != BPF_PROG_TYPE_FLOW_DISSECTOR)
415 return -EINVAL;
416
417 if (size < ETH_HLEN)
418 return -EINVAL;
419
420 data = bpf_test_init(kattr, size, 0, 0);
421 if (IS_ERR(data))
422 return PTR_ERR(data);
423
424 eth = (struct ethhdr *)data;
425
426 if (!repeat)
427 repeat = 1;
428
429 user_ctx = bpf_ctx_init(kattr, sizeof(struct bpf_flow_keys));
430 if (IS_ERR(user_ctx)) {
431 kfree(data);
432 return PTR_ERR(user_ctx);
433 }
434 if (user_ctx) {
435 ret = verify_user_bpf_flow_keys(user_ctx);
436 if (ret)
437 goto out;
438 flags = user_ctx->flags;
439 }
440
441 ctx.flow_keys = &flow_keys;
442 ctx.data = data;
443 ctx.data_end = (__u8 *)data + size;
444
445 rcu_read_lock();
446 preempt_disable();
447 time_start = ktime_get_ns();
448 for (i = 0; i < repeat; i++) {
449 retval = bpf_flow_dissect(prog, &ctx, eth->h_proto, ETH_HLEN,
450 size, flags);
451
452 if (signal_pending(current)) {
453 preempt_enable();
454 rcu_read_unlock();
455
456 ret = -EINTR;
457 goto out;
458 }
459
460 if (need_resched()) {
461 time_spent += ktime_get_ns() - time_start;
462 preempt_enable();
463 rcu_read_unlock();
464
465 cond_resched();
466
467 rcu_read_lock();
468 preempt_disable();
469 time_start = ktime_get_ns();
470 }
471 }
472 time_spent += ktime_get_ns() - time_start;
473 preempt_enable();
474 rcu_read_unlock();
475
476 do_div(time_spent, repeat);
477 duration = time_spent > U32_MAX ? U32_MAX : (u32)time_spent;
478
479 ret = bpf_test_finish(kattr, uattr, &flow_keys, sizeof(flow_keys),
480 retval, duration);
481 if (!ret)
482 ret = bpf_ctx_finish(kattr, uattr, user_ctx,
483 sizeof(struct bpf_flow_keys));
484
485 out:
486 kfree(user_ctx);
487 kfree(data);
488 return ret;
489 }
490