Lines Matching +full:data +full:- +full:out
1 // SPDX-License-Identifier: GPL-2.0-only
13 #include <linux/error-injection.h>
36 return -ENOMEM; in bpf_test_run()
56 ret = -EINTR; in bpf_test_run()
61 time_spent += ktime_get_ns() - time_start; in bpf_test_run()
73 time_spent += ktime_get_ns() - time_start; in bpf_test_run()
87 union bpf_attr __user *uattr, const void *data, in bpf_test_finish() argument
90 void __user *data_out = u64_to_user_ptr(kattr->test.data_out); in bpf_test_finish()
91 int err = -EFAULT; in bpf_test_finish()
97 if (kattr->test.data_size_out && in bpf_test_finish()
98 copy_size > kattr->test.data_size_out) { in bpf_test_finish()
99 copy_size = kattr->test.data_size_out; in bpf_test_finish()
100 err = -ENOSPC; in bpf_test_finish()
103 if (data_out && copy_to_user(data_out, data, copy_size)) in bpf_test_finish()
104 goto out; in bpf_test_finish()
105 if (copy_to_user(&uattr->test.data_size_out, &size, sizeof(size))) in bpf_test_finish()
106 goto out; in bpf_test_finish()
107 if (copy_to_user(&uattr->test.retval, &retval, sizeof(retval))) in bpf_test_finish()
108 goto out; in bpf_test_finish()
109 if (copy_to_user(&uattr->test.duration, &duration, sizeof(duration))) in bpf_test_finish()
110 goto out; in bpf_test_finish()
111 if (err != -ENOSPC) in bpf_test_finish()
113 out: in bpf_test_finish()
123 __diag_ignore(GCC, 8, "-Wmissing-prototypes",
166 return (long)arg->a; in bpf_fentry_test8()
181 void __user *data_in = u64_to_user_ptr(kattr->test.data_in); in bpf_test_init()
182 u32 user_size = kattr->test.data_size_in; in bpf_test_init()
183 void *data; in bpf_test_init() local
185 if (size < ETH_HLEN || size > PAGE_SIZE - headroom - tailroom) in bpf_test_init()
186 return ERR_PTR(-EINVAL); in bpf_test_init()
189 return ERR_PTR(-EMSGSIZE); in bpf_test_init()
191 data = kzalloc(size + headroom + tailroom, GFP_USER); in bpf_test_init()
192 if (!data) in bpf_test_init()
193 return ERR_PTR(-ENOMEM); in bpf_test_init()
195 if (copy_from_user(data + headroom, data_in, user_size)) { in bpf_test_init()
196 kfree(data); in bpf_test_init()
197 return ERR_PTR(-EFAULT); in bpf_test_init()
200 return data; in bpf_test_init()
209 int b = 2, err = -EFAULT; in bpf_prog_test_run_tracing()
212 if (kattr->test.flags || kattr->test.cpu) in bpf_prog_test_run_tracing()
213 return -EINVAL; in bpf_prog_test_run_tracing()
215 switch (prog->expected_attach_type) { in bpf_prog_test_run_tracing()
226 goto out; in bpf_prog_test_run_tracing()
234 goto out; in bpf_prog_test_run_tracing()
238 if (copy_to_user(&uattr->test.retval, &retval, sizeof(retval))) in bpf_prog_test_run_tracing()
239 goto out; in bpf_prog_test_run_tracing()
242 out: in bpf_prog_test_run_tracing()
254 __bpf_prog_test_run_raw_tp(void *data) in __bpf_prog_test_run_raw_tp() argument
256 struct bpf_raw_tp_test_run_info *info = data; in __bpf_prog_test_run_raw_tp()
259 info->retval = BPF_PROG_RUN(info->prog, info->ctx); in __bpf_prog_test_run_raw_tp()
267 void __user *ctx_in = u64_to_user_ptr(kattr->test.ctx_in); in bpf_prog_test_run_raw_tp()
268 __u32 ctx_size_in = kattr->test.ctx_size_in; in bpf_prog_test_run_raw_tp()
270 int cpu = kattr->test.cpu, err = 0; in bpf_prog_test_run_raw_tp()
273 /* doesn't support data_in/out, ctx_out, duration, or repeat */ in bpf_prog_test_run_raw_tp()
274 if (kattr->test.data_in || kattr->test.data_out || in bpf_prog_test_run_raw_tp()
275 kattr->test.ctx_out || kattr->test.duration || in bpf_prog_test_run_raw_tp()
276 kattr->test.repeat) in bpf_prog_test_run_raw_tp()
277 return -EINVAL; in bpf_prog_test_run_raw_tp()
279 if (ctx_size_in < prog->aux->max_ctx_offset || in bpf_prog_test_run_raw_tp()
281 return -EINVAL; in bpf_prog_test_run_raw_tp()
283 if ((kattr->test.flags & BPF_F_TEST_RUN_ON_CPU) == 0 && cpu != 0) in bpf_prog_test_run_raw_tp()
284 return -EINVAL; in bpf_prog_test_run_raw_tp()
289 return -ENOMEM; in bpf_prog_test_run_raw_tp()
291 err = -EFAULT; in bpf_prog_test_run_raw_tp()
292 goto out; in bpf_prog_test_run_raw_tp()
301 if ((kattr->test.flags & BPF_F_TEST_RUN_ON_CPU) == 0 || in bpf_prog_test_run_raw_tp()
307 * space, let's do an extra quick check to filter out in bpf_prog_test_run_raw_tp()
310 err = -ENXIO; in bpf_prog_test_run_raw_tp()
318 copy_to_user(&uattr->test.retval, &info.retval, sizeof(u32))) in bpf_prog_test_run_raw_tp()
319 err = -EFAULT; in bpf_prog_test_run_raw_tp()
321 out: in bpf_prog_test_run_raw_tp()
328 void __user *data_in = u64_to_user_ptr(kattr->test.ctx_in); in bpf_ctx_init()
329 void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out); in bpf_ctx_init()
330 u32 size = kattr->test.ctx_size_in; in bpf_ctx_init()
331 void *data; in bpf_ctx_init() local
337 data = kzalloc(max_size, GFP_USER); in bpf_ctx_init()
338 if (!data) in bpf_ctx_init()
339 return ERR_PTR(-ENOMEM); in bpf_ctx_init()
344 kfree(data); in bpf_ctx_init()
349 if (copy_from_user(data, data_in, size)) { in bpf_ctx_init()
350 kfree(data); in bpf_ctx_init()
351 return ERR_PTR(-EFAULT); in bpf_ctx_init()
354 return data; in bpf_ctx_init()
358 union bpf_attr __user *uattr, const void *data, in bpf_ctx_finish() argument
361 void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out); in bpf_ctx_finish()
362 int err = -EFAULT; in bpf_ctx_finish()
365 if (!data || !data_out) in bpf_ctx_finish()
368 if (copy_size > kattr->test.ctx_size_out) { in bpf_ctx_finish()
369 copy_size = kattr->test.ctx_size_out; in bpf_ctx_finish()
370 err = -ENOSPC; in bpf_ctx_finish()
373 if (copy_to_user(data_out, data, copy_size)) in bpf_ctx_finish()
374 goto out; in bpf_ctx_finish()
375 if (copy_to_user(&uattr->test.ctx_size_out, &size, sizeof(size))) in bpf_ctx_finish()
376 goto out; in bpf_ctx_finish()
377 if (err != -ENOSPC) in bpf_ctx_finish()
379 out: in bpf_ctx_finish()
384 * range_is_zero - test whether buffer is initialized
389 * This function returns true if the there is a non-zero byte
394 return !memchr_inv((u8 *)buf + from, 0, to - from); in range_is_zero()
399 struct qdisc_skb_cb *cb = (struct qdisc_skb_cb *)skb->cb; in convert___skb_to_skb()
406 return -EINVAL; in convert___skb_to_skb()
412 return -EINVAL; in convert___skb_to_skb()
418 return -EINVAL; in convert___skb_to_skb()
424 return -EINVAL; in convert___skb_to_skb()
430 return -EINVAL; in convert___skb_to_skb()
438 return -EINVAL; in convert___skb_to_skb()
444 return -EINVAL; in convert___skb_to_skb()
446 skb->mark = __skb->mark; in convert___skb_to_skb()
447 skb->priority = __skb->priority; in convert___skb_to_skb()
448 skb->tstamp = __skb->tstamp; in convert___skb_to_skb()
449 memcpy(&cb->data, __skb->cb, QDISC_CB_PRIV_LEN); in convert___skb_to_skb()
451 if (__skb->wire_len == 0) { in convert___skb_to_skb()
452 cb->pkt_len = skb->len; in convert___skb_to_skb()
454 if (__skb->wire_len < skb->len || in convert___skb_to_skb()
455 __skb->wire_len > GSO_MAX_SIZE) in convert___skb_to_skb()
456 return -EINVAL; in convert___skb_to_skb()
457 cb->pkt_len = __skb->wire_len; in convert___skb_to_skb()
460 if (__skb->gso_segs > GSO_MAX_SEGS) in convert___skb_to_skb()
461 return -EINVAL; in convert___skb_to_skb()
462 skb_shinfo(skb)->gso_segs = __skb->gso_segs; in convert___skb_to_skb()
463 skb_shinfo(skb)->gso_size = __skb->gso_size; in convert___skb_to_skb()
470 struct qdisc_skb_cb *cb = (struct qdisc_skb_cb *)skb->cb; in convert_skb_to___skb()
475 __skb->mark = skb->mark; in convert_skb_to___skb()
476 __skb->priority = skb->priority; in convert_skb_to___skb()
477 __skb->ifindex = skb->dev->ifindex; in convert_skb_to___skb()
478 __skb->tstamp = skb->tstamp; in convert_skb_to___skb()
479 memcpy(__skb->cb, &cb->data, QDISC_CB_PRIV_LEN); in convert_skb_to___skb()
480 __skb->wire_len = cb->pkt_len; in convert_skb_to___skb()
481 __skb->gso_segs = skb_shinfo(skb)->gso_segs; in convert_skb_to___skb()
494 struct net *net = current->nsproxy->net_ns; in bpf_prog_test_run_skb()
495 struct net_device *dev = net->loopback_dev; in bpf_prog_test_run_skb()
496 u32 size = kattr->test.data_size_in; in bpf_prog_test_run_skb()
497 u32 repeat = kattr->test.repeat; in bpf_prog_test_run_skb()
503 void *data; in bpf_prog_test_run_skb() local
506 if (kattr->test.flags || kattr->test.cpu) in bpf_prog_test_run_skb()
507 return -EINVAL; in bpf_prog_test_run_skb()
509 data = bpf_test_init(kattr, size, NET_SKB_PAD + NET_IP_ALIGN, in bpf_prog_test_run_skb()
511 if (IS_ERR(data)) in bpf_prog_test_run_skb()
512 return PTR_ERR(data); in bpf_prog_test_run_skb()
516 kfree(data); in bpf_prog_test_run_skb()
520 switch (prog->type) { in bpf_prog_test_run_skb()
536 kfree(data); in bpf_prog_test_run_skb()
538 return -ENOMEM; in bpf_prog_test_run_skb()
542 skb = build_skb(data, 0); in bpf_prog_test_run_skb()
544 kfree(data); in bpf_prog_test_run_skb()
547 return -ENOMEM; in bpf_prog_test_run_skb()
549 skb->sk = sk; in bpf_prog_test_run_skb()
553 if (ctx && ctx->ifindex > 1) { in bpf_prog_test_run_skb()
554 dev = dev_get_by_index(net, ctx->ifindex); in bpf_prog_test_run_skb()
556 ret = -ENODEV; in bpf_prog_test_run_skb()
557 goto out; in bpf_prog_test_run_skb()
560 skb->protocol = eth_type_trans(skb, dev); in bpf_prog_test_run_skb()
563 switch (skb->protocol) { in bpf_prog_test_run_skb()
565 sk->sk_family = AF_INET; in bpf_prog_test_run_skb()
567 sk->sk_rcv_saddr = ip_hdr(skb)->saddr; in bpf_prog_test_run_skb()
568 sk->sk_daddr = ip_hdr(skb)->daddr; in bpf_prog_test_run_skb()
573 sk->sk_family = AF_INET6; in bpf_prog_test_run_skb()
575 sk->sk_v6_rcv_saddr = ipv6_hdr(skb)->saddr; in bpf_prog_test_run_skb()
576 sk->sk_v6_daddr = ipv6_hdr(skb)->daddr; in bpf_prog_test_run_skb()
590 goto out; in bpf_prog_test_run_skb()
593 goto out; in bpf_prog_test_run_skb()
596 int nhead = HH_DATA_ALIGN(hh_len - skb_headroom(skb)); in bpf_prog_test_run_skb()
599 ret = -ENOMEM; in bpf_prog_test_run_skb()
600 goto out; in bpf_prog_test_run_skb()
607 size = skb->len; in bpf_prog_test_run_skb()
608 /* bpf program can never convert linear skb to non-linear */ in bpf_prog_test_run_skb()
611 ret = bpf_test_finish(kattr, uattr, skb->data, size, retval, duration); in bpf_prog_test_run_skb()
615 out: in bpf_prog_test_run_skb()
616 if (dev && dev != net->loopback_dev) in bpf_prog_test_run_skb()
629 u32 size = kattr->test.data_size_in; in bpf_prog_test_run_xdp()
630 u32 repeat = kattr->test.repeat; in bpf_prog_test_run_xdp()
635 void *data; in bpf_prog_test_run_xdp() local
638 if (prog->expected_attach_type == BPF_XDP_DEVMAP || in bpf_prog_test_run_xdp()
639 prog->expected_attach_type == BPF_XDP_CPUMAP) in bpf_prog_test_run_xdp()
640 return -EINVAL; in bpf_prog_test_run_xdp()
641 if (kattr->test.ctx_in || kattr->test.ctx_out) in bpf_prog_test_run_xdp()
642 return -EINVAL; in bpf_prog_test_run_xdp()
645 max_data_sz = 4096 - headroom - tailroom; in bpf_prog_test_run_xdp()
647 data = bpf_test_init(kattr, max_data_sz, headroom, tailroom); in bpf_prog_test_run_xdp()
648 if (IS_ERR(data)) in bpf_prog_test_run_xdp()
649 return PTR_ERR(data); in bpf_prog_test_run_xdp()
651 xdp.data_hard_start = data; in bpf_prog_test_run_xdp()
652 xdp.data = data + headroom; in bpf_prog_test_run_xdp()
653 xdp.data_meta = xdp.data; in bpf_prog_test_run_xdp()
654 xdp.data_end = xdp.data + size; in bpf_prog_test_run_xdp()
657 rxqueue = __netif_get_rx_queue(current->nsproxy->net_ns->loopback_dev, 0); in bpf_prog_test_run_xdp()
658 xdp.rxq = &rxqueue->xdp_rxq; in bpf_prog_test_run_xdp()
662 goto out; in bpf_prog_test_run_xdp()
663 if (xdp.data != data + headroom || xdp.data_end != xdp.data + size) in bpf_prog_test_run_xdp()
664 size = xdp.data_end - xdp.data; in bpf_prog_test_run_xdp()
665 ret = bpf_test_finish(kattr, uattr, xdp.data, size, retval, duration); in bpf_prog_test_run_xdp()
666 out: in bpf_prog_test_run_xdp()
668 kfree(data); in bpf_prog_test_run_xdp()
676 return -EINVAL; in verify_user_bpf_flow_keys()
682 return -EINVAL; in verify_user_bpf_flow_keys()
691 u32 size = kattr->test.data_size_in; in bpf_prog_test_run_flow_dissector()
693 u32 repeat = kattr->test.repeat; in bpf_prog_test_run_flow_dissector()
700 void *data; in bpf_prog_test_run_flow_dissector() local
704 if (prog->type != BPF_PROG_TYPE_FLOW_DISSECTOR) in bpf_prog_test_run_flow_dissector()
705 return -EINVAL; in bpf_prog_test_run_flow_dissector()
707 if (kattr->test.flags || kattr->test.cpu) in bpf_prog_test_run_flow_dissector()
708 return -EINVAL; in bpf_prog_test_run_flow_dissector()
711 return -EINVAL; in bpf_prog_test_run_flow_dissector()
713 data = bpf_test_init(kattr, size, 0, 0); in bpf_prog_test_run_flow_dissector()
714 if (IS_ERR(data)) in bpf_prog_test_run_flow_dissector()
715 return PTR_ERR(data); in bpf_prog_test_run_flow_dissector()
717 eth = (struct ethhdr *)data; in bpf_prog_test_run_flow_dissector()
724 kfree(data); in bpf_prog_test_run_flow_dissector()
730 goto out; in bpf_prog_test_run_flow_dissector()
731 flags = user_ctx->flags; in bpf_prog_test_run_flow_dissector()
735 ctx.data = data; in bpf_prog_test_run_flow_dissector()
736 ctx.data_end = (__u8 *)data + size; in bpf_prog_test_run_flow_dissector()
742 retval = bpf_flow_dissect(prog, &ctx, eth->h_proto, ETH_HLEN, in bpf_prog_test_run_flow_dissector()
749 ret = -EINTR; in bpf_prog_test_run_flow_dissector()
750 goto out; in bpf_prog_test_run_flow_dissector()
754 time_spent += ktime_get_ns() - time_start; in bpf_prog_test_run_flow_dissector()
765 time_spent += ktime_get_ns() - time_start; in bpf_prog_test_run_flow_dissector()
778 out: in bpf_prog_test_run_flow_dissector()
780 kfree(data); in bpf_prog_test_run_flow_dissector()