Lines Matching +full:t +full:- +full:head
1 // SPDX-License-Identifier: GPL-2.0-only
19 #include <linux/error-injection.h>
36 static void bpf_test_timer_enter(struct bpf_test_timer *t) in bpf_test_timer_enter() argument
40 if (t->mode == NO_PREEMPT) in bpf_test_timer_enter()
45 t->time_start = ktime_get_ns(); in bpf_test_timer_enter()
48 static void bpf_test_timer_leave(struct bpf_test_timer *t) in bpf_test_timer_leave() argument
51 t->time_start = 0; in bpf_test_timer_leave()
53 if (t->mode == NO_PREEMPT) in bpf_test_timer_leave()
60 static bool bpf_test_timer_continue(struct bpf_test_timer *t, int iterations, in bpf_test_timer_continue() argument
64 t->i += iterations; in bpf_test_timer_continue()
65 if (t->i >= repeat) { in bpf_test_timer_continue()
67 t->time_spent += ktime_get_ns() - t->time_start; in bpf_test_timer_continue()
68 do_div(t->time_spent, t->i); in bpf_test_timer_continue()
69 *duration = t->time_spent > U32_MAX ? U32_MAX : (u32)t->time_spent; in bpf_test_timer_continue()
76 *err = -EINTR; in bpf_test_timer_continue()
82 t->time_spent += ktime_get_ns() - t->time_start; in bpf_test_timer_continue()
83 bpf_test_timer_leave(t); in bpf_test_timer_continue()
85 bpf_test_timer_enter(t); in bpf_test_timer_continue()
92 t->i = 0; in bpf_test_timer_continue()
96 /* We put this struct at the head of each page with a context and frame
97 * initialised when the page is allocated, so we don't have to do this on each
126 #define TEST_XDP_FRAME_SIZE (PAGE_SIZE - sizeof(struct xdp_page_head))
131 struct xdp_page_head *head = phys_to_virt(page_to_phys(page)); in xdp_test_run_init_page() local
139 orig_ctx = xdp->orig_ctx; in xdp_test_run_init_page()
140 frm_len = orig_ctx->data_end - orig_ctx->data_meta; in xdp_test_run_init_page()
141 meta_len = orig_ctx->data - orig_ctx->data_meta; in xdp_test_run_init_page()
142 headroom -= meta_len; in xdp_test_run_init_page()
144 new_ctx = &head->ctx; in xdp_test_run_init_page()
145 frm = head->frame; in xdp_test_run_init_page()
146 data = head->data; in xdp_test_run_init_page()
147 memcpy(data + headroom, orig_ctx->data_meta, frm_len); in xdp_test_run_init_page()
149 xdp_init_buff(new_ctx, TEST_XDP_FRAME_SIZE, &xdp->rxq); in xdp_test_run_init_page()
151 new_ctx->data = new_ctx->data_meta + meta_len; in xdp_test_run_init_page()
154 frm->mem = new_ctx->rxq->mem; in xdp_test_run_init_page()
156 memcpy(&head->orig_ctx, new_ctx, sizeof(head->orig_ctx)); in xdp_test_run_init_page()
162 int err = -ENOMEM; in xdp_test_run_setup()
166 .pool_size = xdp->batch_size, in xdp_test_run_setup()
172 xdp->frames = kvmalloc_array(xdp->batch_size, sizeof(void *), GFP_KERNEL); in xdp_test_run_setup()
173 if (!xdp->frames) in xdp_test_run_setup()
174 return -ENOMEM; in xdp_test_run_setup()
176 xdp->skbs = kvmalloc_array(xdp->batch_size, sizeof(void *), GFP_KERNEL); in xdp_test_run_setup()
177 if (!xdp->skbs) in xdp_test_run_setup()
186 /* will copy 'mem.id' into pp->xdp_mem_id */ in xdp_test_run_setup()
187 err = xdp_reg_mem_model(&xdp->mem, MEM_TYPE_PAGE_POOL, pp); in xdp_test_run_setup()
191 xdp->pp = pp; in xdp_test_run_setup()
196 xdp_rxq_info_reg(&xdp->rxq, orig_ctx->rxq->dev, 0, 0); in xdp_test_run_setup()
197 xdp->rxq.mem.type = MEM_TYPE_PAGE_POOL; in xdp_test_run_setup()
198 xdp->rxq.mem.id = pp->xdp_mem_id; in xdp_test_run_setup()
199 xdp->dev = orig_ctx->rxq->dev; in xdp_test_run_setup()
200 xdp->orig_ctx = orig_ctx; in xdp_test_run_setup()
207 kvfree(xdp->skbs); in xdp_test_run_setup()
209 kvfree(xdp->frames); in xdp_test_run_setup()
215 xdp_unreg_mem_model(&xdp->mem); in xdp_test_run_teardown()
216 page_pool_destroy(xdp->pp); in xdp_test_run_teardown()
217 kfree(xdp->frames); in xdp_test_run_teardown()
218 kfree(xdp->skbs); in xdp_test_run_teardown()
221 static bool frame_was_changed(const struct xdp_page_head *head) in frame_was_changed() argument
227 return head->frame->data != head->orig_ctx.data || in frame_was_changed()
228 head->frame->flags != head->orig_ctx.flags; in frame_was_changed()
231 static bool ctx_was_changed(struct xdp_page_head *head) in ctx_was_changed() argument
233 return head->orig_ctx.data != head->ctx.data || in ctx_was_changed()
234 head->orig_ctx.data_meta != head->ctx.data_meta || in ctx_was_changed()
235 head->orig_ctx.data_end != head->ctx.data_end; in ctx_was_changed()
238 static void reset_ctx(struct xdp_page_head *head) in reset_ctx() argument
240 if (likely(!frame_was_changed(head) && !ctx_was_changed(head))) in reset_ctx()
243 head->ctx.data = head->orig_ctx.data; in reset_ctx()
244 head->ctx.data_meta = head->orig_ctx.data_meta; in reset_ctx()
245 head->ctx.data_end = head->orig_ctx.data_end; in reset_ctx()
246 xdp_update_frame_from_buff(&head->ctx, head->frame); in reset_ctx()
247 head->frame->mem = head->orig_ctx.rxq->mem; in reset_ctx()
262 return -ENOMEM; in xdp_recv_frames()
275 list_add_tail(&skb->list, &list); in xdp_recv_frames()
287 struct xdp_frame **frames = xdp->frames; in xdp_test_run_batch()
288 struct xdp_page_head *head; in xdp_test_run_batch() local
294 batch_sz = min_t(u32, repeat, xdp->batch_size); in xdp_test_run_batch()
300 page = page_pool_dev_alloc_pages(xdp->pp); in xdp_test_run_batch()
302 err = -ENOMEM; in xdp_test_run_batch()
306 head = phys_to_virt(page_to_phys(page)); in xdp_test_run_batch()
307 reset_ctx(head); in xdp_test_run_batch()
308 ctx = &head->ctx; in xdp_test_run_batch()
309 frm = head->frame; in xdp_test_run_batch()
310 xdp->frame_cnt++; in xdp_test_run_batch()
315 if (unlikely(ctx_was_changed(head))) { in xdp_test_run_batch()
325 /* we can't do a real XDP_TX since we're not in the in xdp_test_run_batch()
329 ri->tgt_index = xdp->dev->ifindex; in xdp_test_run_batch()
330 ri->map_id = INT_MAX; in xdp_test_run_batch()
331 ri->map_type = BPF_MAP_TYPE_UNSPEC; in xdp_test_run_batch()
335 ret = xdp_do_redirect_frame(xdp->dev, ctx, frm, prog); in xdp_test_run_batch()
355 ret = xdp_recv_frames(frames, nframes, xdp->skbs, xdp->dev); in xdp_test_run_batch()
370 struct bpf_test_timer t = { .mode = NO_MIGRATE }; in bpf_test_run_xdp_live() local
380 bpf_test_timer_enter(&t); in bpf_test_run_xdp_live()
383 ret = xdp_test_run_batch(&xdp, prog, repeat - t.i); in bpf_test_run_xdp_live()
386 } while (bpf_test_timer_continue(&t, xdp.frame_cnt, repeat, &ret, time)); in bpf_test_run_xdp_live()
387 bpf_test_timer_leave(&t); in bpf_test_run_xdp_live()
399 struct bpf_test_timer t = { NO_MIGRATE }; in bpf_test_run() local
409 return -ENOMEM; in bpf_test_run()
416 bpf_test_timer_enter(&t); in bpf_test_run()
426 } while (bpf_test_timer_continue(&t, 1, repeat, &ret, time)); in bpf_test_run()
428 bpf_test_timer_leave(&t); in bpf_test_run()
441 void __user *data_out = u64_to_user_ptr(kattr->test.data_out); in bpf_test_finish()
442 int err = -EFAULT; in bpf_test_finish()
448 if (kattr->test.data_size_out && in bpf_test_finish()
449 copy_size > kattr->test.data_size_out) { in bpf_test_finish()
450 copy_size = kattr->test.data_size_out; in bpf_test_finish()
451 err = -ENOSPC; in bpf_test_finish()
455 int len = sinfo ? copy_size - sinfo->xdp_frags_size : copy_size; in bpf_test_finish()
458 err = -ENOSPC; in bpf_test_finish()
469 for (i = 0; i < sinfo->nr_frags; i++) { in bpf_test_finish()
470 skb_frag_t *frag = &sinfo->frags[i]; in bpf_test_finish()
473 err = -ENOSPC; in bpf_test_finish()
477 data_len = min_t(u32, copy_size - offset, in bpf_test_finish()
490 if (copy_to_user(&uattr->test.data_size_out, &size, sizeof(size))) in bpf_test_finish()
492 if (copy_to_user(&uattr->test.retval, &retval, sizeof(retval))) in bpf_test_finish()
494 if (copy_to_user(&uattr->test.duration, &duration, sizeof(duration))) in bpf_test_finish()
496 if (err != -ENOSPC) in bpf_test_finish()
508 __diag_ignore_all("-Wmissing-prototypes",
553 return (long)arg->a; in bpf_fentry_test8()
602 refcount_dec(&p->cnt); in bpf_kfunc_call_test_release()
630 void __user *data_in = u64_to_user_ptr(kattr->test.data_in); in BTF_ID_FLAGS()
633 if (user_size < ETH_HLEN || user_size > PAGE_SIZE - headroom - tailroom) in BTF_ID_FLAGS()
634 return ERR_PTR(-EINVAL); in BTF_ID_FLAGS()
639 return ERR_PTR(-ENOMEM); in BTF_ID_FLAGS()
643 return ERR_PTR(-EFAULT); in BTF_ID_FLAGS()
655 int b = 2, err = -EFAULT; in bpf_prog_test_run_tracing()
658 if (kattr->test.flags || kattr->test.cpu || kattr->test.batch_size) in bpf_prog_test_run_tracing()
659 return -EINVAL; in bpf_prog_test_run_tracing()
661 switch (prog->expected_attach_type) { in bpf_prog_test_run_tracing()
689 if (copy_to_user(&uattr->test.retval, &retval, sizeof(retval))) in bpf_prog_test_run_tracing()
714 info->retval = bpf_prog_run(info->prog, info->ctx); in __bpf_prog_test_run_raw_tp()
724 void __user *ctx_in = u64_to_user_ptr(kattr->test.ctx_in); in bpf_prog_test_run_raw_tp()
725 __u32 ctx_size_in = kattr->test.ctx_size_in; in bpf_prog_test_run_raw_tp()
727 int cpu = kattr->test.cpu, err = 0; in bpf_prog_test_run_raw_tp()
730 /* doesn't support data_in/out, ctx_out, duration, or repeat */ in bpf_prog_test_run_raw_tp()
731 if (kattr->test.data_in || kattr->test.data_out || in bpf_prog_test_run_raw_tp()
732 kattr->test.ctx_out || kattr->test.duration || in bpf_prog_test_run_raw_tp()
733 kattr->test.repeat || kattr->test.batch_size) in bpf_prog_test_run_raw_tp()
734 return -EINVAL; in bpf_prog_test_run_raw_tp()
736 if (ctx_size_in < prog->aux->max_ctx_offset || in bpf_prog_test_run_raw_tp()
738 return -EINVAL; in bpf_prog_test_run_raw_tp()
740 if ((kattr->test.flags & BPF_F_TEST_RUN_ON_CPU) == 0 && cpu != 0) in bpf_prog_test_run_raw_tp()
741 return -EINVAL; in bpf_prog_test_run_raw_tp()
754 if ((kattr->test.flags & BPF_F_TEST_RUN_ON_CPU) == 0 || in bpf_prog_test_run_raw_tp()
763 err = -ENXIO; in bpf_prog_test_run_raw_tp()
771 copy_to_user(&uattr->test.retval, &info.retval, sizeof(u32))) in bpf_prog_test_run_raw_tp()
772 err = -EFAULT; in bpf_prog_test_run_raw_tp()
780 void __user *data_in = u64_to_user_ptr(kattr->test.ctx_in); in bpf_ctx_init()
781 void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out); in bpf_ctx_init()
782 u32 size = kattr->test.ctx_size_in; in bpf_ctx_init()
791 return ERR_PTR(-ENOMEM); in bpf_ctx_init()
803 return ERR_PTR(-EFAULT); in bpf_ctx_init()
813 void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out); in bpf_ctx_finish()
814 int err = -EFAULT; in bpf_ctx_finish()
820 if (copy_size > kattr->test.ctx_size_out) { in bpf_ctx_finish()
821 copy_size = kattr->test.ctx_size_out; in bpf_ctx_finish()
822 err = -ENOSPC; in bpf_ctx_finish()
827 if (copy_to_user(&uattr->test.ctx_size_out, &size, sizeof(size))) in bpf_ctx_finish()
829 if (err != -ENOSPC) in bpf_ctx_finish()
836 * range_is_zero - test whether buffer is initialized
841 * This function returns true if the there is a non-zero byte
846 return !memchr_inv((u8 *)buf + from, 0, to - from); in range_is_zero()
851 struct qdisc_skb_cb *cb = (struct qdisc_skb_cb *)skb->cb; in convert___skb_to_skb()
856 /* make sure the fields we don't use are zeroed */ in convert___skb_to_skb()
858 return -EINVAL; in convert___skb_to_skb()
864 return -EINVAL; in convert___skb_to_skb()
872 return -EINVAL; in convert___skb_to_skb()
878 return -EINVAL; in convert___skb_to_skb()
886 return -EINVAL; in convert___skb_to_skb()
892 return -EINVAL; in convert___skb_to_skb()
898 return -EINVAL; in convert___skb_to_skb()
900 skb->mark = __skb->mark; in convert___skb_to_skb()
901 skb->priority = __skb->priority; in convert___skb_to_skb()
902 skb->skb_iif = __skb->ingress_ifindex; in convert___skb_to_skb()
903 skb->tstamp = __skb->tstamp; in convert___skb_to_skb()
904 memcpy(&cb->data, __skb->cb, QDISC_CB_PRIV_LEN); in convert___skb_to_skb()
906 if (__skb->wire_len == 0) { in convert___skb_to_skb()
907 cb->pkt_len = skb->len; in convert___skb_to_skb()
909 if (__skb->wire_len < skb->len || in convert___skb_to_skb()
910 __skb->wire_len > GSO_LEGACY_MAX_SIZE) in convert___skb_to_skb()
911 return -EINVAL; in convert___skb_to_skb()
912 cb->pkt_len = __skb->wire_len; in convert___skb_to_skb()
915 if (__skb->gso_segs > GSO_MAX_SEGS) in convert___skb_to_skb()
916 return -EINVAL; in convert___skb_to_skb()
917 skb_shinfo(skb)->gso_segs = __skb->gso_segs; in convert___skb_to_skb()
918 skb_shinfo(skb)->gso_size = __skb->gso_size; in convert___skb_to_skb()
919 skb_shinfo(skb)->hwtstamps.hwtstamp = __skb->hwtstamp; in convert___skb_to_skb()
926 struct qdisc_skb_cb *cb = (struct qdisc_skb_cb *)skb->cb; in convert_skb_to___skb()
931 __skb->mark = skb->mark; in convert_skb_to___skb()
932 __skb->priority = skb->priority; in convert_skb_to___skb()
933 __skb->ingress_ifindex = skb->skb_iif; in convert_skb_to___skb()
934 __skb->ifindex = skb->dev->ifindex; in convert_skb_to___skb()
935 __skb->tstamp = skb->tstamp; in convert_skb_to___skb()
936 memcpy(__skb->cb, &cb->data, QDISC_CB_PRIV_LEN); in convert_skb_to___skb()
937 __skb->wire_len = cb->pkt_len; in convert_skb_to___skb()
938 __skb->gso_segs = skb_shinfo(skb)->gso_segs; in convert_skb_to___skb()
939 __skb->hwtstamp = skb_shinfo(skb)->hwtstamps.hwtstamp; in convert_skb_to___skb()
952 struct net *net = current->nsproxy->net_ns; in bpf_prog_test_run_skb()
953 struct net_device *dev = net->loopback_dev; in bpf_prog_test_run_skb()
954 u32 size = kattr->test.data_size_in; in bpf_prog_test_run_skb()
955 u32 repeat = kattr->test.repeat; in bpf_prog_test_run_skb()
964 if (kattr->test.flags || kattr->test.cpu || kattr->test.batch_size) in bpf_prog_test_run_skb()
965 return -EINVAL; in bpf_prog_test_run_skb()
967 data = bpf_test_init(kattr, kattr->test.data_size_in, in bpf_prog_test_run_skb()
979 switch (prog->type) { in bpf_prog_test_run_skb()
997 return -ENOMEM; in bpf_prog_test_run_skb()
1006 return -ENOMEM; in bpf_prog_test_run_skb()
1008 skb->sk = sk; in bpf_prog_test_run_skb()
1012 if (ctx && ctx->ifindex > 1) { in bpf_prog_test_run_skb()
1013 dev = dev_get_by_index(net, ctx->ifindex); in bpf_prog_test_run_skb()
1015 ret = -ENODEV; in bpf_prog_test_run_skb()
1019 skb->protocol = eth_type_trans(skb, dev); in bpf_prog_test_run_skb()
1022 switch (skb->protocol) { in bpf_prog_test_run_skb()
1024 sk->sk_family = AF_INET; in bpf_prog_test_run_skb()
1026 sk->sk_rcv_saddr = ip_hdr(skb)->saddr; in bpf_prog_test_run_skb()
1027 sk->sk_daddr = ip_hdr(skb)->daddr; in bpf_prog_test_run_skb()
1032 sk->sk_family = AF_INET6; in bpf_prog_test_run_skb()
1034 sk->sk_v6_rcv_saddr = ipv6_hdr(skb)->saddr; in bpf_prog_test_run_skb()
1035 sk->sk_v6_daddr = ipv6_hdr(skb)->daddr; in bpf_prog_test_run_skb()
1055 int nhead = HH_DATA_ALIGN(hh_len - skb_headroom(skb)); in bpf_prog_test_run_skb()
1058 ret = -ENOMEM; in bpf_prog_test_run_skb()
1066 size = skb->len; in bpf_prog_test_run_skb()
1067 /* bpf program can never convert linear skb to non-linear */ in bpf_prog_test_run_skb()
1070 ret = bpf_test_finish(kattr, uattr, skb->data, NULL, size, retval, in bpf_prog_test_run_skb()
1076 if (dev && dev != net->loopback_dev) in bpf_prog_test_run_skb()
1093 if (xdp_md->egress_ifindex != 0) in xdp_convert_md_to_buff()
1094 return -EINVAL; in xdp_convert_md_to_buff()
1096 ingress_ifindex = xdp_md->ingress_ifindex; in xdp_convert_md_to_buff()
1097 rx_queue_index = xdp_md->rx_queue_index; in xdp_convert_md_to_buff()
1100 return -EINVAL; in xdp_convert_md_to_buff()
1103 device = dev_get_by_index(current->nsproxy->net_ns, in xdp_convert_md_to_buff()
1106 return -ENODEV; in xdp_convert_md_to_buff()
1108 if (rx_queue_index >= device->real_num_rx_queues) in xdp_convert_md_to_buff()
1113 if (!xdp_rxq_info_is_reg(&rxqueue->xdp_rxq)) in xdp_convert_md_to_buff()
1116 xdp->rxq = &rxqueue->xdp_rxq; in xdp_convert_md_to_buff()
1117 /* The device is now tracked in the xdp->rxq for later in xdp_convert_md_to_buff()
1122 xdp->data = xdp->data_meta + xdp_md->data; in xdp_convert_md_to_buff()
1127 return -EINVAL; in xdp_convert_md_to_buff()
1135 xdp_md->data = xdp->data - xdp->data_meta; in xdp_convert_buff_to_md()
1136 xdp_md->data_end = xdp->data_end - xdp->data_meta; in xdp_convert_buff_to_md()
1138 if (xdp_md->ingress_ifindex) in xdp_convert_buff_to_md()
1139 dev_put(xdp->rxq->dev); in xdp_convert_buff_to_md()
1145 bool do_live = (kattr->test.flags & BPF_F_TEST_XDP_LIVE_FRAMES); in bpf_prog_test_run_xdp()
1147 u32 batch_size = kattr->test.batch_size; in bpf_prog_test_run_xdp()
1149 u32 size = kattr->test.data_size_in; in bpf_prog_test_run_xdp()
1151 u32 repeat = kattr->test.repeat; in bpf_prog_test_run_xdp()
1155 int i, ret = -EINVAL; in bpf_prog_test_run_xdp()
1159 if (prog->expected_attach_type == BPF_XDP_DEVMAP || in bpf_prog_test_run_xdp()
1160 prog->expected_attach_type == BPF_XDP_CPUMAP) in bpf_prog_test_run_xdp()
1161 return -EINVAL; in bpf_prog_test_run_xdp()
1163 if (kattr->test.flags & ~BPF_F_TEST_XDP_LIVE_FRAMES) in bpf_prog_test_run_xdp()
1164 return -EINVAL; in bpf_prog_test_run_xdp()
1166 if (bpf_prog_is_dev_bound(prog->aux)) in bpf_prog_test_run_xdp()
1167 return -EINVAL; in bpf_prog_test_run_xdp()
1173 return -E2BIG; in bpf_prog_test_run_xdp()
1177 return -EINVAL; in bpf_prog_test_run_xdp()
1185 /* There can't be user provided data before the meta data */ in bpf_prog_test_run_xdp()
1186 if (ctx->data_meta || ctx->data_end != size || in bpf_prog_test_run_xdp()
1187 ctx->data > ctx->data_end || in bpf_prog_test_run_xdp()
1188 unlikely(xdp_metalen_invalid(ctx->data)) || in bpf_prog_test_run_xdp()
1189 (do_live && (kattr->test.data_out || kattr->test.ctx_out))) in bpf_prog_test_run_xdp()
1192 headroom -= ctx->data; in bpf_prog_test_run_xdp()
1195 max_data_sz = 4096 - headroom - tailroom; in bpf_prog_test_run_xdp()
1209 rxqueue = __netif_get_rx_queue(current->nsproxy->net_ns->loopback_dev, 0); in bpf_prog_test_run_xdp()
1210 rxqueue->xdp_rxq.frag_size = headroom + max_data_sz + tailroom; in bpf_prog_test_run_xdp()
1211 xdp_init_buff(&xdp, rxqueue->xdp_rxq.frag_size, &rxqueue->xdp_rxq); in bpf_prog_test_run_xdp()
1219 if (unlikely(kattr->test.data_size_in > size)) { in bpf_prog_test_run_xdp()
1220 void __user *data_in = u64_to_user_ptr(kattr->test.data_in); in bpf_prog_test_run_xdp()
1222 while (size < kattr->test.data_size_in) { in bpf_prog_test_run_xdp()
1227 if (sinfo->nr_frags == MAX_SKB_FRAGS) { in bpf_prog_test_run_xdp()
1228 ret = -ENOMEM; in bpf_prog_test_run_xdp()
1234 ret = -ENOMEM; in bpf_prog_test_run_xdp()
1238 frag = &sinfo->frags[sinfo->nr_frags++]; in bpf_prog_test_run_xdp()
1240 data_len = min_t(u32, kattr->test.data_size_in - size, in bpf_prog_test_run_xdp()
1246 ret = -EFAULT; in bpf_prog_test_run_xdp()
1249 sinfo->xdp_frags_size += data_len; in bpf_prog_test_run_xdp()
1270 size = xdp.data_end - xdp.data_meta + sinfo->xdp_frags_size; in bpf_prog_test_run_xdp()
1281 for (i = 0; i < sinfo->nr_frags; i++) in bpf_prog_test_run_xdp()
1282 __free_page(skb_frag_page(&sinfo->frags[i])); in bpf_prog_test_run_xdp()
1291 /* make sure the fields we don't use are zeroed */ in verify_user_bpf_flow_keys()
1293 return -EINVAL; in verify_user_bpf_flow_keys()
1299 return -EINVAL; in verify_user_bpf_flow_keys()
1308 struct bpf_test_timer t = { NO_PREEMPT }; in bpf_prog_test_run_flow_dissector() local
1309 u32 size = kattr->test.data_size_in; in bpf_prog_test_run_flow_dissector()
1311 u32 repeat = kattr->test.repeat; in bpf_prog_test_run_flow_dissector()
1320 if (kattr->test.flags || kattr->test.cpu || kattr->test.batch_size) in bpf_prog_test_run_flow_dissector()
1321 return -EINVAL; in bpf_prog_test_run_flow_dissector()
1324 return -EINVAL; in bpf_prog_test_run_flow_dissector()
1326 data = bpf_test_init(kattr, kattr->test.data_size_in, size, 0, 0); in bpf_prog_test_run_flow_dissector()
1344 flags = user_ctx->flags; in bpf_prog_test_run_flow_dissector()
1351 bpf_test_timer_enter(&t); in bpf_prog_test_run_flow_dissector()
1353 retval = bpf_flow_dissect(prog, &ctx, eth->h_proto, ETH_HLEN, in bpf_prog_test_run_flow_dissector()
1355 } while (bpf_test_timer_continue(&t, 1, repeat, &ret, &duration)); in bpf_prog_test_run_flow_dissector()
1356 bpf_test_timer_leave(&t); in bpf_prog_test_run_flow_dissector()
1376 struct bpf_test_timer t = { NO_PREEMPT }; in bpf_prog_test_run_sk_lookup() local
1379 u32 repeat = kattr->test.repeat; in bpf_prog_test_run_sk_lookup()
1382 int ret = -EINVAL; in bpf_prog_test_run_sk_lookup()
1384 if (kattr->test.flags || kattr->test.cpu || kattr->test.batch_size) in bpf_prog_test_run_sk_lookup()
1385 return -EINVAL; in bpf_prog_test_run_sk_lookup()
1387 if (kattr->test.data_in || kattr->test.data_size_in || kattr->test.data_out || in bpf_prog_test_run_sk_lookup()
1388 kattr->test.data_size_out) in bpf_prog_test_run_sk_lookup()
1389 return -EINVAL; in bpf_prog_test_run_sk_lookup()
1399 return -EINVAL; in bpf_prog_test_run_sk_lookup()
1401 if (user_ctx->sk) in bpf_prog_test_run_sk_lookup()
1407 if (user_ctx->local_port > U16_MAX) { in bpf_prog_test_run_sk_lookup()
1408 ret = -ERANGE; in bpf_prog_test_run_sk_lookup()
1412 ctx.family = (u16)user_ctx->family; in bpf_prog_test_run_sk_lookup()
1413 ctx.protocol = (u16)user_ctx->protocol; in bpf_prog_test_run_sk_lookup()
1414 ctx.dport = (u16)user_ctx->local_port; in bpf_prog_test_run_sk_lookup()
1415 ctx.sport = user_ctx->remote_port; in bpf_prog_test_run_sk_lookup()
1419 ctx.v4.daddr = (__force __be32)user_ctx->local_ip4; in bpf_prog_test_run_sk_lookup()
1420 ctx.v4.saddr = (__force __be32)user_ctx->remote_ip4; in bpf_prog_test_run_sk_lookup()
1425 ctx.v6.daddr = (struct in6_addr *)user_ctx->local_ip6; in bpf_prog_test_run_sk_lookup()
1426 ctx.v6.saddr = (struct in6_addr *)user_ctx->remote_ip6; in bpf_prog_test_run_sk_lookup()
1431 ret = -EAFNOSUPPORT; in bpf_prog_test_run_sk_lookup()
1437 ret = -ENOMEM; in bpf_prog_test_run_sk_lookup()
1441 progs->items[0].prog = prog; in bpf_prog_test_run_sk_lookup()
1443 bpf_test_timer_enter(&t); in bpf_prog_test_run_sk_lookup()
1447 } while (bpf_test_timer_continue(&t, 1, repeat, &ret, &duration)); in bpf_prog_test_run_sk_lookup()
1448 bpf_test_timer_leave(&t); in bpf_prog_test_run_sk_lookup()
1453 user_ctx->cookie = 0; in bpf_prog_test_run_sk_lookup()
1455 if (ctx.selected_sk->sk_reuseport && !ctx.no_reuseport) { in bpf_prog_test_run_sk_lookup()
1456 ret = -EOPNOTSUPP; in bpf_prog_test_run_sk_lookup()
1460 user_ctx->cookie = sock_gen_cookie(ctx.selected_sk); in bpf_prog_test_run_sk_lookup()
1477 void __user *ctx_in = u64_to_user_ptr(kattr->test.ctx_in); in bpf_prog_test_run_syscall()
1478 __u32 ctx_size_in = kattr->test.ctx_size_in; in bpf_prog_test_run_syscall()
1483 /* doesn't support data_in/out, ctx_out, duration, or repeat or flags */ in bpf_prog_test_run_syscall()
1484 if (kattr->test.data_in || kattr->test.data_out || in bpf_prog_test_run_syscall()
1485 kattr->test.ctx_out || kattr->test.duration || in bpf_prog_test_run_syscall()
1486 kattr->test.repeat || kattr->test.flags || in bpf_prog_test_run_syscall()
1487 kattr->test.batch_size) in bpf_prog_test_run_syscall()
1488 return -EINVAL; in bpf_prog_test_run_syscall()
1490 if (ctx_size_in < prog->aux->max_ctx_offset || in bpf_prog_test_run_syscall()
1492 return -EINVAL; in bpf_prog_test_run_syscall()
1504 if (copy_to_user(&uattr->test.retval, &retval, sizeof(u32))) { in bpf_prog_test_run_syscall()
1505 err = -EFAULT; in bpf_prog_test_run_syscall()
1510 err = -EFAULT; in bpf_prog_test_run_syscall()
1520 if (user->in || user->out) in verify_and_copy_hook_state()
1521 return -EINVAL; in verify_and_copy_hook_state()
1523 if (user->net || user->sk || user->okfn) in verify_and_copy_hook_state()
1524 return -EINVAL; in verify_and_copy_hook_state()
1526 switch (user->pf) { in verify_and_copy_hook_state()
1529 switch (state->hook) { in verify_and_copy_hook_state()
1531 state->in = dev; in verify_and_copy_hook_state()
1534 state->in = dev; in verify_and_copy_hook_state()
1537 state->in = dev; in verify_and_copy_hook_state()
1538 state->out = dev; in verify_and_copy_hook_state()
1541 state->out = dev; in verify_and_copy_hook_state()
1544 state->out = dev; in verify_and_copy_hook_state()
1550 return -EINVAL; in verify_and_copy_hook_state()
1553 state->pf = user->pf; in verify_and_copy_hook_state()
1554 state->hook = user->hook; in verify_and_copy_hook_state()
1575 struct net *net = current->nsproxy->net_ns; in bpf_prog_test_run_nf()
1576 struct net_device *dev = net->loopback_dev; in bpf_prog_test_run_nf()
1581 u32 size = kattr->test.data_size_in; in bpf_prog_test_run_nf()
1582 u32 repeat = kattr->test.repeat; in bpf_prog_test_run_nf()
1591 if (kattr->test.flags || kattr->test.cpu || kattr->test.batch_size) in bpf_prog_test_run_nf()
1592 return -EINVAL; in bpf_prog_test_run_nf()
1595 return -EINVAL; in bpf_prog_test_run_nf()
1597 data = bpf_test_init(kattr, kattr->test.data_size_in, size, in bpf_prog_test_run_nf()
1620 ret = -ENOMEM; in bpf_prog_test_run_nf()
1629 ret = -EINVAL; in bpf_prog_test_run_nf()
1635 skb->protocol = eth_type_trans(skb, dev); in bpf_prog_test_run_nf()
1636 switch (skb->protocol) { in bpf_prog_test_run_nf()
1648 ret = -EPROTO; in bpf_prog_test_run_nf()
1654 skb->protocol = nfproto_eth(hook_state.pf); in bpf_prog_test_run_nf()