| /kernel/linux/linux-5.10/include/net/ |
| D | xdp.h | 84 #define xdp_data_hard_end(xdp) \ argument
85 ((xdp)->data_hard_start + (xdp)->frame_sz - \
89 xdp_get_shared_info_from_buff(struct xdp_buff *xdp) in xdp_get_shared_info_from_buff() argument
91 return (struct skb_shared_info *)xdp_data_hard_end(xdp); in xdp_get_shared_info_from_buff()
134 struct xdp_frame *xdp_convert_zc_to_xdp_frame(struct xdp_buff *xdp);
137 void xdp_convert_frame_to_buff(struct xdp_frame *frame, struct xdp_buff *xdp) in xdp_convert_frame_to_buff() argument
139 xdp->data_hard_start = frame->data - frame->headroom - sizeof(*frame); in xdp_convert_frame_to_buff()
140 xdp->data = frame->data; in xdp_convert_frame_to_buff()
141 xdp->data_end = frame->data + frame->len; in xdp_convert_frame_to_buff()
142 xdp->data_meta = frame->data - frame->metasize; in xdp_convert_frame_to_buff()
[all …]
|
| D | xdp_sock_drv.h | 60 static inline dma_addr_t xsk_buff_xdp_get_dma(struct xdp_buff *xdp) in xsk_buff_xdp_get_dma() argument
62 struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp); in xsk_buff_xdp_get_dma()
67 static inline dma_addr_t xsk_buff_xdp_get_frame_dma(struct xdp_buff *xdp) in xsk_buff_xdp_get_frame_dma() argument
69 struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp); in xsk_buff_xdp_get_frame_dma()
84 static inline void xsk_buff_free(struct xdp_buff *xdp) in xsk_buff_free() argument
86 struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp); in xsk_buff_free()
102 static inline void xsk_buff_dma_sync_for_cpu(struct xdp_buff *xdp, struct xsk_buff_pool *pool) in xsk_buff_dma_sync_for_cpu() argument
104 struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp); in xsk_buff_dma_sync_for_cpu()
193 static inline dma_addr_t xsk_buff_xdp_get_dma(struct xdp_buff *xdp) in xsk_buff_xdp_get_dma() argument
198 static inline dma_addr_t xsk_buff_xdp_get_frame_dma(struct xdp_buff *xdp) in xsk_buff_xdp_get_frame_dma() argument
[all …]
|
| /kernel/linux/linux-5.10/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/ |
| D | rx.c | 32 struct xdp_buff *xdp = wi->umr.dma_info[page_idx].xsk; in mlx5e_xsk_skb_from_cqe_mpwrq_linear() local
48 xdp->data_end = xdp->data + cqe_bcnt32; in mlx5e_xsk_skb_from_cqe_mpwrq_linear()
49 xdp_set_data_meta_invalid(xdp); in mlx5e_xsk_skb_from_cqe_mpwrq_linear()
50 xsk_buff_dma_sync_for_cpu(xdp, rq->xsk_pool); in mlx5e_xsk_skb_from_cqe_mpwrq_linear()
51 net_prefetch(xdp->data); in mlx5e_xsk_skb_from_cqe_mpwrq_linear()
68 if (likely(mlx5e_xdp_handle(rq, NULL, &cqe_bcnt32, xdp))) { in mlx5e_xsk_skb_from_cqe_mpwrq_linear()
77 return mlx5e_xsk_construct_skb(rq, xdp->data, cqe_bcnt32); in mlx5e_xsk_skb_from_cqe_mpwrq_linear()
85 struct xdp_buff *xdp = wi->di->xsk; in mlx5e_xsk_skb_from_cqe_linear() local
94 xdp->data_end = xdp->data + cqe_bcnt; in mlx5e_xsk_skb_from_cqe_linear()
95 xdp_set_data_meta_invalid(xdp); in mlx5e_xsk_skb_from_cqe_linear()
[all …]
|
| /kernel/linux/linux-5.10/drivers/net/hyperv/ |
| D | netvsc_bpf.c | 24 struct xdp_buff *xdp) in netvsc_run_xdp() argument
32 xdp->data_hard_start = NULL; in netvsc_run_xdp()
47 xdp->data_hard_start = page_address(page); in netvsc_run_xdp()
48 xdp->data = xdp->data_hard_start + NETVSC_XDP_HDRM; in netvsc_run_xdp()
49 xdp_set_data_meta_invalid(xdp); in netvsc_run_xdp()
50 xdp->data_end = xdp->data + len; in netvsc_run_xdp()
51 xdp->rxq = &nvchan->xdp_rxq; in netvsc_run_xdp()
52 xdp->frame_sz = PAGE_SIZE; in netvsc_run_xdp()
54 memcpy(xdp->data, data, len); in netvsc_run_xdp()
56 act = bpf_prog_run_xdp(prog, xdp); in netvsc_run_xdp()
[all …]
|
| /kernel/linux/linux-5.10/drivers/net/ethernet/broadcom/bnxt/ |
| D | bnxt_xdp.c | 118 struct xdp_buff xdp; in bnxt_rx_xdp() local
136 xdp.data_hard_start = *data_ptr - offset; in bnxt_rx_xdp()
137 xdp.data = *data_ptr; in bnxt_rx_xdp()
138 xdp_set_data_meta_invalid(&xdp); in bnxt_rx_xdp()
139 xdp.data_end = *data_ptr + *len; in bnxt_rx_xdp()
140 xdp.rxq = &rxr->xdp_rxq; in bnxt_rx_xdp()
141 xdp.frame_sz = PAGE_SIZE; /* BNXT_RX_PAGE_MODE(bp) when XDP enabled */ in bnxt_rx_xdp()
142 orig_data = xdp.data; in bnxt_rx_xdp()
145 act = bpf_prog_run_xdp(xdp_prog, &xdp); in bnxt_rx_xdp()
155 *len = xdp.data_end - xdp.data; in bnxt_rx_xdp()
[all …]
|
| /kernel/linux/linux-5.10/samples/bpf/ |
| D | xdp_adjust_tail_kern.c | 70 static __always_inline int send_icmp4_too_big(struct xdp_md *xdp) in send_icmp4_too_big() argument
74 if (bpf_xdp_adjust_head(xdp, 0 - headroom)) in send_icmp4_too_big()
76 void *data = (void *)(long)xdp->data; in send_icmp4_too_big()
77 void *data_end = (void *)(long)xdp->data_end; in send_icmp4_too_big()
120 static __always_inline int handle_ipv4(struct xdp_md *xdp) in handle_ipv4() argument
122 void *data_end = (void *)(long)xdp->data_end; in handle_ipv4()
123 void *data = (void *)(long)xdp->data; in handle_ipv4()
129 if (bpf_xdp_adjust_tail(xdp, 0 - offset)) in handle_ipv4()
131 return send_icmp4_too_big(xdp); in handle_ipv4()
137 int _xdp_icmp(struct xdp_md *xdp) in _xdp_icmp() argument
[all …]
|
| D | xdp_tx_iptunnel_kern.c | 77 static __always_inline int handle_ipv4(struct xdp_md *xdp) in handle_ipv4() argument
79 void *data_end = (void *)(long)xdp->data_end; in handle_ipv4()
80 void *data = (void *)(long)xdp->data; in handle_ipv4()
112 if (bpf_xdp_adjust_head(xdp, 0 - (int)sizeof(struct iphdr))) in handle_ipv4()
115 data = (void *)(long)xdp->data; in handle_ipv4()
116 data_end = (void *)(long)xdp->data_end; in handle_ipv4()
152 static __always_inline int handle_ipv6(struct xdp_md *xdp) in handle_ipv6() argument
154 void *data_end = (void *)(long)xdp->data_end; in handle_ipv6()
155 void *data = (void *)(long)xdp->data; in handle_ipv6()
184 if (bpf_xdp_adjust_head(xdp, 0 - (int)sizeof(struct ipv6hdr))) in handle_ipv6()
[all …]
|
| /kernel/linux/linux-5.10/tools/testing/selftests/bpf/progs/ |
| D | test_xdp.c | 80 static __always_inline int handle_ipv4(struct xdp_md *xdp) in handle_ipv4() argument
82 void *data_end = (void *)(long)xdp->data_end; in handle_ipv4()
83 void *data = (void *)(long)xdp->data; in handle_ipv4()
113 if (bpf_xdp_adjust_head(xdp, 0 - (int)sizeof(struct iphdr))) in handle_ipv4()
116 data = (void *)(long)xdp->data; in handle_ipv4()
117 data_end = (void *)(long)xdp->data_end; in handle_ipv4()
153 static __always_inline int handle_ipv6(struct xdp_md *xdp) in handle_ipv6() argument
155 void *data_end = (void *)(long)xdp->data_end; in handle_ipv6()
156 void *data = (void *)(long)xdp->data; in handle_ipv6()
183 if (bpf_xdp_adjust_head(xdp, 0 - (int)sizeof(struct ipv6hdr))) in handle_ipv6()
[all …]
|
| D | test_xdp_loop.c | 76 static __always_inline int handle_ipv4(struct xdp_md *xdp) in handle_ipv4() argument
78 void *data_end = (void *)(long)xdp->data_end; in handle_ipv4()
79 void *data = (void *)(long)xdp->data; in handle_ipv4()
109 if (bpf_xdp_adjust_head(xdp, 0 - (int)sizeof(struct iphdr))) in handle_ipv4()
112 data = (void *)(long)xdp->data; in handle_ipv4()
113 data_end = (void *)(long)xdp->data_end; in handle_ipv4()
149 static __always_inline int handle_ipv6(struct xdp_md *xdp) in handle_ipv6() argument
151 void *data_end = (void *)(long)xdp->data_end; in handle_ipv6()
152 void *data = (void *)(long)xdp->data; in handle_ipv6()
179 if (bpf_xdp_adjust_head(xdp, 0 - (int)sizeof(struct ipv6hdr))) in handle_ipv6()
[all …]
|
| D | test_xdp_bpf2bpf.c | 45 int BPF_PROG(trace_on_entry, struct xdp_buff *xdp) in BPF_PROG() argument
48 void *data_end = (void *)(long)xdp->data_end; in BPF_PROG()
49 void *data = (void *)(long)xdp->data; in BPF_PROG()
51 meta.ifindex = xdp->rxq->dev->ifindex; in BPF_PROG()
53 bpf_xdp_output(xdp, &perf_buf_map, in BPF_PROG()
58 test_result_fentry = xdp->rxq->dev->ifindex; in BPF_PROG()
64 int BPF_PROG(trace_on_exit, struct xdp_buff *xdp, int ret) in BPF_PROG() argument
|
| D | test_xdp_adjust_tail_shrink.c | 15 int _xdp_adjust_tail_shrink(struct xdp_md *xdp) in _xdp_adjust_tail_shrink() argument
17 void *data_end = (void *)(long)xdp->data_end; in _xdp_adjust_tail_shrink()
18 void *data = (void *)(long)xdp->data; in _xdp_adjust_tail_shrink()
25 if (bpf_xdp_adjust_tail(xdp, 0 - offset)) in _xdp_adjust_tail_shrink()
|
| D | test_xdp_adjust_tail_grow.c | 6 int _xdp_adjust_tail_grow(struct xdp_md *xdp) in _xdp_adjust_tail_grow() argument
8 void *data_end = (void *)(long)xdp->data_end; in _xdp_adjust_tail_grow()
9 void *data = (void *)(long)xdp->data; in _xdp_adjust_tail_grow()
28 if (bpf_xdp_adjust_tail(xdp, offset)) in _xdp_adjust_tail_grow()
|
| D | test_xdp_noinline.c | 278 bool encap_v6(struct xdp_md *xdp, struct ctl_value *cval, in encap_v6() argument
289 if (bpf_xdp_adjust_head(xdp, 0 - (int)sizeof(struct ipv6hdr))) in encap_v6()
291 data = (void *)(long)xdp->data; in encap_v6()
292 data_end = (void *)(long)xdp->data_end; in encap_v6()
321 bool encap_v4(struct xdp_md *xdp, struct ctl_value *cval, in encap_v4() argument
337 if (bpf_xdp_adjust_head(xdp, 0 - (int)sizeof(struct iphdr))) in encap_v4()
339 data = (void *)(long)xdp->data; in encap_v4()
340 data_end = (void *)(long)xdp->data_end; in encap_v4()
369 if (bpf_xdp_adjust_head(xdp, (int)sizeof(struct iphdr))) in encap_v4()
375 bool decap_v6(struct xdp_md *xdp, void **data, void **data_end, bool inner_v4) in decap_v6() argument
[all …]
|
| /kernel/linux/linux-5.10/drivers/net/ethernet/intel/ixgbe/ |
| D | ixgbe_xsk.c | 99 struct xdp_buff *xdp) in ixgbe_run_xdp_zc() argument
108 act = bpf_prog_run_xdp(xdp_prog, xdp); in ixgbe_run_xdp_zc()
111 err = xdp_do_redirect(rx_ring->netdev, xdp, xdp_prog); in ixgbe_run_xdp_zc()
122 xdpf = xdp_convert_buff_to_frame(xdp); in ixgbe_run_xdp_zc()
161 bi->xdp = xsk_buff_alloc(rx_ring->xsk_pool); in ixgbe_alloc_rx_buffers_zc()
162 if (!bi->xdp) { in ixgbe_alloc_rx_buffers_zc()
167 dma = xsk_buff_xdp_get_dma(bi->xdp); in ixgbe_alloc_rx_buffers_zc()
209 unsigned int metasize = bi->xdp->data - bi->xdp->data_meta; in ixgbe_construct_skb_zc()
210 unsigned int datasize = bi->xdp->data_end - bi->xdp->data; in ixgbe_construct_skb_zc()
215 bi->xdp->data_end - bi->xdp->data_hard_start, in ixgbe_construct_skb_zc()
[all …]
|
| /kernel/linux/linux-5.10/include/trace/events/ |
| D | xdp.h | 3 #define TRACE_SYSTEM xdp
31 const struct bpf_prog *xdp, u32 act),
33 TP_ARGS(dev, xdp, act),
42 __entry->prog_id = xdp->aux->id;
97 const struct bpf_prog *xdp,
101 TP_ARGS(dev, xdp, tgt, err, map, index),
114 __entry->prog_id = xdp->aux->id;
134 const struct bpf_prog *xdp,
137 TP_ARGS(dev, xdp, tgt, err, map, index)
142 const struct bpf_prog *xdp,
[all …]
|
| /kernel/linux/linux-5.10/tools/testing/selftests/bpf/ |
| D | test_xdp_veth.sh | 61 if ! ip link set dev lo xdp off > /dev/null 2>&1; then
101 xdp_redirect_map.o $BPF_DIR/progs type xdp \
106 ip link set dev veth1 xdp pinned $BPF_DIR/progs/redirect_map_0
107 ip link set dev veth2 xdp pinned $BPF_DIR/progs/redirect_map_1
108 ip link set dev veth3 xdp pinned $BPF_DIR/progs/redirect_map_2
110 ip -n ns1 link set dev veth11 xdp obj xdp_dummy.o sec xdp_dummy
111 ip -n ns2 link set dev veth22 xdp obj xdp_tx.o sec xdp
112 ip -n ns3 link set dev veth33 xdp obj xdp_dummy.o sec xdp_dummy
|
| D | test_tcp_check_syncookie_user.c | 81 static int get_map_fd_by_prog_id(int prog_id, bool *xdp) in get_map_fd_by_prog_id() argument
108 *xdp = info.type == BPF_PROG_TYPE_XDP; in get_map_fd_by_prog_id()
119 static int run_test(int server_fd, int results_fd, bool xdp) in run_test() argument
170 if (xdp && value_gen == 0) { in run_test()
210 bool xdp; in main() local
217 results = get_map_fd_by_prog_id(atoi(argv[1]), &xdp); in main()
242 if (run_test(server, results, xdp)) in main()
245 if (run_test(server_v6, results, xdp)) in main()
|
| /kernel/linux/linux-5.10/net/core/ |
| D | xdp.c | 341 struct xdp_buff *xdp) in __xdp_return() argument
366 xsk_buff_free(xdp); in __xdp_return()
387 void xdp_return_buff(struct xdp_buff *xdp) in xdp_return_buff() argument
389 __xdp_return(xdp->data, &xdp->rxq->mem, true, xdp); in xdp_return_buff()
417 struct xdp_frame *xdp_convert_zc_to_xdp_frame(struct xdp_buff *xdp) in xdp_convert_zc_to_xdp_frame() argument
425 metasize = xdp_data_meta_unsupported(xdp) ? 0 : in xdp_convert_zc_to_xdp_frame()
426 xdp->data - xdp->data_meta; in xdp_convert_zc_to_xdp_frame()
427 totsize = xdp->data_end - xdp->data + metasize; in xdp_convert_zc_to_xdp_frame()
441 data_to_copy = metasize ? xdp->data_meta : xdp->data; in xdp_convert_zc_to_xdp_frame()
451 xsk_buff_free(xdp); in xdp_convert_zc_to_xdp_frame()
|
| /kernel/linux/linux-5.10/drivers/net/ |
| D | veth.c | 242 static void *veth_xdp_to_ptr(struct xdp_frame *xdp) in veth_xdp_to_ptr() argument
244 return (void *)((unsigned long)xdp | VETH_XDP_FLAG); in veth_xdp_to_ptr()
276 struct veth_rq *rq, bool xdp) in veth_forward_skb() argument
278 return __dev_forward_skb(dev, skb) ?: xdp ? in veth_forward_skb()
547 static int veth_xdp_tx(struct veth_rq *rq, struct xdp_buff *xdp, in veth_xdp_tx() argument
550 struct xdp_frame *frame = xdp_convert_buff_to_frame(xdp); in veth_xdp_tx()
581 struct xdp_buff xdp; in veth_xdp_rcv_one() local
584 xdp_convert_frame_to_buff(frame, &xdp); in veth_xdp_rcv_one()
585 xdp.rxq = &rq->xdp_rxq; in veth_xdp_rcv_one()
587 act = bpf_prog_run_xdp(xdp_prog, &xdp); in veth_xdp_rcv_one()
[all …]
|
| /kernel/linux/linux-5.10/drivers/net/ethernet/intel/ice/ |
| D | ice_xsk.c | 444 rx_buf->xdp = xsk_buff_alloc(rx_ring->xsk_pool); in ice_alloc_rx_bufs_zc()
445 if (!rx_buf->xdp) { in ice_alloc_rx_bufs_zc()
450 dma = xsk_buff_xdp_get_dma(rx_buf->xdp); in ice_alloc_rx_bufs_zc()
499 unsigned int metasize = rx_buf->xdp->data - rx_buf->xdp->data_meta; in ice_construct_skb_zc()
500 unsigned int datasize = rx_buf->xdp->data_end - rx_buf->xdp->data; in ice_construct_skb_zc()
501 unsigned int datasize_hard = rx_buf->xdp->data_end - in ice_construct_skb_zc()
502 rx_buf->xdp->data_hard_start; in ice_construct_skb_zc()
510 skb_reserve(skb, rx_buf->xdp->data - rx_buf->xdp->data_hard_start); in ice_construct_skb_zc()
511 memcpy(__skb_put(skb, datasize), rx_buf->xdp->data, datasize); in ice_construct_skb_zc()
515 xsk_buff_free(rx_buf->xdp); in ice_construct_skb_zc()
[all …]
|
| D | ice_txrx.c | 537 ice_run_xdp(struct ice_ring *rx_ring, struct xdp_buff *xdp, in ice_run_xdp() argument
544 act = bpf_prog_run_xdp(xdp_prog, xdp); in ice_run_xdp()
550 result = ice_xmit_xdp_buff(xdp, xdp_ring); in ice_run_xdp()
555 err = xdp_do_redirect(rx_ring->netdev, xdp, xdp_prog); in ice_run_xdp()
915 struct xdp_buff *xdp) in ice_build_skb() argument
917 u8 metasize = xdp->data - xdp->data_meta; in ice_build_skb()
922 SKB_DATA_ALIGN(xdp->data_end - in ice_build_skb()
923 xdp->data_hard_start); in ice_build_skb()
932 net_prefetch(xdp->data_meta); in ice_build_skb()
934 skb = build_skb(xdp->data_hard_start, truesize); in ice_build_skb()
[all …]
|
| /kernel/linux/linux-5.10/drivers/net/ethernet/intel/i40e/ |
| D | i40e_xsk.c | 199 static int i40e_run_xdp_zc(struct i40e_ring *rx_ring, struct xdp_buff *xdp) in i40e_run_xdp_zc() argument
211 act = bpf_prog_run_xdp(xdp_prog, xdp); in i40e_run_xdp_zc()
214 err = xdp_do_redirect(rx_ring->netdev, xdp, xdp_prog); in i40e_run_xdp_zc()
226 result = i40e_xmit_xdp_tx_ring(xdp, xdp_ring); in i40e_run_xdp_zc()
249 struct xdp_buff **bi, *xdp; in i40e_alloc_rx_buffers_zc() local
256 xdp = xsk_buff_alloc(rx_ring->xsk_pool); in i40e_alloc_rx_buffers_zc()
257 if (!xdp) { in i40e_alloc_rx_buffers_zc()
261 *bi = xdp; in i40e_alloc_rx_buffers_zc()
262 dma = xsk_buff_xdp_get_dma(xdp); in i40e_alloc_rx_buffers_zc()
299 struct xdp_buff *xdp) in i40e_construct_skb_zc() argument
[all …]
|
| /kernel/linux/linux-5.10/drivers/net/ethernet/sfc/ |
| D | rx.c | 261 struct xdp_buff xdp; in efx_do_xdp() local
296 xdp.data = *ehp; in efx_do_xdp()
297 xdp.data_hard_start = xdp.data - EFX_XDP_HEADROOM; in efx_do_xdp()
300 xdp_set_data_meta_invalid(&xdp); in efx_do_xdp()
301 xdp.data_end = xdp.data + rx_buf->len; in efx_do_xdp()
302 xdp.rxq = &rx_queue->xdp_rxq_info; in efx_do_xdp()
303 xdp.frame_sz = efx->rx_page_buf_step; in efx_do_xdp()
305 xdp_act = bpf_prog_run_xdp(xdp_prog, &xdp); in efx_do_xdp()
308 offset = (u8 *)xdp.data - *ehp; in efx_do_xdp()
324 xdpf = xdp_convert_buff_to_frame(&xdp); in efx_do_xdp()
[all …]
|
| /kernel/linux/linux-5.10/tools/bpf/bpftool/Documentation/ |
| D | bpftool-net.rst | 29 | *ATTACH_TYPE* := { **xdp** | **xdpgeneric** | **xdpdrv** | **xdpoffload** }
36 Currently, only device driver xdp attachments and tc filter
47 The current output will start with all xdp program attachments, followed by
48 all tc class/qdisc bpf program attachments. Both xdp programs and
62 … **xdp** - try native XDP and fallback to generic XDP if NIC driver does not support it;
87 xdp:
103 "xdp": [{
154 xdp:
164 xdp:
174 xdp:
|
| /kernel/linux/linux-5.10/net/xdp/ |
| D | xsk.c | 143 u64 offset = xskb->xdp.data - xskb->xdp.data_hard_start; in xp_get_handle()
151 static int __xsk_rcv_zc(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len) in __xsk_rcv_zc() argument
153 struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp); in __xsk_rcv_zc()
186 static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len, in __xsk_rcv() argument
203 xsk_copy_xdp(xsk_xdp, xdp, len); in __xsk_rcv()
210 xdp_return_buff(xdp); in __xsk_rcv()
232 static int xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, in xsk_rcv() argument
240 if (xs->dev != xdp->rxq->dev || xs->queue_id != xdp->rxq->queue_index) in xsk_rcv()
243 len = xdp->data_end - xdp->data; in xsk_rcv()
245 return xdp->rxq->mem.type == MEM_TYPE_XSK_BUFF_POOL ? in xsk_rcv()
[all …]
|