| /kernel/linux/linux-5.10/kernel/bpf/ |
| D | cpumap.c | 158 static struct sk_buff *cpu_map_build_skb(struct xdp_frame *xdpf, in cpu_map_build_skb() argument
166 hard_start_headroom = sizeof(struct xdp_frame) + xdpf->headroom; in cpu_map_build_skb()
171 frame_size = xdpf->frame_sz; in cpu_map_build_skb()
173 pkt_data_start = xdpf->data - hard_start_headroom; in cpu_map_build_skb()
179 __skb_put(skb, xdpf->len); in cpu_map_build_skb()
180 if (xdpf->metasize) in cpu_map_build_skb()
181 skb_metadata_set(skb, xdpf->metasize); in cpu_map_build_skb()
184 skb->protocol = eth_type_trans(skb, xdpf->dev_rx); in cpu_map_build_skb()
193 xdp_release_frame(xdpf); in cpu_map_build_skb()
196 xdp_scrub_frame(xdpf); in cpu_map_build_skb()
[all …]
|
| D | devmap.c | 354 struct xdp_frame *xdpf = bq->q[i]; in bq_xmit_all() local
356 prefetch(xdpf); in bq_xmit_all()
378 struct xdp_frame *xdpf = bq->q[i]; in bq_xmit_all() local
380 xdp_return_frame_rx_napi(xdpf); in bq_xmit_all()
424 static void bq_enqueue(struct net_device *dev, struct xdp_frame *xdpf, in bq_enqueue() argument
440 bq->q[bq->count++] = xdpf; in bq_enqueue()
449 struct xdp_frame *xdpf; in __xdp_enqueue() local
459 xdpf = xdp_convert_buff_to_frame(xdp); in __xdp_enqueue()
460 if (unlikely(!xdpf)) in __xdp_enqueue()
463 bq_enqueue(dev, xdpf, dev_rx); in __xdp_enqueue()
|
| /kernel/linux/linux-5.10/net/core/ |
| D | xdp.c | 375 void xdp_return_frame(struct xdp_frame *xdpf) in xdp_return_frame() argument
377 __xdp_return(xdpf->data, &xdpf->mem, false, NULL); in xdp_return_frame()
381 void xdp_return_frame_rx_napi(struct xdp_frame *xdpf) in xdp_return_frame_rx_napi() argument
383 __xdp_return(xdpf->data, &xdpf->mem, true, NULL); in xdp_return_frame_rx_napi()
421 struct xdp_frame *xdpf; in xdp_convert_zc_to_xdp_frame() local
429 if (sizeof(*xdpf) + totsize > PAGE_SIZE) in xdp_convert_zc_to_xdp_frame()
437 xdpf = addr; in xdp_convert_zc_to_xdp_frame()
438 memset(xdpf, 0, sizeof(*xdpf)); in xdp_convert_zc_to_xdp_frame()
440 addr += sizeof(*xdpf); in xdp_convert_zc_to_xdp_frame()
444 xdpf->data = addr + metasize; in xdp_convert_zc_to_xdp_frame()
[all …]
|
| /kernel/linux/linux-5.10/drivers/net/ethernet/mellanox/mlx5/core/en/ |
| D | xdp.c | 64 struct xdp_frame *xdpf; in mlx5e_xmit_xdp_buff() local
67 xdpf = xdp_convert_buff_to_frame(xdp); in mlx5e_xmit_xdp_buff()
68 if (unlikely(!xdpf)) in mlx5e_xmit_xdp_buff()
71 xdptxd.data = xdpf->data; in mlx5e_xmit_xdp_buff()
72 xdptxd.len = xdpf->len; in mlx5e_xmit_xdp_buff()
92 xdp_return_frame(xdpf); in mlx5e_xmit_xdp_buff()
97 xdpi.frame.xdpf = xdpf; in mlx5e_xmit_xdp_buff()
108 dma_addr = di->addr + (xdpf->data - (void *)xdpf); in mlx5e_xmit_xdp_buff()
381 xdpi.frame.xdpf->len, DMA_TO_DEVICE); in mlx5e_free_xdpsq_desc()
382 xdp_return_frame(xdpi.frame.xdpf); in mlx5e_free_xdpsq_desc()
[all …]
|
| /kernel/linux/linux-5.10/include/net/ |
| D | xdp.h | 194 void xdp_return_frame(struct xdp_frame *xdpf);
195 void xdp_return_frame_rx_napi(struct xdp_frame *xdpf);
204 static inline void xdp_release_frame(struct xdp_frame *xdpf) in xdp_release_frame() argument
206 struct xdp_mem_info *mem = &xdpf->mem; in xdp_release_frame()
210 __xdp_release_frame(xdpf->data, mem); in xdp_release_frame()
|
| /kernel/linux/linux-5.10/drivers/net/ethernet/intel/ixgbe/ |
| D | ixgbe_xsk.c | 103 struct xdp_frame *xdpf; in ixgbe_run_xdp_zc() local
122 xdpf = xdp_convert_buff_to_frame(xdp); in ixgbe_run_xdp_zc()
123 if (unlikely(!xdpf)) in ixgbe_run_xdp_zc()
125 result = ixgbe_xmit_xdp_ring(adapter, xdpf); in ixgbe_run_xdp_zc()
408 tx_bi->xdpf = NULL; in ixgbe_xmit_zc()
439 xdp_return_frame(tx_bi->xdpf); in ixgbe_clean_xdp_tx_buffer()
466 if (tx_bi->xdpf) in ixgbe_clean_xdp_tx_irq()
471 tx_bi->xdpf = NULL; in ixgbe_clean_xdp_tx_irq()
545 if (tx_bi->xdpf) in ixgbe_xsk_clean_tx_ring()
550 tx_bi->xdpf = NULL; in ixgbe_xsk_clean_tx_ring()
|
| D | ixgbe_txrx_common.h | 16 struct xdp_frame *xdpf);
|
| /kernel/linux/linux-5.10/drivers/net/ethernet/intel/ice/ |
| D | ice_txrx_lib.c | 262 struct xdp_frame *xdpf = xdp_convert_buff_to_frame(xdp); in ice_xmit_xdp_buff() local
264 if (unlikely(!xdpf)) in ice_xmit_xdp_buff()
267 return ice_xmit_xdp_ring(xdpf->data, xdpf->len, xdp_ring); in ice_xmit_xdp_buff()
|
| /kernel/linux/linux-5.10/drivers/net/ethernet/ti/ |
| D | cpsw_priv.c | 54 struct xdp_frame *xdpf; in cpsw_tx_handler() local
61 xdpf = cpsw_handle_to_xdpf(token); in cpsw_tx_handler()
62 xmeta = (void *)xdpf + CPSW_XMETA_OFFSET; in cpsw_tx_handler()
65 xdp_return_frame(xdpf); in cpsw_tx_handler()
1288 int cpsw_xdp_tx_frame(struct cpsw_priv *priv, struct xdp_frame *xdpf, in cpsw_xdp_tx_frame() argument
1297 xmeta = (void *)xdpf + CPSW_XMETA_OFFSET; in cpsw_xdp_tx_frame()
1304 dma += xdpf->headroom + sizeof(struct xdp_frame); in cpsw_xdp_tx_frame()
1305 ret = cpdma_chan_submit_mapped(txch, cpsw_xdpf_to_handle(xdpf), in cpsw_xdp_tx_frame()
1306 dma, xdpf->len, port); in cpsw_xdp_tx_frame()
1308 if (sizeof(*xmeta) > xdpf->headroom) { in cpsw_xdp_tx_frame()
[all …]
|
| D | cpsw_priv.h | 419 static inline void *cpsw_xdpf_to_handle(struct xdp_frame *xdpf) in cpsw_xdpf_to_handle() argument
421 return (void *)((unsigned long)xdpf | BIT(0)); in cpsw_xdpf_to_handle()
440 int cpsw_xdp_tx_frame(struct cpsw_priv *priv, struct xdp_frame *xdpf,
|
| D | cpsw.c | 1133 struct xdp_frame *xdpf; in cpsw_ndo_xdp_xmit() local
1140 xdpf = frames[i]; in cpsw_ndo_xdp_xmit()
1141 if (xdpf->len < CPSW_MIN_PACKET_SIZE) { in cpsw_ndo_xdp_xmit()
1142 xdp_return_frame_rx_napi(xdpf); in cpsw_ndo_xdp_xmit()
1148 if (cpsw_xdp_tx_frame(priv, xdpf, NULL, port)) in cpsw_ndo_xdp_xmit()
|
| /kernel/linux/linux-5.10/drivers/net/ethernet/broadcom/bnxt/ |
| D | bnxt_xdp.c | 60 struct xdp_frame *xdpf) in __bnxt_xmit_xdp_redirect() argument
66 tx_buf->xdpf = xdpf; in __bnxt_xmit_xdp_redirect()
91 xdp_return_frame(tx_buf->xdpf); in bnxt_tx_int_xdp()
93 tx_buf->xdpf = NULL; in bnxt_tx_int_xdp()
|
| /kernel/linux/linux-5.10/drivers/net/ethernet/qlogic/qede/ |
| D | qede_fp.c | 306 u16 len, struct page *page, struct xdp_frame *xdpf) in qede_xdp_xmit() argument
333 xdp->xdpf = xdpf; in qede_xdp_xmit()
346 struct xdp_frame *xdpf; in qede_xdp_transmit() local
363 xdpf = frames[i]; in qede_xdp_transmit()
365 mapping = dma_map_single(dmadev, xdpf->data, xdpf->len, in qede_xdp_transmit()
368 xdp_return_frame_rx_napi(xdpf); in qede_xdp_transmit()
374 if (unlikely(qede_xdp_xmit(xdp_tx, mapping, 0, xdpf->len, in qede_xdp_transmit()
375 NULL, xdpf))) { in qede_xdp_transmit()
376 xdp_return_frame_rx_napi(xdpf); in qede_xdp_transmit()
410 struct xdp_frame *xdpf; in qede_xdp_tx_int() local
[all …]
|
| /kernel/linux/linux-5.10/drivers/net/ethernet/socionext/ |
| D | netsec.c | 272 struct xdp_frame *xdpf; member
667 bytes += desc->xdpf->len; in netsec_clean_tx_dring()
668 xdp_return_frame(desc->xdpf); in netsec_clean_tx_dring()
807 dring->desc[idx].xdpf = buf; in netsec_set_tx_de()
815 struct xdp_frame *xdpf, bool is_ndo) in netsec_xdp_queue_one() argument
819 struct page *page = virt_to_page(xdpf->data); in netsec_xdp_queue_one()
837 dma_handle = dma_map_single(priv->dev, xdpf->data, xdpf->len, in netsec_xdp_queue_one()
851 dma_handle = page_pool_get_dma_addr(page) + xdpf->headroom + in netsec_xdp_queue_one()
852 sizeof(*xdpf); in netsec_xdp_queue_one()
853 dma_sync_single_for_device(priv->dev, dma_handle, xdpf->len, in netsec_xdp_queue_one()
[all …]
|
| /kernel/linux/linux-5.10/drivers/net/ethernet/sfc/ |
| D | tx.c | 434 struct xdp_frame *xdpf; in efx_xdp_tx_buffers() local
464 xdpf = xdpfs[i]; in efx_xdp_tx_buffers()
472 len = xdpf->len; in efx_xdp_tx_buffers()
476 xdpf->data, len, in efx_xdp_tx_buffers()
483 tx_buffer->xdpf = xdpf; in efx_xdp_tx_buffers()
|
| D | rx.c | 260 struct xdp_frame *xdpf; in efx_do_xdp() local
324 xdpf = xdp_convert_buff_to_frame(&xdp); in efx_do_xdp()
325 err = efx_xdp_tx_buffers(efx, 1, &xdpf, true); in efx_do_xdp()
|
| /kernel/linux/linux-5.10/drivers/net/ethernet/intel/i40e/ |
| D | i40e_xsk.c | 456 xdp_return_frame(tx_bi->xdpf); in i40e_clean_xdp_tx_buffer()
496 if (tx_bi->xdpf) { in i40e_clean_xdp_tx_irq()
498 tx_bi->xdpf = NULL; in i40e_clean_xdp_tx_irq()
597 if (tx_bi->xdpf) in i40e_xsk_clean_tx_ring()
602 tx_bi->xdpf = NULL; in i40e_xsk_clean_tx_ring()
|
| D | i40e_txrx.c | 609 xdp_return_frame(tx_buffer->xdpf); in i40e_unmap_and_free_tx_resource()
815 xdp_return_frame(tx_buf->xdpf); in i40e_clean_tx_irq()
2168 static int i40e_xmit_xdp_ring(struct xdp_frame *xdpf,
2173 struct xdp_frame *xdpf = xdp_convert_buff_to_frame(xdp); in i40e_xmit_xdp_tx_ring() local
2175 if (unlikely(!xdpf)) in i40e_xmit_xdp_tx_ring()
2178 return i40e_xmit_xdp_ring(xdpf, xdp_ring); in i40e_xmit_xdp_tx_ring()
3581 static int i40e_xmit_xdp_ring(struct xdp_frame *xdpf, in i40e_xmit_xdp_ring() argument
3587 void *data = xdpf->data; in i40e_xmit_xdp_ring()
3588 u32 size = xdpf->len; in i40e_xmit_xdp_ring()
3602 tx_bi->xdpf = xdpf; in i40e_xmit_xdp_ring()
[all …]
|
| /kernel/linux/linux-5.10/drivers/net/ |
| D | virtio_net.c | 466 struct xdp_frame *xdpf) in __virtnet_xdp_xmit_one() argument
471 if (unlikely(xdpf->headroom < vi->hdr_len)) in __virtnet_xdp_xmit_one()
475 xdpf->data -= vi->hdr_len; in __virtnet_xdp_xmit_one()
477 hdr = xdpf->data; in __virtnet_xdp_xmit_one()
479 xdpf->len += vi->hdr_len; in __virtnet_xdp_xmit_one()
481 sg_init_one(sq->sg, xdpf->data, xdpf->len); in __virtnet_xdp_xmit_one()
483 err = virtqueue_add_outbuf(sq->vq, sq->sg, 1, xdp_to_ptr(xdpf), in __virtnet_xdp_xmit_one()
576 struct xdp_frame *xdpf = frames[i]; in virtnet_xdp_xmit() local
578 err = __virtnet_xdp_xmit_one(vi, sq, xdpf); in virtnet_xdp_xmit()
580 xdp_return_frame_rx_napi(xdpf); in virtnet_xdp_xmit()
[all …]
|
| D | xen-netfront.c | 609 struct xdp_frame *xdpf) in xennet_xdp_xmit_one() argument
616 .page = virt_to_page(xdpf->data), in xennet_xdp_xmit_one()
621 offset_in_page(xdpf->data), in xennet_xdp_xmit_one()
622 xdpf->len); in xennet_xdp_xmit_one()
631 tx_stats->bytes += xdpf->len; in xennet_xdp_xmit_one()
659 struct xdp_frame *xdpf = frames[i]; in xennet_xdp_xmit() local
661 if (!xdpf) in xennet_xdp_xmit()
663 err = xennet_xdp_xmit_one(dev, queue, xdpf); in xennet_xdp_xmit()
665 xdp_return_frame_rx_napi(xdpf); in xennet_xdp_xmit()
916 struct xdp_frame *xdpf; in xennet_run_xdp() local
[all …]
|
| /kernel/linux/linux-5.10/drivers/net/ethernet/marvell/ |
| D | mvneta.c | 615 struct xdp_frame *xdpf; member
1858 xdp_return_frame_rx_napi(buf->xdpf); in mvneta_txq_bufs_free()
1860 xdp_return_frame(buf->xdpf); in mvneta_txq_bufs_free()
2042 struct xdp_frame *xdpf, bool dma_map) in mvneta_xdp_submit_frame() argument
2056 dma_addr = dma_map_single(pp->dev->dev.parent, xdpf->data, in mvneta_xdp_submit_frame()
2057 xdpf->len, DMA_TO_DEVICE); in mvneta_xdp_submit_frame()
2064 struct page *page = virt_to_page(xdpf->data); in mvneta_xdp_submit_frame()
2067 sizeof(*xdpf) + xdpf->headroom; in mvneta_xdp_submit_frame()
2069 xdpf->len, DMA_BIDIRECTIONAL); in mvneta_xdp_submit_frame()
2072 buf->xdpf = xdpf; in mvneta_xdp_submit_frame()
[all …]
|
| /kernel/linux/linux-5.10/drivers/net/ethernet/amazon/ena/ |
| D | ena_netdev.c | 239 tx_info->xdpf = xdp_convert_buff_to_frame(xdp); in ena_xdp_tx_map_buff()
240 size = tx_info->xdpf->len; in ena_xdp_tx_map_buff()
245 *push_hdr = tx_info->xdpf->data; in ena_xdp_tx_map_buff()
270 xdp_return_frame_rx_napi(tx_info->xdpf); in ena_xdp_tx_map_buff()
271 tx_info->xdpf = NULL; in ena_xdp_tx_map_buff()
329 tx_info->xdpf = NULL; in ena_xdp_xmit_buff()
1216 if (likely(tx_info->xdpf)) in validate_xdp_req_id()
1801 struct xdp_frame *xdpf; in ena_clean_xdp_irq() local
1818 xdpf = tx_info->xdpf; in ena_clean_xdp_irq()
1820 tx_info->xdpf = NULL; in ena_clean_xdp_irq()
[all …]
|
| /kernel/linux/linux-5.10/drivers/net/ethernet/freescale/dpaa2/ |
| D | dpaa2-eth.c | 1039 xdp_return_frame(swa->xdp.xdpf); in dpaa2_eth_free_tx_fd()
2295 struct xdp_frame *xdpf, in dpaa2_eth_xdp_create_fd() argument
2308 if (xdpf->headroom < needed_headroom) in dpaa2_eth_xdp_create_fd()
2315 buffer_start = xdpf->data - needed_headroom; in dpaa2_eth_xdp_create_fd()
2318 if (aligned_start >= xdpf->data - xdpf->headroom) in dpaa2_eth_xdp_create_fd()
2324 swa->xdp.dma_size = xdpf->data + xdpf->len - buffer_start; in dpaa2_eth_xdp_create_fd()
2325 swa->xdp.xdpf = xdpf; in dpaa2_eth_xdp_create_fd()
2334 dpaa2_fd_set_offset(fd, xdpf->data - buffer_start); in dpaa2_eth_xdp_create_fd()
2335 dpaa2_fd_set_len(fd, xdpf->len); in dpaa2_eth_xdp_create_fd()
|
| /kernel/linux/linux-5.10/drivers/net/ethernet/intel/igb/ |
| D | igb.h | 269 struct xdp_frame *xdpf; member
718 struct xdp_frame *xdpf);
|
| /kernel/linux/linux-5.10/drivers/net/ethernet/marvell/mvpp2/ |
| D | mvpp2_main.c | 308 tx_buf->xdpf = data; in mvpp2_txq_inc_put()
2458 xdp_return_frame(tx_buf->xdpf); in mvpp2_txq_bufs_free()
3286 struct xdp_frame *xdpf, bool dma_map) in mvpp2_xdp_submit_frame() argument
3313 mvpp2_txdesc_size_set(port, tx_desc, xdpf->len); in mvpp2_xdp_submit_frame()
3317 dma_addr = dma_map_single(port->dev->dev.parent, xdpf->data, in mvpp2_xdp_submit_frame()
3318 xdpf->len, DMA_TO_DEVICE); in mvpp2_xdp_submit_frame()
3329 struct page *page = virt_to_page(xdpf->data); in mvpp2_xdp_submit_frame()
3332 sizeof(*xdpf) + xdpf->headroom; in mvpp2_xdp_submit_frame()
3334 xdpf->len, DMA_BIDIRECTIONAL); in mvpp2_xdp_submit_frame()
3342 mvpp2_txq_inc_put(port, txq_pcpu, xdpf, tx_desc, buf_type); in mvpp2_xdp_submit_frame()
[all …]
|