44 				     struct efx_rx_buffer *rx_buf,  in efx_rx_packet__check_len()  argument
48 	unsigned max_len = rx_buf->len - efx->type->rx_buffer_padding;  in efx_rx_packet__check_len()
56 	rx_buf->flags |= EFX_RX_PKT_DISCARD;  in efx_rx_packet__check_len()
68 				     struct efx_rx_buffer *rx_buf,  in efx_rx_mk_skb()  argument
84 	EFX_WARN_ON_ONCE_PARANOID(rx_buf->len < hdr_len);  in efx_rx_mk_skb()
92 	if (rx_buf->len > hdr_len) {  in efx_rx_mk_skb()
93 		rx_buf->page_offset += hdr_len;  in efx_rx_mk_skb()
94 		rx_buf->len -= hdr_len;  in efx_rx_mk_skb()
98 					   rx_buf->page, rx_buf->page_offset,  in efx_rx_mk_skb()
99 					   rx_buf->len);  in efx_rx_mk_skb()
100 			rx_buf->page = NULL;  in efx_rx_mk_skb()
101 			skb->len += rx_buf->len;  in efx_rx_mk_skb()
102 			skb->data_len += rx_buf->len;  in efx_rx_mk_skb()
106 			rx_buf = efx_rx_buf_next(&channel->rx_queue, rx_buf);  in efx_rx_mk_skb()
109 		__free_pages(rx_buf->page, efx->rx_buffer_order);  in efx_rx_mk_skb()
110 		rx_buf->page = NULL;  in efx_rx_mk_skb()
129 	struct efx_rx_buffer *rx_buf;  in efx_rx_packet()  local
133 	rx_buf = efx_rx_buffer(rx_queue, index);  in efx_rx_packet()
134 	rx_buf->flags |= flags;  in efx_rx_packet()
139 			efx_rx_packet__check_len(rx_queue, rx_buf, len);  in efx_rx_packet()
147 		WARN_ON(!(len == 0 && rx_buf->flags & EFX_RX_PKT_DISCARD));  in efx_rx_packet()
148 		rx_buf->flags |= EFX_RX_PKT_DISCARD;  in efx_rx_packet()
155 		   (rx_buf->flags & EFX_RX_PKT_CSUMMED) ? " [SUMMED]" : "",  in efx_rx_packet()
156 		   (rx_buf->flags & EFX_RX_PKT_DISCARD) ? " [DISCARD]" : "");  in efx_rx_packet()
161 	if (unlikely(rx_buf->flags & EFX_RX_PKT_DISCARD)) {  in efx_rx_packet()
163 		efx_discard_rx_packet(channel, rx_buf, n_frags);  in efx_rx_packet()
168 		rx_buf->len = len;  in efx_rx_packet()
173 	efx_sync_rx_buffer(efx, rx_buf, rx_buf->len);  in efx_rx_packet()
178 	prefetch(efx_rx_buf_va(rx_buf));  in efx_rx_packet()
180 	rx_buf->page_offset += efx->rx_prefix_size;  in efx_rx_packet()
181 	rx_buf->len -= efx->rx_prefix_size;  in efx_rx_packet()
190 			rx_buf = efx_rx_buf_next(rx_queue, rx_buf);  in efx_rx_packet()
193 			efx_sync_rx_buffer(efx, rx_buf, efx->rx_dma_len);  in efx_rx_packet()
195 		rx_buf->len = len - (n_frags - 1) * efx->rx_dma_len;  in efx_rx_packet()
196 		efx_sync_rx_buffer(efx, rx_buf, rx_buf->len);  in efx_rx_packet()
200 	rx_buf = efx_rx_buffer(rx_queue, index);  in efx_rx_packet()
201 	efx_recycle_rx_pages(channel, rx_buf, n_frags);  in efx_rx_packet()
212 			   struct efx_rx_buffer *rx_buf,  in efx_rx_deliver()  argument
216 	u16 hdr_len = min_t(u16, rx_buf->len, EFX_SKB_HEADERS);  in efx_rx_deliver()
218 	skb = efx_rx_mk_skb(channel, rx_buf, n_frags, eh, hdr_len);  in efx_rx_deliver()
223 		efx_free_rx_buffers(rx_queue, rx_buf, n_frags);  in efx_rx_deliver()
230 	if (likely(rx_buf->flags & EFX_RX_PKT_CSUMMED)) {  in efx_rx_deliver()
232 		skb->csum_level = !!(rx_buf->flags & EFX_RX_PKT_CSUM_LEVEL);  in efx_rx_deliver()
255 		       struct efx_rx_buffer *rx_buf, u8 **ehp)  in efx_do_xdp()  argument
278 		efx_free_rx_buffers(rx_queue, rx_buf,  in efx_do_xdp()
288 	dma_sync_single_for_cpu(&efx->pci_dev->dev, rx_buf->dma_addr,  in efx_do_xdp()
289 				rx_buf->len, DMA_FROM_DEVICE);  in efx_do_xdp()
301 	xdp.data_end = xdp.data + rx_buf->len;  in efx_do_xdp()
315 			rx_buf->page_offset += offset;  in efx_do_xdp()
316 			rx_buf->len -= offset;  in efx_do_xdp()
327 			efx_free_rx_buffers(rx_queue, rx_buf, 1);  in efx_do_xdp()
341 			efx_free_rx_buffers(rx_queue, rx_buf, 1);  in efx_do_xdp()
354 		efx_free_rx_buffers(rx_queue, rx_buf, 1);  in efx_do_xdp()
363 		efx_free_rx_buffers(rx_queue, rx_buf, 1);  in efx_do_xdp()
375 	struct efx_rx_buffer *rx_buf =  in __efx_rx_packet()  local
377 	u8 *eh = efx_rx_buf_va(rx_buf);  in __efx_rx_packet()
382 	if (rx_buf->flags & EFX_RX_PKT_PREFIX_LEN)  in __efx_rx_packet()
383 		rx_buf->len = le16_to_cpup((__le16 *)  in __efx_rx_packet()
392 		efx_loopback_rx_packet(efx, eh, rx_buf->len);  in __efx_rx_packet()
394 		efx_free_rx_buffers(rx_queue, rx_buf,  in __efx_rx_packet()
399 	if (!efx_do_xdp(efx, channel, rx_buf, &eh))  in __efx_rx_packet()
403 		rx_buf->flags &= ~EFX_RX_PKT_CSUMMED;  in __efx_rx_packet()
405 	if ((rx_buf->flags & EFX_RX_PKT_TCP) && !channel->type->receive_skb)  in __efx_rx_packet()
406 		efx_rx_packet_gro(channel, rx_buf, channel->rx_pkt_n_frags, eh, 0);  in __efx_rx_packet()
408 		efx_rx_deliver(channel, eh, rx_buf, channel->rx_pkt_n_frags);  in __efx_rx_packet()