Lines Matching refs:tx_queue
25 static inline u8 *ef4_tx_get_copy_buffer(struct ef4_tx_queue *tx_queue, in ef4_tx_get_copy_buffer() argument
28 unsigned int index = ef4_tx_queue_get_insert_index(tx_queue); in ef4_tx_get_copy_buffer()
30 &tx_queue->cb_page[index >> (PAGE_SHIFT - EF4_TX_CB_ORDER)]; in ef4_tx_get_copy_buffer()
35 ef4_nic_alloc_buffer(tx_queue->efx, page_buf, PAGE_SIZE, in ef4_tx_get_copy_buffer()
43 u8 *ef4_tx_get_copy_buffer_limited(struct ef4_tx_queue *tx_queue, in ef4_tx_get_copy_buffer_limited() argument
48 return ef4_tx_get_copy_buffer(tx_queue, buffer); in ef4_tx_get_copy_buffer_limited()
51 static void ef4_dequeue_buffer(struct ef4_tx_queue *tx_queue, in ef4_dequeue_buffer() argument
57 struct device *dma_dev = &tx_queue->efx->pci_dev->dev; in ef4_dequeue_buffer()
72 netif_vdbg(tx_queue->efx, tx_done, tx_queue->efx->net_dev, in ef4_dequeue_buffer()
74 tx_queue->queue, tx_queue->read_count); in ef4_dequeue_buffer()
147 static int ef4_enqueue_skb_copy(struct ef4_tx_queue *tx_queue, in ef4_enqueue_skb_copy() argument
150 unsigned int min_len = tx_queue->tx_min_size; in ef4_enqueue_skb_copy()
158 buffer = ef4_tx_queue_get_insert_buffer(tx_queue); in ef4_enqueue_skb_copy()
160 copy_buffer = ef4_tx_get_copy_buffer(tx_queue, buffer); in ef4_enqueue_skb_copy()
176 ++tx_queue->insert_count; in ef4_enqueue_skb_copy()
180 static struct ef4_tx_buffer *ef4_tx_map_chunk(struct ef4_tx_queue *tx_queue, in ef4_tx_map_chunk() argument
184 const struct ef4_nic_type *nic_type = tx_queue->efx->type; in ef4_tx_map_chunk()
190 buffer = ef4_tx_queue_get_insert_buffer(tx_queue); in ef4_tx_map_chunk()
191 dma_len = nic_type->tx_limit_len(tx_queue, dma_addr, len); in ef4_tx_map_chunk()
198 ++tx_queue->insert_count; in ef4_tx_map_chunk()
206 static int ef4_tx_map_data(struct ef4_tx_queue *tx_queue, struct sk_buff *skb) in ef4_tx_map_data() argument
208 struct ef4_nic *efx = tx_queue->efx; in ef4_tx_map_data()
233 buffer = ef4_tx_map_chunk(tx_queue, dma_addr, len); in ef4_tx_map_data()
268 static void ef4_enqueue_unwind(struct ef4_tx_queue *tx_queue) in ef4_enqueue_unwind() argument
273 while (tx_queue->insert_count != tx_queue->write_count) { in ef4_enqueue_unwind()
274 --tx_queue->insert_count; in ef4_enqueue_unwind()
275 buffer = __ef4_tx_queue_get_insert_buffer(tx_queue); in ef4_enqueue_unwind()
276 ef4_dequeue_buffer(tx_queue, buffer, NULL, NULL); in ef4_enqueue_unwind()
296 netdev_tx_t ef4_enqueue_skb(struct ef4_tx_queue *tx_queue, struct sk_buff *skb) in ef4_enqueue_skb() argument
304 if (skb_len < tx_queue->tx_min_size || in ef4_enqueue_skb()
307 if (ef4_enqueue_skb_copy(tx_queue, skb)) in ef4_enqueue_skb()
309 tx_queue->cb_packets++; in ef4_enqueue_skb()
314 if (!data_mapped && (ef4_tx_map_data(tx_queue, skb))) in ef4_enqueue_skb()
318 netdev_tx_sent_queue(tx_queue->core_txq, skb_len); in ef4_enqueue_skb()
321 if (!netdev_xmit_more() || netif_xmit_stopped(tx_queue->core_txq)) { in ef4_enqueue_skb()
322 struct ef4_tx_queue *txq2 = ef4_tx_queue_partner(tx_queue); in ef4_enqueue_skb()
331 ef4_nic_push_buffers(tx_queue); in ef4_enqueue_skb()
333 tx_queue->xmit_more_available = netdev_xmit_more(); in ef4_enqueue_skb()
336 tx_queue->tx_packets++; in ef4_enqueue_skb()
338 ef4_tx_maybe_stop_queue(tx_queue); in ef4_enqueue_skb()
344 ef4_enqueue_unwind(tx_queue); in ef4_enqueue_skb()
354 static void ef4_dequeue_buffers(struct ef4_tx_queue *tx_queue, in ef4_dequeue_buffers() argument
359 struct ef4_nic *efx = tx_queue->efx; in ef4_dequeue_buffers()
362 stop_index = (index + 1) & tx_queue->ptr_mask; in ef4_dequeue_buffers()
363 read_ptr = tx_queue->read_count & tx_queue->ptr_mask; in ef4_dequeue_buffers()
366 struct ef4_tx_buffer *buffer = &tx_queue->buffer[read_ptr]; in ef4_dequeue_buffers()
372 tx_queue->queue, read_ptr); in ef4_dequeue_buffers()
377 ef4_dequeue_buffer(tx_queue, buffer, pkts_compl, bytes_compl); in ef4_dequeue_buffers()
379 ++tx_queue->read_count; in ef4_dequeue_buffers()
380 read_ptr = tx_queue->read_count & tx_queue->ptr_mask; in ef4_dequeue_buffers()
397 struct ef4_tx_queue *tx_queue; in ef4_hard_start_xmit() local
408 tx_queue = ef4_get_tx_queue(efx, index, type); in ef4_hard_start_xmit()
410 return ef4_enqueue_skb(tx_queue, skb); in ef4_hard_start_xmit()
413 void ef4_init_tx_queue_core_txq(struct ef4_tx_queue *tx_queue) in ef4_init_tx_queue_core_txq() argument
415 struct ef4_nic *efx = tx_queue->efx; in ef4_init_tx_queue_core_txq()
418 tx_queue->core_txq = in ef4_init_tx_queue_core_txq()
420 tx_queue->queue / EF4_TXQ_TYPES + in ef4_init_tx_queue_core_txq()
421 ((tx_queue->queue & EF4_TXQ_TYPE_HIGHPRI) ? in ef4_init_tx_queue_core_txq()
431 struct ef4_tx_queue *tx_queue; in ef4_setup_tc() local
456 ef4_for_each_possible_channel_tx_queue(tx_queue, in ef4_setup_tc()
458 if (!(tx_queue->queue & EF4_TXQ_TYPE_HIGHPRI)) in ef4_setup_tc()
460 if (!tx_queue->buffer) { in ef4_setup_tc()
461 rc = ef4_probe_tx_queue(tx_queue); in ef4_setup_tc()
465 if (!tx_queue->initialised) in ef4_setup_tc()
466 ef4_init_tx_queue(tx_queue); in ef4_setup_tc()
467 ef4_init_tx_queue_core_txq(tx_queue); in ef4_setup_tc()
491 void ef4_xmit_done(struct ef4_tx_queue *tx_queue, unsigned int index) in ef4_xmit_done() argument
494 struct ef4_nic *efx = tx_queue->efx; in ef4_xmit_done()
498 EF4_BUG_ON_PARANOID(index > tx_queue->ptr_mask); in ef4_xmit_done()
500 ef4_dequeue_buffers(tx_queue, index, &pkts_compl, &bytes_compl); in ef4_xmit_done()
501 tx_queue->pkts_compl += pkts_compl; in ef4_xmit_done()
502 tx_queue->bytes_compl += bytes_compl; in ef4_xmit_done()
505 ++tx_queue->merge_events; in ef4_xmit_done()
512 if (unlikely(netif_tx_queue_stopped(tx_queue->core_txq)) && in ef4_xmit_done()
515 txq2 = ef4_tx_queue_partner(tx_queue); in ef4_xmit_done()
516 fill_level = max(tx_queue->insert_count - tx_queue->read_count, in ef4_xmit_done()
519 netif_tx_wake_queue(tx_queue->core_txq); in ef4_xmit_done()
523 if ((int)(tx_queue->read_count - tx_queue->old_write_count) >= 0) { in ef4_xmit_done()
524 tx_queue->old_write_count = READ_ONCE(tx_queue->write_count); in ef4_xmit_done()
525 if (tx_queue->read_count == tx_queue->old_write_count) { in ef4_xmit_done()
527 tx_queue->empty_read_count = in ef4_xmit_done()
528 tx_queue->read_count | EF4_EMPTY_COUNT_VALID; in ef4_xmit_done()
533 static unsigned int ef4_tx_cb_page_count(struct ef4_tx_queue *tx_queue) in ef4_tx_cb_page_count() argument
535 return DIV_ROUND_UP(tx_queue->ptr_mask + 1, PAGE_SIZE >> EF4_TX_CB_ORDER); in ef4_tx_cb_page_count()
538 int ef4_probe_tx_queue(struct ef4_tx_queue *tx_queue) in ef4_probe_tx_queue() argument
540 struct ef4_nic *efx = tx_queue->efx; in ef4_probe_tx_queue()
547 tx_queue->ptr_mask = entries - 1; in ef4_probe_tx_queue()
551 tx_queue->queue, efx->txq_entries, tx_queue->ptr_mask); in ef4_probe_tx_queue()
554 tx_queue->buffer = kcalloc(entries, sizeof(*tx_queue->buffer), in ef4_probe_tx_queue()
556 if (!tx_queue->buffer) in ef4_probe_tx_queue()
559 tx_queue->cb_page = kcalloc(ef4_tx_cb_page_count(tx_queue), in ef4_probe_tx_queue()
560 sizeof(tx_queue->cb_page[0]), GFP_KERNEL); in ef4_probe_tx_queue()
561 if (!tx_queue->cb_page) { in ef4_probe_tx_queue()
567 rc = ef4_nic_probe_tx(tx_queue); in ef4_probe_tx_queue()
574 kfree(tx_queue->cb_page); in ef4_probe_tx_queue()
575 tx_queue->cb_page = NULL; in ef4_probe_tx_queue()
577 kfree(tx_queue->buffer); in ef4_probe_tx_queue()
578 tx_queue->buffer = NULL; in ef4_probe_tx_queue()
582 void ef4_init_tx_queue(struct ef4_tx_queue *tx_queue) in ef4_init_tx_queue() argument
584 struct ef4_nic *efx = tx_queue->efx; in ef4_init_tx_queue()
587 "initialising TX queue %d\n", tx_queue->queue); in ef4_init_tx_queue()
589 tx_queue->insert_count = 0; in ef4_init_tx_queue()
590 tx_queue->write_count = 0; in ef4_init_tx_queue()
591 tx_queue->old_write_count = 0; in ef4_init_tx_queue()
592 tx_queue->read_count = 0; in ef4_init_tx_queue()
593 tx_queue->old_read_count = 0; in ef4_init_tx_queue()
594 tx_queue->empty_read_count = 0 | EF4_EMPTY_COUNT_VALID; in ef4_init_tx_queue()
595 tx_queue->xmit_more_available = false; in ef4_init_tx_queue()
598 tx_queue->tx_min_size = EF4_WORKAROUND_15592(efx) ? 33 : 0; in ef4_init_tx_queue()
601 ef4_nic_init_tx(tx_queue); in ef4_init_tx_queue()
603 tx_queue->initialised = true; in ef4_init_tx_queue()
606 void ef4_fini_tx_queue(struct ef4_tx_queue *tx_queue) in ef4_fini_tx_queue() argument
610 netif_dbg(tx_queue->efx, drv, tx_queue->efx->net_dev, in ef4_fini_tx_queue()
611 "shutting down TX queue %d\n", tx_queue->queue); in ef4_fini_tx_queue()
613 if (!tx_queue->buffer) in ef4_fini_tx_queue()
617 while (tx_queue->read_count != tx_queue->write_count) { in ef4_fini_tx_queue()
619 buffer = &tx_queue->buffer[tx_queue->read_count & tx_queue->ptr_mask]; in ef4_fini_tx_queue()
620 ef4_dequeue_buffer(tx_queue, buffer, &pkts_compl, &bytes_compl); in ef4_fini_tx_queue()
622 ++tx_queue->read_count; in ef4_fini_tx_queue()
624 tx_queue->xmit_more_available = false; in ef4_fini_tx_queue()
625 netdev_tx_reset_queue(tx_queue->core_txq); in ef4_fini_tx_queue()
628 void ef4_remove_tx_queue(struct ef4_tx_queue *tx_queue) in ef4_remove_tx_queue() argument
632 if (!tx_queue->buffer) in ef4_remove_tx_queue()
635 netif_dbg(tx_queue->efx, drv, tx_queue->efx->net_dev, in ef4_remove_tx_queue()
636 "destroying TX queue %d\n", tx_queue->queue); in ef4_remove_tx_queue()
637 ef4_nic_remove_tx(tx_queue); in ef4_remove_tx_queue()
639 if (tx_queue->cb_page) { in ef4_remove_tx_queue()
640 for (i = 0; i < ef4_tx_cb_page_count(tx_queue); i++) in ef4_remove_tx_queue()
641 ef4_nic_free_buffer(tx_queue->efx, in ef4_remove_tx_queue()
642 &tx_queue->cb_page[i]); in ef4_remove_tx_queue()
643 kfree(tx_queue->cb_page); in ef4_remove_tx_queue()
644 tx_queue->cb_page = NULL; in ef4_remove_tx_queue()
647 kfree(tx_queue->buffer); in ef4_remove_tx_queue()
648 tx_queue->buffer = NULL; in ef4_remove_tx_queue()