Lines Matching refs:txq
192 #define IS_TSO_HEADER(txq, addr) \ argument
193 ((addr >= txq->tso_hdrs_dma) && \
194 (addr < txq->tso_hdrs_dma + txq->tx_ring_size * TSO_HEADER_SIZE))
423 struct tx_queue txq[8]; member
461 static struct mv643xx_eth_private *txq_to_mp(struct tx_queue *txq) in txq_to_mp() argument
463 return container_of(txq, struct mv643xx_eth_private, txq[txq->index]); in txq_to_mp()
482 static void txq_reset_hw_ptr(struct tx_queue *txq) in txq_reset_hw_ptr() argument
484 struct mv643xx_eth_private *mp = txq_to_mp(txq); in txq_reset_hw_ptr()
487 addr = (u32)txq->tx_desc_dma; in txq_reset_hw_ptr()
488 addr += txq->tx_curr_desc * sizeof(struct tx_desc); in txq_reset_hw_ptr()
489 wrlp(mp, TXQ_CURRENT_DESC_PTR(txq->index), addr); in txq_reset_hw_ptr()
492 static void txq_enable(struct tx_queue *txq) in txq_enable() argument
494 struct mv643xx_eth_private *mp = txq_to_mp(txq); in txq_enable()
495 wrlp(mp, TXQ_COMMAND, 1 << txq->index); in txq_enable()
498 static void txq_disable(struct tx_queue *txq) in txq_disable() argument
500 struct mv643xx_eth_private *mp = txq_to_mp(txq); in txq_disable()
501 u8 mask = 1 << txq->index; in txq_disable()
508 static void txq_maybe_wake(struct tx_queue *txq) in txq_maybe_wake() argument
510 struct mv643xx_eth_private *mp = txq_to_mp(txq); in txq_maybe_wake()
511 struct netdev_queue *nq = netdev_get_tx_queue(mp->dev, txq->index); in txq_maybe_wake()
515 if (txq->tx_desc_count <= txq->tx_wake_threshold) in txq_maybe_wake()
746 txq_put_data_tso(struct net_device *dev, struct tx_queue *txq, in txq_put_data_tso() argument
754 tx_index = txq->tx_curr_desc++; in txq_put_data_tso()
755 if (txq->tx_curr_desc == txq->tx_ring_size) in txq_put_data_tso()
756 txq->tx_curr_desc = 0; in txq_put_data_tso()
757 desc = &txq->tx_desc_area[tx_index]; in txq_put_data_tso()
758 txq->tx_desc_mapping[tx_index] = DESC_DMA_MAP_SINGLE; in txq_put_data_tso()
765 memcpy(txq->tso_hdrs + tx_index * TSO_HEADER_SIZE, in txq_put_data_tso()
767 desc->buf_ptr = txq->tso_hdrs_dma in txq_put_data_tso()
771 txq->tx_desc_mapping[tx_index] = DESC_DMA_MAP_SINGLE; in txq_put_data_tso()
794 txq_put_hdr_tso(struct sk_buff *skb, struct tx_queue *txq, int length, in txq_put_hdr_tso() argument
797 struct mv643xx_eth_private *mp = txq_to_mp(txq); in txq_put_hdr_tso()
806 tx_index = txq->tx_curr_desc; in txq_put_hdr_tso()
807 desc = &txq->tx_desc_area[tx_index]; in txq_put_hdr_tso()
818 desc->buf_ptr = txq->tso_hdrs_dma + in txq_put_hdr_tso()
819 txq->tx_curr_desc * TSO_HEADER_SIZE; in txq_put_hdr_tso()
831 txq->tx_curr_desc++; in txq_put_hdr_tso()
832 if (txq->tx_curr_desc == txq->tx_ring_size) in txq_put_hdr_tso()
833 txq->tx_curr_desc = 0; in txq_put_hdr_tso()
836 static int txq_submit_tso(struct tx_queue *txq, struct sk_buff *skb, in txq_submit_tso() argument
839 struct mv643xx_eth_private *mp = txq_to_mp(txq); in txq_submit_tso()
848 if ((txq->tx_desc_count + tso_count_descs(skb)) >= txq->tx_ring_size) { in txq_submit_tso()
853 first_tx_desc = &txq->tx_desc_area[txq->tx_curr_desc]; in txq_submit_tso()
868 hdr = txq->tso_hdrs + txq->tx_curr_desc * TSO_HEADER_SIZE; in txq_submit_tso()
870 txq_put_hdr_tso(skb, txq, data_left, &first_cmd_sts, in txq_submit_tso()
878 ret = txq_put_data_tso(dev, txq, skb, tso.data, size, in txq_submit_tso()
888 __skb_queue_tail(&txq->tx_skb, skb); in txq_submit_tso()
896 mp->work_tx_end &= ~(1 << txq->index); in txq_submit_tso()
900 txq_enable(txq); in txq_submit_tso()
901 txq->tx_desc_count += desc_count; in txq_submit_tso()
910 static void txq_submit_frag_skb(struct tx_queue *txq, struct sk_buff *skb) in txq_submit_frag_skb() argument
912 struct mv643xx_eth_private *mp = txq_to_mp(txq); in txq_submit_frag_skb()
922 tx_index = txq->tx_curr_desc++; in txq_submit_frag_skb()
923 if (txq->tx_curr_desc == txq->tx_ring_size) in txq_submit_frag_skb()
924 txq->tx_curr_desc = 0; in txq_submit_frag_skb()
925 desc = &txq->tx_desc_area[tx_index]; in txq_submit_frag_skb()
926 txq->tx_desc_mapping[tx_index] = DESC_DMA_MAP_PAGE; in txq_submit_frag_skb()
948 static int txq_submit_skb(struct tx_queue *txq, struct sk_buff *skb, in txq_submit_skb() argument
951 struct mv643xx_eth_private *mp = txq_to_mp(txq); in txq_submit_skb()
962 if (txq->tx_ring_size - txq->tx_desc_count < MAX_SKB_FRAGS + 1) { in txq_submit_skb()
973 tx_index = txq->tx_curr_desc++; in txq_submit_skb()
974 if (txq->tx_curr_desc == txq->tx_ring_size) in txq_submit_skb()
975 txq->tx_curr_desc = 0; in txq_submit_skb()
976 desc = &txq->tx_desc_area[tx_index]; in txq_submit_skb()
977 txq->tx_desc_mapping[tx_index] = DESC_DMA_MAP_SINGLE; in txq_submit_skb()
980 txq_submit_frag_skb(txq, skb); in txq_submit_skb()
992 __skb_queue_tail(&txq->tx_skb, skb); in txq_submit_skb()
1001 mp->work_tx_end &= ~(1 << txq->index); in txq_submit_skb()
1005 txq_enable(txq); in txq_submit_skb()
1007 txq->tx_desc_count += nr_frags + 1; in txq_submit_skb()
1016 struct tx_queue *txq; in mv643xx_eth_xmit() local
1020 txq = mp->txq + queue; in mv643xx_eth_xmit()
1032 ret = txq_submit_tso(txq, skb, dev); in mv643xx_eth_xmit()
1034 ret = txq_submit_skb(txq, skb, dev); in mv643xx_eth_xmit()
1036 txq->tx_bytes += length; in mv643xx_eth_xmit()
1037 txq->tx_packets++; in mv643xx_eth_xmit()
1039 if (txq->tx_desc_count >= txq->tx_stop_threshold) in mv643xx_eth_xmit()
1042 txq->tx_dropped++; in mv643xx_eth_xmit()
1051 static void txq_kick(struct tx_queue *txq) in txq_kick() argument
1053 struct mv643xx_eth_private *mp = txq_to_mp(txq); in txq_kick()
1054 struct netdev_queue *nq = netdev_get_tx_queue(mp->dev, txq->index); in txq_kick()
1060 if (rdlp(mp, TXQ_COMMAND) & (1 << txq->index)) in txq_kick()
1063 hw_desc_ptr = rdlp(mp, TXQ_CURRENT_DESC_PTR(txq->index)); in txq_kick()
1064 expected_ptr = (u32)txq->tx_desc_dma + in txq_kick()
1065 txq->tx_curr_desc * sizeof(struct tx_desc); in txq_kick()
1068 txq_enable(txq); in txq_kick()
1073 mp->work_tx_end &= ~(1 << txq->index); in txq_kick()
1076 static int txq_reclaim(struct tx_queue *txq, int budget, int force) in txq_reclaim() argument
1078 struct mv643xx_eth_private *mp = txq_to_mp(txq); in txq_reclaim()
1079 struct netdev_queue *nq = netdev_get_tx_queue(mp->dev, txq->index); in txq_reclaim()
1085 while (reclaimed < budget && txq->tx_desc_count > 0) { in txq_reclaim()
1091 tx_index = txq->tx_used_desc; in txq_reclaim()
1092 desc = &txq->tx_desc_area[tx_index]; in txq_reclaim()
1093 desc_dma_map = txq->tx_desc_mapping[tx_index]; in txq_reclaim()
1103 txq->tx_used_desc = tx_index + 1; in txq_reclaim()
1104 if (txq->tx_used_desc == txq->tx_ring_size) in txq_reclaim()
1105 txq->tx_used_desc = 0; in txq_reclaim()
1108 txq->tx_desc_count--; in txq_reclaim()
1110 if (!IS_TSO_HEADER(txq, desc->buf_ptr)) { in txq_reclaim()
1125 struct sk_buff *skb = __skb_dequeue(&txq->tx_skb); in txq_reclaim()
1141 mp->work_tx &= ~(1 << txq->index); in txq_reclaim()
1184 static void txq_set_rate(struct tx_queue *txq, int rate, int burst) in txq_set_rate() argument
1186 struct mv643xx_eth_private *mp = txq_to_mp(txq); in txq_set_rate()
1198 wrlp(mp, TXQ_BW_TOKENS(txq->index), token_rate << 14); in txq_set_rate()
1199 wrlp(mp, TXQ_BW_CONF(txq->index), (bucket_size << 10) | token_rate); in txq_set_rate()
1202 static void txq_set_fixed_prio_mode(struct tx_queue *txq) in txq_set_fixed_prio_mode() argument
1204 struct mv643xx_eth_private *mp = txq_to_mp(txq); in txq_set_fixed_prio_mode()
1223 val |= 1 << txq->index; in txq_set_fixed_prio_mode()
1281 struct tx_queue *txq = mp->txq + i; in mv643xx_eth_get_stats() local
1283 tx_packets += txq->tx_packets; in mv643xx_eth_get_stats()
1284 tx_bytes += txq->tx_bytes; in mv643xx_eth_get_stats()
1285 tx_dropped += txq->tx_dropped; in mv643xx_eth_get_stats()
2047 struct tx_queue *txq = mp->txq + index; in txq_init() local
2053 txq->index = index; in txq_init()
2055 txq->tx_ring_size = mp->tx_ring_size; in txq_init()
2061 txq->tx_stop_threshold = txq->tx_ring_size - MV643XX_MAX_SKB_DESCS; in txq_init()
2062 txq->tx_wake_threshold = txq->tx_stop_threshold / 2; in txq_init()
2064 txq->tx_desc_count = 0; in txq_init()
2065 txq->tx_curr_desc = 0; in txq_init()
2066 txq->tx_used_desc = 0; in txq_init()
2068 size = txq->tx_ring_size * sizeof(struct tx_desc); in txq_init()
2071 txq->tx_desc_area = ioremap(mp->tx_desc_sram_addr, in txq_init()
2073 txq->tx_desc_dma = mp->tx_desc_sram_addr; in txq_init()
2075 txq->tx_desc_area = dma_alloc_coherent(mp->dev->dev.parent, in txq_init()
2076 size, &txq->tx_desc_dma, in txq_init()
2080 if (txq->tx_desc_area == NULL) { in txq_init()
2085 memset(txq->tx_desc_area, 0, size); in txq_init()
2087 txq->tx_desc_area_size = size; in txq_init()
2089 tx_desc = txq->tx_desc_area; in txq_init()
2090 for (i = 0; i < txq->tx_ring_size; i++) { in txq_init()
2095 if (nexti == txq->tx_ring_size) in txq_init()
2099 txd->next_desc_ptr = txq->tx_desc_dma + in txq_init()
2103 txq->tx_desc_mapping = kcalloc(txq->tx_ring_size, sizeof(char), in txq_init()
2105 if (!txq->tx_desc_mapping) { in txq_init()
2111 txq->tso_hdrs = dma_alloc_coherent(mp->dev->dev.parent, in txq_init()
2112 txq->tx_ring_size * TSO_HEADER_SIZE, in txq_init()
2113 &txq->tso_hdrs_dma, GFP_KERNEL); in txq_init()
2114 if (txq->tso_hdrs == NULL) { in txq_init()
2118 skb_queue_head_init(&txq->tx_skb); in txq_init()
2123 kfree(txq->tx_desc_mapping); in txq_init()
2126 iounmap(txq->tx_desc_area); in txq_init()
2128 dma_free_coherent(mp->dev->dev.parent, txq->tx_desc_area_size, in txq_init()
2129 txq->tx_desc_area, txq->tx_desc_dma); in txq_init()
2133 static void txq_deinit(struct tx_queue *txq) in txq_deinit() argument
2135 struct mv643xx_eth_private *mp = txq_to_mp(txq); in txq_deinit()
2137 txq_disable(txq); in txq_deinit()
2138 txq_reclaim(txq, txq->tx_ring_size, 1); in txq_deinit()
2140 BUG_ON(txq->tx_used_desc != txq->tx_curr_desc); in txq_deinit()
2142 if (txq->index == 0 && in txq_deinit()
2143 txq->tx_desc_area_size <= mp->tx_desc_sram_size) in txq_deinit()
2144 iounmap(txq->tx_desc_area); in txq_deinit()
2146 dma_free_coherent(mp->dev->dev.parent, txq->tx_desc_area_size, in txq_deinit()
2147 txq->tx_desc_area, txq->tx_desc_dma); in txq_deinit()
2148 kfree(txq->tx_desc_mapping); in txq_deinit()
2150 if (txq->tso_hdrs) in txq_deinit()
2152 txq->tx_ring_size * TSO_HEADER_SIZE, in txq_deinit()
2153 txq->tso_hdrs, txq->tso_hdrs_dma); in txq_deinit()
2223 struct tx_queue *txq = mp->txq + i; in handle_link_event() local
2225 txq_reclaim(txq, txq->tx_ring_size, 1); in handle_link_event()
2226 txq_reset_hw_ptr(txq); in handle_link_event()
2299 txq_kick(mp->txq + queue); in mv643xx_eth_poll()
2301 work_done += txq_reclaim(mp->txq + queue, work_tbd, 0); in mv643xx_eth_poll()
2302 txq_maybe_wake(mp->txq + queue); in mv643xx_eth_poll()
2364 struct tx_queue *txq = mp->txq + i; in port_start() local
2366 txq_reset_hw_ptr(txq); in port_start()
2367 txq_set_rate(txq, 1000000000, 16777216); in port_start()
2368 txq_set_fixed_prio_mode(txq); in port_start()
2475 txq_deinit(mp->txq + i); in mv643xx_eth_open()
2507 txq_disable(mp->txq + i); in port_reset()
2551 txq_deinit(mp->txq + i); in mv643xx_eth_stop()