Lines Matching +full:rx +full:- +full:queues +full:- +full:to +full:- +full:use
4 * Copyright (c) 2002-2005, K A Fraser
11 * software packages, subject to the following license:
13 * Permission is hereby granted, free of charge, to any person obtaining a copy
14 * of this source file (the "Software"), to deal in the Software without
15 * restriction, including without limitation the rights to use, copy, modify,
17 * and to permit persons to whom the Software is furnished to do so, subject to
24 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
64 "Maximum number of queues per virtual interface");
74 #define NETFRONT_SKB_CB(skb) ((struct netfront_cb *)((skb)->cb))
83 /* Minimum number of Rx slots (includes slot for GSO metadata). */
86 /* Queue name is interface name with "-qNNN" appended */
89 /* IRQ name is queue name with "-tx" or "-rx" appended */
103 unsigned int id; /* Queue ID, 0-based */
104 char name[QUEUE_NAME_SIZE]; /* DEVNAME-qN */
115 char tx_irq_name[IRQ_NAME_SIZE]; /* DEVNAME-qN-tx */
116 char rx_irq_name[IRQ_NAME_SIZE]; /* DEVNAME-qN-rx */
123 * {tx,rx}_skbs store outstanding skbuffs. Free tx_skb entries
126 * NB. Freelist index entries are always going to be less than
127 * PAGE_OFFSET, whereas pointers to skbs will always be equal or
128 * greater than PAGE_OFFSET: we use this property to distinguish
141 struct xen_netif_rx_front_ring rx; member
157 /* Multi-queue support */
158 struct netfront_queue *queues; member
168 struct xen_netif_rx_response rx; member
169 struct xen_netif_extra_info extras[XEN_NETIF_EXTRA_TYPE_MAX - 1];
174 list->link = id; in skb_entry_set_link()
179 BUILD_BUG_ON(sizeof(list->skb) != sizeof(list->link)); in skb_entry_is_link()
180 return (unsigned long)list->skb < PAGE_OFFSET; in skb_entry_is_link()
204 return idx & (NET_RX_RING_SIZE - 1); in xennet_rxidx()
211 struct sk_buff *skb = queue->rx_skbs[i]; in xennet_get_rx_skb()
212 queue->rx_skbs[i] = NULL; in xennet_get_rx_skb()
220 grant_ref_t ref = queue->grant_rx_ref[i]; in xennet_get_rx_ref()
221 queue->grant_rx_ref[i] = GRANT_INVALID_REF; in xennet_get_rx_ref()
231 return dev->features & NETIF_F_SG; in xennet_can_sg()
238 napi_schedule(&queue->napi); in rx_refill_timeout()
243 return (queue->tx.req_prod_pvt - queue->tx.rsp_cons) < in netfront_tx_slot_available()
244 (NET_TX_RING_SIZE - XEN_NETIF_NR_SLOTS_MIN - 1); in netfront_tx_slot_available()
249 struct net_device *dev = queue->info->netdev; in xennet_maybe_wake_tx()
250 struct netdev_queue *dev_queue = netdev_get_tx_queue(dev, queue->id); in xennet_maybe_wake_tx()
255 netif_tx_wake_queue(netdev_get_tx_queue(dev, queue->id)); in xennet_maybe_wake_tx()
264 skb = __netdev_alloc_skb(queue->info->netdev, in xennet_alloc_one_rx_buffer()
277 /* Align ip header to a 16 bytes boundary */ in xennet_alloc_one_rx_buffer()
279 skb->dev = queue->info->netdev; in xennet_alloc_one_rx_buffer()
287 RING_IDX req_prod = queue->rx.req_prod_pvt; in xennet_alloc_rx_buffers()
291 if (unlikely(!netif_carrier_ok(queue->info->netdev))) in xennet_alloc_rx_buffers()
294 for (req_prod = queue->rx.req_prod_pvt; in xennet_alloc_rx_buffers()
295 req_prod - queue->rx.rsp_cons < NET_RX_RING_SIZE; in xennet_alloc_rx_buffers()
305 err = -ENOMEM; in xennet_alloc_rx_buffers()
311 BUG_ON(queue->rx_skbs[id]); in xennet_alloc_rx_buffers()
312 queue->rx_skbs[id] = skb; in xennet_alloc_rx_buffers()
314 ref = gnttab_claim_grant_reference(&queue->gref_rx_head); in xennet_alloc_rx_buffers()
316 queue->grant_rx_ref[id] = ref; in xennet_alloc_rx_buffers()
318 page = skb_frag_page(&skb_shinfo(skb)->frags[0]); in xennet_alloc_rx_buffers()
320 req = RING_GET_REQUEST(&queue->rx, req_prod); in xennet_alloc_rx_buffers()
322 queue->info->xbdev->otherend_id, in xennet_alloc_rx_buffers()
325 req->id = id; in xennet_alloc_rx_buffers()
326 req->gref = ref; in xennet_alloc_rx_buffers()
329 queue->rx.req_prod_pvt = req_prod; in xennet_alloc_rx_buffers()
336 if (req_prod - queue->rx.rsp_cons < NET_RX_SLOTS_MIN || in xennet_alloc_rx_buffers()
338 mod_timer(&queue->rx_refill_timer, jiffies + (HZ/10)); in xennet_alloc_rx_buffers()
344 RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&queue->rx, notify); in xennet_alloc_rx_buffers()
346 notify_remote_via_irq(queue->rx_irq); in xennet_alloc_rx_buffers()
352 unsigned int num_queues = dev->real_num_tx_queues; in xennet_open()
356 if (!np->queues) in xennet_open()
357 return -ENODEV; in xennet_open()
360 queue = &np->queues[i]; in xennet_open()
361 napi_enable(&queue->napi); in xennet_open()
363 spin_lock_bh(&queue->rx_lock); in xennet_open()
366 queue->rx.sring->rsp_event = queue->rx.rsp_cons + 1; in xennet_open()
367 if (RING_HAS_UNCONSUMED_RESPONSES(&queue->rx)) in xennet_open()
368 napi_schedule(&queue->napi); in xennet_open()
370 spin_unlock_bh(&queue->rx_lock); in xennet_open()
385 BUG_ON(!netif_carrier_ok(queue->info->netdev)); in xennet_tx_buf_gc()
388 prod = queue->tx.sring->rsp_prod; in xennet_tx_buf_gc()
389 rmb(); /* Ensure we see responses up to 'rp'. */ in xennet_tx_buf_gc()
391 for (cons = queue->tx.rsp_cons; cons != prod; cons++) { in xennet_tx_buf_gc()
394 txrsp = RING_GET_RESPONSE(&queue->tx, cons); in xennet_tx_buf_gc()
395 if (txrsp->status == XEN_NETIF_RSP_NULL) in xennet_tx_buf_gc()
398 id = txrsp->id; in xennet_tx_buf_gc()
399 skb = queue->tx_skbs[id].skb; in xennet_tx_buf_gc()
401 queue->grant_tx_ref[id]) != 0)) { in xennet_tx_buf_gc()
402 pr_alert("%s: warning -- grant still in use by backend domain\n", in xennet_tx_buf_gc()
407 queue->grant_tx_ref[id], GNTMAP_readonly); in xennet_tx_buf_gc()
409 &queue->gref_tx_head, queue->grant_tx_ref[id]); in xennet_tx_buf_gc()
410 queue->grant_tx_ref[id] = GRANT_INVALID_REF; in xennet_tx_buf_gc()
411 queue->grant_tx_page[id] = NULL; in xennet_tx_buf_gc()
412 add_id_to_freelist(&queue->tx_skb_freelist, queue->tx_skbs, id); in xennet_tx_buf_gc()
416 queue->tx.rsp_cons = prod; in xennet_tx_buf_gc()
418 RING_FINAL_CHECK_FOR_RESPONSES(&queue->tx, more_to_do); in xennet_tx_buf_gc()
440 struct page *page = info->page; in xennet_tx_setup_grant()
441 struct netfront_queue *queue = info->queue; in xennet_tx_setup_grant()
442 struct sk_buff *skb = info->skb; in xennet_tx_setup_grant()
444 id = get_id_from_freelist(&queue->tx_skb_freelist, queue->tx_skbs); in xennet_tx_setup_grant()
445 tx = RING_GET_REQUEST(&queue->tx, queue->tx.req_prod_pvt++); in xennet_tx_setup_grant()
446 ref = gnttab_claim_grant_reference(&queue->gref_tx_head); in xennet_tx_setup_grant()
449 gnttab_grant_foreign_access_ref(ref, queue->info->xbdev->otherend_id, in xennet_tx_setup_grant()
452 queue->tx_skbs[id].skb = skb; in xennet_tx_setup_grant()
453 queue->grant_tx_page[id] = page; in xennet_tx_setup_grant()
454 queue->grant_tx_ref[id] = ref; in xennet_tx_setup_grant()
456 tx->id = id; in xennet_tx_setup_grant()
457 tx->gref = ref; in xennet_tx_setup_grant()
458 tx->offset = offset; in xennet_tx_setup_grant()
459 tx->size = len; in xennet_tx_setup_grant()
460 tx->flags = 0; in xennet_tx_setup_grant()
462 info->tx = tx; in xennet_tx_setup_grant()
463 info->size += tx->size; in xennet_tx_setup_grant()
487 info->tx->flags |= XEN_NETTXF_more_data; in xennet_make_one_txreq()
488 skb_get(info->skb); in xennet_make_one_txreq()
517 len -= info.size; in xennet_make_txreqs()
524 * Count how many ring slots are required to send this skb. Each frag
529 int i, frags = skb_shinfo(skb)->nr_frags; in xennet_count_skb_slots()
532 slots = gnttab_count_grant(offset_in_page(skb->data), in xennet_count_skb_slots()
536 skb_frag_t *frag = skb_shinfo(skb)->frags + i; in xennet_count_skb_slots()
538 unsigned long offset = frag->page_offset; in xennet_count_skb_slots()
553 unsigned int num_queues = dev->real_num_tx_queues; in xennet_select_queue()
573 struct netfront_stats *tx_stats = this_cpu_ptr(np->tx_stats); in xennet_start_xmit()
583 unsigned int num_queues = dev->real_num_tx_queues; in xennet_start_xmit()
587 /* Drop the packet if no queues are set up */ in xennet_start_xmit()
590 /* Determine which queue to transmit this SKB on */ in xennet_start_xmit()
592 queue = &np->queues[queue_index]; in xennet_start_xmit()
594 /* If skb->len is too big for wire format, drop skb and alert in xennet_start_xmit()
597 if (unlikely(skb->len > XEN_NETIF_MAX_TX_SIZE)) { in xennet_start_xmit()
599 "xennet: skb->len = %u, too big for wire format\n", in xennet_start_xmit()
600 skb->len); in xennet_start_xmit()
607 slots, skb->len); in xennet_start_xmit()
612 page = virt_to_page(skb->data); in xennet_start_xmit()
613 offset = offset_in_page(skb->data); in xennet_start_xmit()
618 if (unlikely(PAGE_SIZE - offset < ETH_HLEN)) { in xennet_start_xmit()
624 page = virt_to_page(skb->data); in xennet_start_xmit()
625 offset = offset_in_page(skb->data); in xennet_start_xmit()
630 spin_lock_irqsave(&queue->tx_lock, flags); in xennet_start_xmit()
635 spin_unlock_irqrestore(&queue->tx_lock, flags); in xennet_start_xmit()
642 offset += tx->size; in xennet_start_xmit()
647 len -= tx->size; in xennet_start_xmit()
649 if (skb->ip_summed == CHECKSUM_PARTIAL) in xennet_start_xmit()
651 tx->flags |= XEN_NETTXF_csum_blank | XEN_NETTXF_data_validated; in xennet_start_xmit()
652 else if (skb->ip_summed == CHECKSUM_UNNECESSARY) in xennet_start_xmit()
654 tx->flags |= XEN_NETTXF_data_validated; in xennet_start_xmit()
657 if (skb_shinfo(skb)->gso_size) { in xennet_start_xmit()
661 RING_GET_REQUEST(&queue->tx, queue->tx.req_prod_pvt++); in xennet_start_xmit()
663 tx->flags |= XEN_NETTXF_extra_info; in xennet_start_xmit()
665 gso->u.gso.size = skb_shinfo(skb)->gso_size; in xennet_start_xmit()
666 gso->u.gso.type = (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6) ? in xennet_start_xmit()
669 gso->u.gso.pad = 0; in xennet_start_xmit()
670 gso->u.gso.features = 0; in xennet_start_xmit()
672 gso->type = XEN_NETIF_EXTRA_TYPE_GSO; in xennet_start_xmit()
673 gso->flags = 0; in xennet_start_xmit()
680 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { in xennet_start_xmit()
681 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; in xennet_start_xmit()
683 skb_frag_page(frag), frag->page_offset, in xennet_start_xmit()
688 first_tx->size = skb->len; in xennet_start_xmit()
690 RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&queue->tx, notify); in xennet_start_xmit()
692 notify_remote_via_irq(queue->tx_irq); in xennet_start_xmit()
694 u64_stats_update_begin(&tx_stats->syncp); in xennet_start_xmit()
695 tx_stats->bytes += skb->len; in xennet_start_xmit()
696 tx_stats->packets++; in xennet_start_xmit()
697 u64_stats_update_end(&tx_stats->syncp); in xennet_start_xmit()
699 /* Note: It is not safe to access skb after xennet_tx_buf_gc()! */ in xennet_start_xmit()
703 netif_tx_stop_queue(netdev_get_tx_queue(dev, queue->id)); in xennet_start_xmit()
705 spin_unlock_irqrestore(&queue->tx_lock, flags); in xennet_start_xmit()
710 dev->stats.tx_dropped++; in xennet_start_xmit()
718 unsigned int num_queues = dev->real_num_tx_queues; in xennet_close()
721 netif_tx_stop_all_queues(np->netdev); in xennet_close()
723 queue = &np->queues[i]; in xennet_close()
724 napi_disable(&queue->napi); in xennet_close()
732 int new = xennet_rxidx(queue->rx.req_prod_pvt); in xennet_move_rx_slot()
734 BUG_ON(queue->rx_skbs[new]); in xennet_move_rx_slot()
735 queue->rx_skbs[new] = skb; in xennet_move_rx_slot()
736 queue->grant_rx_ref[new] = ref; in xennet_move_rx_slot()
737 RING_GET_REQUEST(&queue->rx, queue->rx.req_prod_pvt)->id = new; in xennet_move_rx_slot()
738 RING_GET_REQUEST(&queue->rx, queue->rx.req_prod_pvt)->gref = ref; in xennet_move_rx_slot()
739 queue->rx.req_prod_pvt++; in xennet_move_rx_slot()
748 struct device *dev = &queue->info->netdev->dev; in xennet_get_extras()
749 RING_IDX cons = queue->rx.rsp_cons; in xennet_get_extras()
759 err = -EBADR; in xennet_get_extras()
764 RING_GET_RESPONSE(&queue->rx, ++cons); in xennet_get_extras()
766 if (unlikely(!extra->type || in xennet_get_extras()
767 extra->type >= XEN_NETIF_EXTRA_TYPE_MAX)) { in xennet_get_extras()
770 extra->type); in xennet_get_extras()
771 err = -EINVAL; in xennet_get_extras()
773 memcpy(&extras[extra->type - 1], extra, in xennet_get_extras()
780 } while (extra->flags & XEN_NETIF_EXTRA_FLAG_MORE); in xennet_get_extras()
782 queue->rx.rsp_cons = cons; in xennet_get_extras()
790 struct xen_netif_rx_response *rx = &rinfo->rx; in xennet_get_responses() local
791 struct xen_netif_extra_info *extras = rinfo->extras; in xennet_get_responses()
792 struct device *dev = &queue->info->netdev->dev; in xennet_get_responses()
793 RING_IDX cons = queue->rx.rsp_cons; in xennet_get_responses()
796 int max = XEN_NETIF_NR_SLOTS_MIN + (rx->status <= RX_COPY_THRESHOLD); in xennet_get_responses()
801 if (rx->flags & XEN_NETRXF_extra_info) { in xennet_get_responses()
803 cons = queue->rx.rsp_cons; in xennet_get_responses()
807 if (unlikely(rx->status < 0 || in xennet_get_responses()
808 rx->offset + rx->status > XEN_PAGE_SIZE)) { in xennet_get_responses()
810 dev_warn(dev, "rx->offset: %u, size: %d\n", in xennet_get_responses()
811 rx->offset, rx->status); in xennet_get_responses()
813 err = -EINVAL; in xennet_get_responses()
820 * situation to the system controller to reboot the backend. in xennet_get_responses()
824 dev_warn(dev, "Bad rx response id %d.\n", in xennet_get_responses()
825 rx->id); in xennet_get_responses()
826 err = -EINVAL; in xennet_get_responses()
833 gnttab_release_grant_reference(&queue->gref_rx_head, ref); in xennet_get_responses()
838 if (!(rx->flags & XEN_NETRXF_more_data)) in xennet_get_responses()
844 err = -ENOENT; in xennet_get_responses()
848 rx = RING_GET_RESPONSE(&queue->rx, cons + slots); in xennet_get_responses()
857 err = -E2BIG; in xennet_get_responses()
861 queue->rx.rsp_cons = cons + slots; in xennet_get_responses()
869 if (!gso->u.gso.size) { in xennet_set_skb_gso()
872 return -EINVAL; in xennet_set_skb_gso()
875 if (gso->u.gso.type != XEN_NETIF_GSO_TYPE_TCPV4 && in xennet_set_skb_gso()
876 gso->u.gso.type != XEN_NETIF_GSO_TYPE_TCPV6) { in xennet_set_skb_gso()
878 pr_warn("Bad GSO type %d\n", gso->u.gso.type); in xennet_set_skb_gso()
879 return -EINVAL; in xennet_set_skb_gso()
882 skb_shinfo(skb)->gso_size = gso->u.gso.size; in xennet_set_skb_gso()
883 skb_shinfo(skb)->gso_type = in xennet_set_skb_gso()
884 (gso->u.gso.type == XEN_NETIF_GSO_TYPE_TCPV4) ? in xennet_set_skb_gso()
889 skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY; in xennet_set_skb_gso()
890 skb_shinfo(skb)->gso_segs = 0; in xennet_set_skb_gso()
899 RING_IDX cons = queue->rx.rsp_cons; in xennet_fill_frags()
903 struct xen_netif_rx_response *rx = in xennet_fill_frags() local
904 RING_GET_RESPONSE(&queue->rx, ++cons); in xennet_fill_frags()
905 skb_frag_t *nfrag = &skb_shinfo(nskb)->frags[0]; in xennet_fill_frags()
907 if (skb_shinfo(skb)->nr_frags == MAX_SKB_FRAGS) { in xennet_fill_frags()
908 unsigned int pull_to = NETFRONT_SKB_CB(skb)->pull_to; in xennet_fill_frags()
911 __pskb_pull_tail(skb, pull_to - skb_headlen(skb)); in xennet_fill_frags()
913 if (unlikely(skb_shinfo(skb)->nr_frags >= MAX_SKB_FRAGS)) { in xennet_fill_frags()
914 queue->rx.rsp_cons = ++cons + skb_queue_len(list); in xennet_fill_frags()
916 return -ENOENT; in xennet_fill_frags()
919 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, in xennet_fill_frags()
921 rx->offset, rx->status, PAGE_SIZE); in xennet_fill_frags()
923 skb_shinfo(nskb)->nr_frags = 0; in xennet_fill_frags()
927 queue->rx.rsp_cons = cons; in xennet_fill_frags()
938 * peers can fail to set NETRXF_csum_blank when sending a GSO in checksum_setup()
939 * frame. In this case force the SKB to CHECKSUM_PARTIAL and in checksum_setup()
942 if (skb->ip_summed != CHECKSUM_PARTIAL && skb_is_gso(skb)) { in checksum_setup()
944 atomic_inc(&np->rx_gso_checksum_fixup); in checksum_setup()
945 skb->ip_summed = CHECKSUM_PARTIAL; in checksum_setup()
949 /* A non-CHECKSUM_PARTIAL SKB does not require setup. */ in checksum_setup()
950 if (skb->ip_summed != CHECKSUM_PARTIAL) in checksum_setup()
959 struct netfront_stats *rx_stats = this_cpu_ptr(queue->info->rx_stats); in handle_incoming_queue()
964 int pull_to = NETFRONT_SKB_CB(skb)->pull_to; in handle_incoming_queue()
967 __pskb_pull_tail(skb, pull_to - skb_headlen(skb)); in handle_incoming_queue()
969 /* Ethernet work: Delayed to here as it peeks the header. */ in handle_incoming_queue()
970 skb->protocol = eth_type_trans(skb, queue->info->netdev); in handle_incoming_queue()
973 if (checksum_setup(queue->info->netdev, skb)) { in handle_incoming_queue()
976 queue->info->netdev->stats.rx_errors++; in handle_incoming_queue()
980 u64_stats_update_begin(&rx_stats->syncp); in handle_incoming_queue()
981 rx_stats->packets++; in handle_incoming_queue()
982 rx_stats->bytes += skb->len; in handle_incoming_queue()
983 u64_stats_update_end(&rx_stats->syncp); in handle_incoming_queue()
986 napi_gro_receive(&queue->napi, skb); in handle_incoming_queue()
995 struct net_device *dev = queue->info->netdev; in xennet_poll()
998 struct xen_netif_rx_response *rx = &rinfo.rx; in xennet_poll() local
1007 spin_lock(&queue->rx_lock); in xennet_poll()
1013 rp = queue->rx.sring->rsp_prod; in xennet_poll()
1014 rmb(); /* Ensure we see queued responses up to 'rp'. */ in xennet_poll()
1016 i = queue->rx.rsp_cons; in xennet_poll()
1019 memcpy(rx, RING_GET_RESPONSE(&queue->rx, i), sizeof(*rx)); in xennet_poll()
1028 dev->stats.rx_errors++; in xennet_poll()
1029 i = queue->rx.rsp_cons; in xennet_poll()
1035 if (extras[XEN_NETIF_EXTRA_TYPE_GSO - 1].type) { in xennet_poll()
1037 gso = &extras[XEN_NETIF_EXTRA_TYPE_GSO - 1]; in xennet_poll()
1041 queue->rx.rsp_cons += skb_queue_len(&tmpq); in xennet_poll()
1046 NETFRONT_SKB_CB(skb)->pull_to = rx->status; in xennet_poll()
1047 if (NETFRONT_SKB_CB(skb)->pull_to > RX_COPY_THRESHOLD) in xennet_poll()
1048 NETFRONT_SKB_CB(skb)->pull_to = RX_COPY_THRESHOLD; in xennet_poll()
1050 skb_shinfo(skb)->frags[0].page_offset = rx->offset; in xennet_poll()
1051 skb_frag_size_set(&skb_shinfo(skb)->frags[0], rx->status); in xennet_poll()
1052 skb->data_len = rx->status; in xennet_poll()
1053 skb->len += rx->status; in xennet_poll()
1058 if (rx->flags & XEN_NETRXF_csum_blank) in xennet_poll()
1059 skb->ip_summed = CHECKSUM_PARTIAL; in xennet_poll()
1060 else if (rx->flags & XEN_NETRXF_data_validated) in xennet_poll()
1061 skb->ip_summed = CHECKSUM_UNNECESSARY; in xennet_poll()
1065 i = ++queue->rx.rsp_cons; in xennet_poll()
1071 work_done -= handle_incoming_queue(queue, &rxq); in xennet_poll()
1080 RING_FINAL_CHECK_FOR_RESPONSES(&queue->rx, more_to_do); in xennet_poll()
1085 spin_unlock(&queue->rx_lock); in xennet_poll()
1095 return -EINVAL; in xennet_change_mtu()
1096 dev->mtu = mtu; in xennet_change_mtu()
1107 struct netfront_stats *rx_stats = per_cpu_ptr(np->rx_stats, cpu); in xennet_get_stats64()
1108 struct netfront_stats *tx_stats = per_cpu_ptr(np->tx_stats, cpu); in xennet_get_stats64()
1113 start = u64_stats_fetch_begin_irq(&tx_stats->syncp); in xennet_get_stats64()
1114 tx_packets = tx_stats->packets; in xennet_get_stats64()
1115 tx_bytes = tx_stats->bytes; in xennet_get_stats64()
1116 } while (u64_stats_fetch_retry_irq(&tx_stats->syncp, start)); in xennet_get_stats64()
1119 start = u64_stats_fetch_begin_irq(&rx_stats->syncp); in xennet_get_stats64()
1120 rx_packets = rx_stats->packets; in xennet_get_stats64()
1121 rx_bytes = rx_stats->bytes; in xennet_get_stats64()
1122 } while (u64_stats_fetch_retry_irq(&rx_stats->syncp, start)); in xennet_get_stats64()
1124 tot->rx_packets += rx_packets; in xennet_get_stats64()
1125 tot->tx_packets += tx_packets; in xennet_get_stats64()
1126 tot->rx_bytes += rx_bytes; in xennet_get_stats64()
1127 tot->tx_bytes += tx_bytes; in xennet_get_stats64()
1130 tot->rx_errors = dev->stats.rx_errors; in xennet_get_stats64()
1131 tot->tx_dropped = dev->stats.tx_dropped; in xennet_get_stats64()
1141 if (skb_entry_is_link(&queue->tx_skbs[i])) in xennet_release_tx_bufs()
1144 skb = queue->tx_skbs[i].skb; in xennet_release_tx_bufs()
1145 get_page(queue->grant_tx_page[i]); in xennet_release_tx_bufs()
1146 gnttab_end_foreign_access(queue->grant_tx_ref[i], in xennet_release_tx_bufs()
1148 (unsigned long)page_address(queue->grant_tx_page[i])); in xennet_release_tx_bufs()
1149 queue->grant_tx_page[i] = NULL; in xennet_release_tx_bufs()
1150 queue->grant_tx_ref[i] = GRANT_INVALID_REF; in xennet_release_tx_bufs()
1151 add_id_to_freelist(&queue->tx_skb_freelist, queue->tx_skbs, i); in xennet_release_tx_bufs()
1160 spin_lock_bh(&queue->rx_lock); in xennet_release_rx_bufs()
1166 skb = queue->rx_skbs[id]; in xennet_release_rx_bufs()
1170 ref = queue->grant_rx_ref[id]; in xennet_release_rx_bufs()
1174 page = skb_frag_page(&skb_shinfo(skb)->frags[0]); in xennet_release_rx_bufs()
1182 queue->grant_rx_ref[id] = GRANT_INVALID_REF; in xennet_release_rx_bufs()
1187 spin_unlock_bh(&queue->rx_lock); in xennet_release_rx_bufs()
1196 !xenbus_read_unsigned(np->xbdev->otherend, "feature-sg", 0)) in xennet_fix_features()
1200 !xenbus_read_unsigned(np->xbdev->otherend, in xennet_fix_features()
1201 "feature-ipv6-csum-offload", 0)) in xennet_fix_features()
1205 !xenbus_read_unsigned(np->xbdev->otherend, "feature-gso-tcpv4", 0)) in xennet_fix_features()
1209 !xenbus_read_unsigned(np->xbdev->otherend, "feature-gso-tcpv6", 0)) in xennet_fix_features()
1218 if (!(features & NETIF_F_SG) && dev->mtu > ETH_DATA_LEN) { in xennet_set_features()
1220 dev->mtu = ETH_DATA_LEN; in xennet_set_features()
1231 spin_lock_irqsave(&queue->tx_lock, flags); in xennet_tx_interrupt()
1233 spin_unlock_irqrestore(&queue->tx_lock, flags); in xennet_tx_interrupt()
1241 struct net_device *dev = queue->info->netdev; in xennet_rx_interrupt()
1244 RING_HAS_UNCONSUMED_RESPONSES(&queue->rx))) in xennet_rx_interrupt()
1245 napi_schedule(&queue->napi); in xennet_rx_interrupt()
1262 unsigned int num_queues = dev->real_num_tx_queues; in xennet_poll_controller()
1265 xennet_interrupt(0, &info->queues[i]); in xennet_poll_controller()
1289 free_percpu(np->rx_stats); in xennet_free_netdev()
1290 free_percpu(np->tx_stats); in xennet_free_netdev()
1302 return ERR_PTR(-ENOMEM); in xennet_create_dev()
1305 np->xbdev = dev; in xennet_create_dev()
1307 np->queues = NULL; in xennet_create_dev()
1309 err = -ENOMEM; in xennet_create_dev()
1310 np->rx_stats = netdev_alloc_pcpu_stats(struct netfront_stats); in xennet_create_dev()
1311 if (np->rx_stats == NULL) in xennet_create_dev()
1313 np->tx_stats = netdev_alloc_pcpu_stats(struct netfront_stats); in xennet_create_dev()
1314 if (np->tx_stats == NULL) in xennet_create_dev()
1317 netdev->netdev_ops = &xennet_netdev_ops; in xennet_create_dev()
1319 netdev->features = NETIF_F_IP_CSUM | NETIF_F_RXCSUM | in xennet_create_dev()
1321 netdev->hw_features = NETIF_F_SG | in xennet_create_dev()
1327 * will be adjusted by the call to netdev_update_features() in in xennet_create_dev()
1331 netdev->features |= netdev->hw_features; in xennet_create_dev()
1333 netdev->ethtool_ops = &xennet_ethtool_ops; in xennet_create_dev()
1334 netdev->min_mtu = ETH_MIN_MTU; in xennet_create_dev()
1335 netdev->max_mtu = XEN_NETIF_MAX_TX_SIZE; in xennet_create_dev()
1336 SET_NETDEV_DEV(netdev, &dev->dev); in xennet_create_dev()
1338 np->netdev = netdev; in xennet_create_dev()
1345 xenbus_read_driver_state(dev->otherend) != in xennet_create_dev()
1347 xenbus_read_driver_state(dev->otherend) != in xennet_create_dev()
1359 * Entry point to this code when a new device is created. Allocate the basic
1378 dev_set_drvdata(&dev->dev, info); in netfront_probe()
1380 info->netdev->sysfs_groups[0] = &xennet_dev_group; in netfront_probe()
1388 /* This frees the page as a side-effect */ in xennet_end_access()
1396 unsigned int num_queues = info->netdev->real_num_tx_queues; in xennet_disconnect_backend()
1398 netif_carrier_off(info->netdev); in xennet_disconnect_backend()
1400 for (i = 0; i < num_queues && info->queues; ++i) { in xennet_disconnect_backend()
1401 struct netfront_queue *queue = &info->queues[i]; in xennet_disconnect_backend()
1403 del_timer_sync(&queue->rx_refill_timer); in xennet_disconnect_backend()
1405 if (queue->tx_irq && (queue->tx_irq == queue->rx_irq)) in xennet_disconnect_backend()
1406 unbind_from_irqhandler(queue->tx_irq, queue); in xennet_disconnect_backend()
1407 if (queue->tx_irq && (queue->tx_irq != queue->rx_irq)) { in xennet_disconnect_backend()
1408 unbind_from_irqhandler(queue->tx_irq, queue); in xennet_disconnect_backend()
1409 unbind_from_irqhandler(queue->rx_irq, queue); in xennet_disconnect_backend()
1411 queue->tx_evtchn = queue->rx_evtchn = 0; in xennet_disconnect_backend()
1412 queue->tx_irq = queue->rx_irq = 0; in xennet_disconnect_backend()
1414 if (netif_running(info->netdev)) in xennet_disconnect_backend()
1415 napi_synchronize(&queue->napi); in xennet_disconnect_backend()
1419 gnttab_free_grant_references(queue->gref_tx_head); in xennet_disconnect_backend()
1420 gnttab_free_grant_references(queue->gref_rx_head); in xennet_disconnect_backend()
1423 xennet_end_access(queue->tx_ring_ref, queue->tx.sring); in xennet_disconnect_backend()
1424 xennet_end_access(queue->rx_ring_ref, queue->rx.sring); in xennet_disconnect_backend()
1426 queue->tx_ring_ref = GRANT_INVALID_REF; in xennet_disconnect_backend()
1427 queue->rx_ring_ref = GRANT_INVALID_REF; in xennet_disconnect_backend()
1428 queue->tx.sring = NULL; in xennet_disconnect_backend()
1429 queue->rx.sring = NULL; in xennet_disconnect_backend()
1434 * We are reconnecting to the backend, due to a suspend/resume, or a backend
1436 * leave the device-layer structures intact so that this is transparent to the
1441 struct netfront_info *info = dev_get_drvdata(&dev->dev); in netfront_resume()
1443 dev_dbg(&dev->dev, "%s\n", dev->nodename); in netfront_resume()
1454 macstr = s = xenbus_read(XBT_NIL, dev->nodename, "mac", NULL); in xen_net_read_mac()
1460 if ((s == e) || (*e != ((i == ETH_ALEN-1) ? '\0' : ':'))) { in xen_net_read_mac()
1462 return -ENOENT; in xen_net_read_mac()
1475 err = xenbus_alloc_evtchn(queue->info->xbdev, &queue->tx_evtchn); in setup_netfront_single()
1479 err = bind_evtchn_to_irqhandler(queue->tx_evtchn, in setup_netfront_single()
1481 0, queue->info->netdev->name, queue); in setup_netfront_single()
1484 queue->rx_evtchn = queue->tx_evtchn; in setup_netfront_single()
1485 queue->rx_irq = queue->tx_irq = err; in setup_netfront_single()
1490 xenbus_free_evtchn(queue->info->xbdev, queue->tx_evtchn); in setup_netfront_single()
1491 queue->tx_evtchn = 0; in setup_netfront_single()
1500 err = xenbus_alloc_evtchn(queue->info->xbdev, &queue->tx_evtchn); in setup_netfront_split()
1503 err = xenbus_alloc_evtchn(queue->info->xbdev, &queue->rx_evtchn); in setup_netfront_split()
1507 snprintf(queue->tx_irq_name, sizeof(queue->tx_irq_name), in setup_netfront_split()
1508 "%s-tx", queue->name); in setup_netfront_split()
1509 err = bind_evtchn_to_irqhandler(queue->tx_evtchn, in setup_netfront_split()
1511 0, queue->tx_irq_name, queue); in setup_netfront_split()
1514 queue->tx_irq = err; in setup_netfront_split()
1516 snprintf(queue->rx_irq_name, sizeof(queue->rx_irq_name), in setup_netfront_split()
1517 "%s-rx", queue->name); in setup_netfront_split()
1518 err = bind_evtchn_to_irqhandler(queue->rx_evtchn, in setup_netfront_split()
1520 0, queue->rx_irq_name, queue); in setup_netfront_split()
1523 queue->rx_irq = err; in setup_netfront_split()
1528 unbind_from_irqhandler(queue->tx_irq, queue); in setup_netfront_split()
1529 queue->tx_irq = 0; in setup_netfront_split()
1531 xenbus_free_evtchn(queue->info->xbdev, queue->rx_evtchn); in setup_netfront_split()
1532 queue->rx_evtchn = 0; in setup_netfront_split()
1534 xenbus_free_evtchn(queue->info->xbdev, queue->tx_evtchn); in setup_netfront_split()
1535 queue->tx_evtchn = 0; in setup_netfront_split()
1548 queue->tx_ring_ref = GRANT_INVALID_REF; in setup_netfront()
1549 queue->rx_ring_ref = GRANT_INVALID_REF; in setup_netfront()
1550 queue->rx.sring = NULL; in setup_netfront()
1551 queue->tx.sring = NULL; in setup_netfront()
1555 err = -ENOMEM; in setup_netfront()
1560 FRONT_RING_INIT(&queue->tx, txs, XEN_PAGE_SIZE); in setup_netfront()
1565 queue->tx_ring_ref = gref; in setup_netfront()
1569 err = -ENOMEM; in setup_netfront()
1570 xenbus_dev_fatal(dev, err, "allocating rx ring page"); in setup_netfront()
1574 FRONT_RING_INIT(&queue->rx, rxs, XEN_PAGE_SIZE); in setup_netfront()
1579 queue->rx_ring_ref = gref; in setup_netfront()
1584 * a) feature-split-event-channels == 0 in setup_netfront()
1585 * b) feature-split-event-channels == 1 but failed to setup in setup_netfront()
1595 /* If we fail to setup netfront, it is safe to just revoke access to in setup_netfront()
1599 gnttab_end_foreign_access_ref(queue->rx_ring_ref, 0); in setup_netfront()
1603 gnttab_end_foreign_access_ref(queue->tx_ring_ref, 0); in setup_netfront()
1610 /* Queue-specific initialisation
1611 * This used to be done in xennet_create_dev() but must now
1612 * be run per-queue.
1620 spin_lock_init(&queue->tx_lock); in xennet_init_queue()
1621 spin_lock_init(&queue->rx_lock); in xennet_init_queue()
1623 timer_setup(&queue->rx_refill_timer, rx_refill_timeout, 0); in xennet_init_queue()
1625 devid = strrchr(queue->info->xbdev->nodename, '/') + 1; in xennet_init_queue()
1626 snprintf(queue->name, sizeof(queue->name), "vif%s-q%u", in xennet_init_queue()
1627 devid, queue->id); in xennet_init_queue()
1630 queue->tx_skb_freelist = 0; in xennet_init_queue()
1632 skb_entry_set_link(&queue->tx_skbs[i], i+1); in xennet_init_queue()
1633 queue->grant_tx_ref[i] = GRANT_INVALID_REF; in xennet_init_queue()
1634 queue->grant_tx_page[i] = NULL; in xennet_init_queue()
1639 queue->rx_skbs[i] = NULL; in xennet_init_queue()
1640 queue->grant_rx_ref[i] = GRANT_INVALID_REF; in xennet_init_queue()
1645 &queue->gref_tx_head) < 0) { in xennet_init_queue()
1647 err = -ENOMEM; in xennet_init_queue()
1651 /* A grant for every rx ring slot */ in xennet_init_queue()
1653 &queue->gref_rx_head) < 0) { in xennet_init_queue()
1654 pr_alert("can't alloc rx grant refs\n"); in xennet_init_queue()
1655 err = -ENOMEM; in xennet_init_queue()
1662 gnttab_free_grant_references(queue->gref_tx_head); in xennet_init_queue()
1670 /* Write the queue-specific keys into XenStore in the traditional in write_queue_xenstore_keys()
1672 * queues. in write_queue_xenstore_keys()
1674 struct xenbus_device *dev = queue->info->xbdev; in write_queue_xenstore_keys()
1680 /* Choose the correct place to write the keys */ in write_queue_xenstore_keys()
1682 pathsize = strlen(dev->nodename) + 10; in write_queue_xenstore_keys()
1685 err = -ENOMEM; in write_queue_xenstore_keys()
1689 snprintf(path, pathsize, "%s/queue-%u", in write_queue_xenstore_keys()
1690 dev->nodename, queue->id); in write_queue_xenstore_keys()
1692 path = (char *)dev->nodename; in write_queue_xenstore_keys()
1696 err = xenbus_printf(*xbt, path, "tx-ring-ref", "%u", in write_queue_xenstore_keys()
1697 queue->tx_ring_ref); in write_queue_xenstore_keys()
1699 message = "writing tx-ring-ref"; in write_queue_xenstore_keys()
1703 err = xenbus_printf(*xbt, path, "rx-ring-ref", "%u", in write_queue_xenstore_keys()
1704 queue->rx_ring_ref); in write_queue_xenstore_keys()
1706 message = "writing rx-ring-ref"; in write_queue_xenstore_keys()
1713 if (queue->tx_evtchn == queue->rx_evtchn) { in write_queue_xenstore_keys()
1716 "event-channel", "%u", queue->tx_evtchn); in write_queue_xenstore_keys()
1718 message = "writing event-channel"; in write_queue_xenstore_keys()
1724 "event-channel-tx", "%u", queue->tx_evtchn); in write_queue_xenstore_keys()
1726 message = "writing event-channel-tx"; in write_queue_xenstore_keys()
1731 "event-channel-rx", "%u", queue->rx_evtchn); in write_queue_xenstore_keys()
1733 message = "writing event-channel-rx"; in write_queue_xenstore_keys()
1753 for (i = 0; i < info->netdev->real_num_tx_queues; i++) { in xennet_destroy_queues()
1754 struct netfront_queue *queue = &info->queues[i]; in xennet_destroy_queues()
1756 if (netif_running(info->netdev)) in xennet_destroy_queues()
1757 napi_disable(&queue->napi); in xennet_destroy_queues()
1758 netif_napi_del(&queue->napi); in xennet_destroy_queues()
1761 kfree(info->queues); in xennet_destroy_queues()
1762 info->queues = NULL; in xennet_destroy_queues()
1771 info->queues = kcalloc(*num_queues, sizeof(struct netfront_queue), in xennet_create_queues()
1773 if (!info->queues) in xennet_create_queues()
1774 return -ENOMEM; in xennet_create_queues()
1777 struct netfront_queue *queue = &info->queues[i]; in xennet_create_queues()
1779 queue->id = i; in xennet_create_queues()
1780 queue->info = info; in xennet_create_queues()
1784 dev_warn(&info->xbdev->dev, in xennet_create_queues()
1785 "only created %d queues\n", i); in xennet_create_queues()
1790 netif_napi_add(queue->info->netdev, &queue->napi, in xennet_create_queues()
1792 if (netif_running(info->netdev)) in xennet_create_queues()
1793 napi_enable(&queue->napi); in xennet_create_queues()
1796 netif_set_real_num_tx_queues(info->netdev, *num_queues); in xennet_create_queues()
1799 dev_err(&info->xbdev->dev, "no queues\n"); in xennet_create_queues()
1800 return -EINVAL; in xennet_create_queues()
1818 info->netdev->irq = 0; in talk_to_netback()
1820 /* Check if backend supports multiple queues */ in talk_to_netback()
1821 max_queues = xenbus_read_unsigned(info->xbdev->otherend, in talk_to_netback()
1822 "multi-queue-max-queues", 1); in talk_to_netback()
1825 /* Check feature-split-event-channels */ in talk_to_netback()
1826 feature_split_evtchn = xenbus_read_unsigned(info->xbdev->otherend, in talk_to_netback()
1827 "feature-split-event-channels", 0); in talk_to_netback()
1830 err = xen_net_read_mac(dev, info->netdev->dev_addr); in talk_to_netback()
1832 xenbus_dev_fatal(dev, err, "parsing %s/mac", dev->nodename); in talk_to_netback()
1837 if (info->queues) in talk_to_netback()
1842 xenbus_dev_fatal(dev, err, "creating queues"); in talk_to_netback()
1843 kfree(info->queues); in talk_to_netback()
1844 info->queues = NULL; in talk_to_netback()
1849 /* Create shared ring, alloc event channel -- for each queue */ in talk_to_netback()
1851 queue = &info->queues[i]; in talk_to_netback()
1865 info->xbdev->otherend, "multi-queue-max-queues")) { in talk_to_netback()
1866 /* Write the number of queues */ in talk_to_netback()
1867 err = xenbus_printf(xbt, dev->nodename, in talk_to_netback()
1868 "multi-queue-num-queues", "%u", num_queues); in talk_to_netback()
1870 message = "writing multi-queue-num-queues"; in talk_to_netback()
1876 err = write_queue_xenstore_keys(&info->queues[0], &xbt, 0); /* flat */ in talk_to_netback()
1882 queue = &info->queues[i]; in talk_to_netback()
1889 /* The remaining keys are not queue-specific */ in talk_to_netback()
1890 err = xenbus_printf(xbt, dev->nodename, "request-rx-copy", "%u", in talk_to_netback()
1893 message = "writing request-rx-copy"; in talk_to_netback()
1897 err = xenbus_printf(xbt, dev->nodename, "feature-rx-notify", "%d", 1); in talk_to_netback()
1899 message = "writing feature-rx-notify"; in talk_to_netback()
1903 err = xenbus_printf(xbt, dev->nodename, "feature-sg", "%d", 1); in talk_to_netback()
1905 message = "writing feature-sg"; in talk_to_netback()
1909 err = xenbus_printf(xbt, dev->nodename, "feature-gso-tcpv4", "%d", 1); in talk_to_netback()
1911 message = "writing feature-gso-tcpv4"; in talk_to_netback()
1915 err = xenbus_write(xbt, dev->nodename, "feature-gso-tcpv6", "1"); in talk_to_netback()
1917 message = "writing feature-gso-tcpv6"; in talk_to_netback()
1921 err = xenbus_write(xbt, dev->nodename, "feature-ipv6-csum-offload", in talk_to_netback()
1924 message = "writing feature-ipv6-csum-offload"; in talk_to_netback()
1930 if (err == -EAGAIN) in talk_to_netback()
1949 device_unregister(&dev->dev); in talk_to_netback()
1961 if (!xenbus_read_unsigned(np->xbdev->otherend, "feature-rx-copy", 0)) { in xennet_connect()
1962 dev_info(&dev->dev, in xennet_connect()
1964 return -ENODEV; in xennet_connect()
1967 err = talk_to_netback(np->xbdev, np); in xennet_connect()
1971 /* talk_to_netback() sets the correct number of queues */ in xennet_connect()
1972 num_queues = dev->real_num_tx_queues; in xennet_connect()
1974 if (dev->reg_state == NETREG_UNINITIALIZED) { in xennet_connect()
1978 device_unregister(&np->xbdev->dev); in xennet_connect()
1989 * ready to start sending and receiving packets and give the driver in xennet_connect()
1993 netif_carrier_on(np->netdev); in xennet_connect()
1995 queue = &np->queues[j]; in xennet_connect()
1997 notify_remote_via_irq(queue->tx_irq); in xennet_connect()
1998 if (queue->tx_irq != queue->rx_irq) in xennet_connect()
1999 notify_remote_via_irq(queue->rx_irq); in xennet_connect()
2001 spin_lock_irq(&queue->tx_lock); in xennet_connect()
2003 spin_unlock_irq(&queue->tx_lock); in xennet_connect()
2005 spin_lock_bh(&queue->rx_lock); in xennet_connect()
2007 spin_unlock_bh(&queue->rx_lock); in xennet_connect()
2019 struct netfront_info *np = dev_get_drvdata(&dev->dev); in netback_changed()
2020 struct net_device *netdev = np->netdev; in netback_changed()
2022 dev_dbg(&dev->dev, "%s\n", xenbus_strstate(backend_state)); in netback_changed()
2035 if (dev->state != XenbusStateInitialising) in netback_changed()
2047 if (dev->state == XenbusStateClosed) in netback_changed()
2049 /* Missed the backend's CLOSING state -- fallthrough */ in netback_changed()
2072 return -EINVAL; in xennet_get_sset_count()
2123 return -EPERM; in store_rxbuf()
2127 return -EBADMSG; in store_rxbuf()
2154 if (xenbus_read_driver_state(dev->otherend) == XenbusStateClosed) in xennet_bus_close()
2159 xenbus_read_driver_state(dev->otherend) == in xennet_bus_close()
2161 xenbus_read_driver_state(dev->otherend) == in xennet_bus_close()
2163 xenbus_read_driver_state(dev->otherend) == in xennet_bus_close()
2168 if (xenbus_read_driver_state(dev->otherend) == XenbusStateClosed) in xennet_bus_close()
2174 xenbus_read_driver_state(dev->otherend) == in xennet_bus_close()
2176 xenbus_read_driver_state(dev->otherend) == in xennet_bus_close()
2184 struct netfront_info *info = dev_get_drvdata(&dev->dev); in xennet_remove()
2189 if (info->netdev->reg_state == NETREG_REGISTERED) in xennet_remove()
2190 unregister_netdev(info->netdev); in xennet_remove()
2192 if (info->queues) { in xennet_remove()
2197 xennet_free_netdev(info->netdev); in xennet_remove()
2218 return -ENODEV; in netif_init()
2221 return -ENODEV; in netif_init()
2225 /* Allow as many queues as there are CPUs inut max. 8 if user has not in netif_init()