• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Copyright (c) 2009, Microsoft Corporation.
3  *
4  * This program is free software; you can redistribute it and/or modify it
5  * under the terms and conditions of the GNU General Public License,
6  * version 2, as published by the Free Software Foundation.
7  *
8  * This program is distributed in the hope it will be useful, but WITHOUT
9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
11  * more details.
12  *
13  * You should have received a copy of the GNU General Public License along with
14  * this program; if not, see <http://www.gnu.org/licenses/>.
15  *
16  * Authors:
17  *   Haiyang Zhang <haiyangz@microsoft.com>
18  *   Hank Janssen  <hjanssen@microsoft.com>
19  */
20 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
21 
22 #include <linux/init.h>
23 #include <linux/atomic.h>
24 #include <linux/module.h>
25 #include <linux/highmem.h>
26 #include <linux/device.h>
27 #include <linux/io.h>
28 #include <linux/delay.h>
29 #include <linux/netdevice.h>
30 #include <linux/inetdevice.h>
31 #include <linux/etherdevice.h>
32 #include <linux/pci.h>
33 #include <linux/skbuff.h>
34 #include <linux/if_vlan.h>
35 #include <linux/in.h>
36 #include <linux/slab.h>
37 #include <linux/rtnetlink.h>
38 #include <linux/netpoll.h>
39 
40 #include <net/arp.h>
41 #include <net/route.h>
42 #include <net/sock.h>
43 #include <net/pkt_sched.h>
44 #include <net/checksum.h>
45 #include <net/ip6_checksum.h>
46 
47 #include "hyperv_net.h"
48 
49 #define RING_SIZE_MIN	64
50 #define RETRY_US_LO	5000
51 #define RETRY_US_HI	10000
52 #define RETRY_MAX	2000	/* >10 sec */
53 
54 #define LINKCHANGE_INT (2 * HZ)
55 #define VF_TAKEOVER_INT (HZ / 10)
56 
57 static int ring_size = 128;
58 module_param(ring_size, int, S_IRUGO);
59 MODULE_PARM_DESC(ring_size, "Ring buffer size (# of pages)");
60 
61 static const u32 default_msg = NETIF_MSG_DRV | NETIF_MSG_PROBE |
62 				NETIF_MSG_LINK | NETIF_MSG_IFUP |
63 				NETIF_MSG_IFDOWN | NETIF_MSG_RX_ERR |
64 				NETIF_MSG_TX_ERR;
65 
66 static int debug = -1;
67 module_param(debug, int, S_IRUGO);
68 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
69 
70 static LIST_HEAD(netvsc_dev_list);
71 
netvsc_change_rx_flags(struct net_device * net,int change)72 static void netvsc_change_rx_flags(struct net_device *net, int change)
73 {
74 	struct net_device_context *ndev_ctx = netdev_priv(net);
75 	struct net_device *vf_netdev = rtnl_dereference(ndev_ctx->vf_netdev);
76 	int inc;
77 
78 	if (!vf_netdev)
79 		return;
80 
81 	if (change & IFF_PROMISC) {
82 		inc = (net->flags & IFF_PROMISC) ? 1 : -1;
83 		dev_set_promiscuity(vf_netdev, inc);
84 	}
85 
86 	if (change & IFF_ALLMULTI) {
87 		inc = (net->flags & IFF_ALLMULTI) ? 1 : -1;
88 		dev_set_allmulti(vf_netdev, inc);
89 	}
90 }
91 
netvsc_set_rx_mode(struct net_device * net)92 static void netvsc_set_rx_mode(struct net_device *net)
93 {
94 	struct net_device_context *ndev_ctx = netdev_priv(net);
95 	struct net_device *vf_netdev;
96 	struct netvsc_device *nvdev;
97 
98 	rcu_read_lock();
99 	vf_netdev = rcu_dereference(ndev_ctx->vf_netdev);
100 	if (vf_netdev) {
101 		dev_uc_sync(vf_netdev, net);
102 		dev_mc_sync(vf_netdev, net);
103 	}
104 
105 	nvdev = rcu_dereference(ndev_ctx->nvdev);
106 	if (nvdev)
107 		rndis_filter_update(nvdev);
108 	rcu_read_unlock();
109 }
110 
netvsc_tx_enable(struct netvsc_device * nvscdev,struct net_device * ndev)111 static void netvsc_tx_enable(struct netvsc_device *nvscdev,
112 			     struct net_device *ndev)
113 {
114 	nvscdev->tx_disable = false;
115 	virt_wmb(); /* ensure queue wake up mechanism is on */
116 
117 	netif_tx_wake_all_queues(ndev);
118 }
119 
netvsc_open(struct net_device * net)120 static int netvsc_open(struct net_device *net)
121 {
122 	struct net_device_context *ndev_ctx = netdev_priv(net);
123 	struct net_device *vf_netdev = rtnl_dereference(ndev_ctx->vf_netdev);
124 	struct netvsc_device *nvdev = rtnl_dereference(ndev_ctx->nvdev);
125 	struct rndis_device *rdev;
126 	int ret = 0;
127 
128 	netif_carrier_off(net);
129 
130 	/* Open up the device */
131 	ret = rndis_filter_open(nvdev);
132 	if (ret != 0) {
133 		netdev_err(net, "unable to open device (ret %d).\n", ret);
134 		return ret;
135 	}
136 
137 	rdev = nvdev->extension;
138 	if (!rdev->link_state) {
139 		netif_carrier_on(net);
140 		netvsc_tx_enable(nvdev, net);
141 	}
142 
143 	if (vf_netdev) {
144 		/* Setting synthetic device up transparently sets
145 		 * slave as up. If open fails, then slave will be
146 		 * still be offline (and not used).
147 		 */
148 		ret = dev_open(vf_netdev);
149 		if (ret)
150 			netdev_warn(net,
151 				    "unable to open slave: %s: %d\n",
152 				    vf_netdev->name, ret);
153 	}
154 	return 0;
155 }
156 
netvsc_wait_until_empty(struct netvsc_device * nvdev)157 static int netvsc_wait_until_empty(struct netvsc_device *nvdev)
158 {
159 	unsigned int retry = 0;
160 	int i;
161 
162 	/* Ensure pending bytes in ring are read */
163 	for (;;) {
164 		u32 aread = 0;
165 
166 		for (i = 0; i < nvdev->num_chn; i++) {
167 			struct vmbus_channel *chn
168 				= nvdev->chan_table[i].channel;
169 
170 			if (!chn)
171 				continue;
172 
173 			/* make sure receive not running now */
174 			napi_synchronize(&nvdev->chan_table[i].napi);
175 
176 			aread = hv_get_bytes_to_read(&chn->inbound);
177 			if (aread)
178 				break;
179 
180 			aread = hv_get_bytes_to_read(&chn->outbound);
181 			if (aread)
182 				break;
183 		}
184 
185 		if (aread == 0)
186 			return 0;
187 
188 		if (++retry > RETRY_MAX)
189 			return -ETIMEDOUT;
190 
191 		usleep_range(RETRY_US_LO, RETRY_US_HI);
192 	}
193 }
194 
netvsc_tx_disable(struct netvsc_device * nvscdev,struct net_device * ndev)195 static void netvsc_tx_disable(struct netvsc_device *nvscdev,
196 			      struct net_device *ndev)
197 {
198 	if (nvscdev) {
199 		nvscdev->tx_disable = true;
200 		virt_wmb(); /* ensure txq will not wake up after stop */
201 	}
202 
203 	netif_tx_disable(ndev);
204 }
205 
netvsc_close(struct net_device * net)206 static int netvsc_close(struct net_device *net)
207 {
208 	struct net_device_context *net_device_ctx = netdev_priv(net);
209 	struct net_device *vf_netdev
210 		= rtnl_dereference(net_device_ctx->vf_netdev);
211 	struct netvsc_device *nvdev = rtnl_dereference(net_device_ctx->nvdev);
212 	int ret;
213 
214 	netvsc_tx_disable(nvdev, net);
215 
216 	/* No need to close rndis filter if it is removed already */
217 	if (!nvdev)
218 		return 0;
219 
220 	ret = rndis_filter_close(nvdev);
221 	if (ret != 0) {
222 		netdev_err(net, "unable to close device (ret %d).\n", ret);
223 		return ret;
224 	}
225 
226 	ret = netvsc_wait_until_empty(nvdev);
227 	if (ret)
228 		netdev_err(net, "Ring buffer not empty after closing rndis\n");
229 
230 	if (vf_netdev)
231 		dev_close(vf_netdev);
232 
233 	return ret;
234 }
235 
init_ppi_data(struct rndis_message * msg,u32 ppi_size,int pkt_type)236 static void *init_ppi_data(struct rndis_message *msg, u32 ppi_size,
237 			   int pkt_type)
238 {
239 	struct rndis_packet *rndis_pkt;
240 	struct rndis_per_packet_info *ppi;
241 
242 	rndis_pkt = &msg->msg.pkt;
243 	rndis_pkt->data_offset += ppi_size;
244 
245 	ppi = (struct rndis_per_packet_info *)((void *)rndis_pkt +
246 		rndis_pkt->per_pkt_info_offset + rndis_pkt->per_pkt_info_len);
247 
248 	ppi->size = ppi_size;
249 	ppi->type = pkt_type;
250 	ppi->ppi_offset = sizeof(struct rndis_per_packet_info);
251 
252 	rndis_pkt->per_pkt_info_len += ppi_size;
253 
254 	return ppi;
255 }
256 
257 /* Azure hosts don't support non-TCP port numbers in hashing for fragmented
258  * packets. We can use ethtool to change UDP hash level when necessary.
259  */
netvsc_get_hash(struct sk_buff * skb,const struct net_device_context * ndc)260 static inline u32 netvsc_get_hash(
261 	struct sk_buff *skb,
262 	const struct net_device_context *ndc)
263 {
264 	struct flow_keys flow;
265 	u32 hash;
266 	static u32 hashrnd __read_mostly;
267 
268 	net_get_random_once(&hashrnd, sizeof(hashrnd));
269 
270 	if (!skb_flow_dissect_flow_keys(skb, &flow, 0))
271 		return 0;
272 
273 	if (flow.basic.ip_proto == IPPROTO_TCP ||
274 	    (flow.basic.ip_proto == IPPROTO_UDP &&
275 	     ((flow.basic.n_proto == htons(ETH_P_IP) && ndc->udp4_l4_hash) ||
276 	      (flow.basic.n_proto == htons(ETH_P_IPV6) &&
277 	       ndc->udp6_l4_hash)))) {
278 		return skb_get_hash(skb);
279 	} else {
280 		if (flow.basic.n_proto == htons(ETH_P_IP))
281 			hash = jhash2((u32 *)&flow.addrs.v4addrs, 2, hashrnd);
282 		else if (flow.basic.n_proto == htons(ETH_P_IPV6))
283 			hash = jhash2((u32 *)&flow.addrs.v6addrs, 8, hashrnd);
284 		else
285 			return 0;
286 
287 		__skb_set_sw_hash(skb, hash, false);
288 	}
289 
290 	return hash;
291 }
292 
netvsc_get_tx_queue(struct net_device * ndev,struct sk_buff * skb,int old_idx)293 static inline int netvsc_get_tx_queue(struct net_device *ndev,
294 				      struct sk_buff *skb, int old_idx)
295 {
296 	const struct net_device_context *ndc = netdev_priv(ndev);
297 	struct sock *sk = skb->sk;
298 	int q_idx;
299 
300 	q_idx = ndc->tx_table[netvsc_get_hash(skb, ndc) &
301 			      (VRSS_SEND_TAB_SIZE - 1)];
302 
303 	/* If queue index changed record the new value */
304 	if (q_idx != old_idx &&
305 	    sk && sk_fullsock(sk) && rcu_access_pointer(sk->sk_dst_cache))
306 		sk_tx_queue_set(sk, q_idx);
307 
308 	return q_idx;
309 }
310 
311 /*
312  * Select queue for transmit.
313  *
314  * If a valid queue has already been assigned, then use that.
315  * Otherwise compute tx queue based on hash and the send table.
316  *
317  * This is basically similar to default (__netdev_pick_tx) with the added step
318  * of using the host send_table when no other queue has been assigned.
319  *
320  * TODO support XPS - but get_xps_queue not exported
321  */
netvsc_pick_tx(struct net_device * ndev,struct sk_buff * skb)322 static u16 netvsc_pick_tx(struct net_device *ndev, struct sk_buff *skb)
323 {
324 	int q_idx = sk_tx_queue_get(skb->sk);
325 
326 	if (q_idx < 0 || skb->ooo_okay || q_idx >= ndev->real_num_tx_queues) {
327 		/* If forwarding a packet, we use the recorded queue when
328 		 * available for better cache locality.
329 		 */
330 		if (skb_rx_queue_recorded(skb))
331 			q_idx = skb_get_rx_queue(skb);
332 		else
333 			q_idx = netvsc_get_tx_queue(ndev, skb, q_idx);
334 	}
335 
336 	return q_idx;
337 }
338 
netvsc_select_queue(struct net_device * ndev,struct sk_buff * skb,void * accel_priv,select_queue_fallback_t fallback)339 static u16 netvsc_select_queue(struct net_device *ndev, struct sk_buff *skb,
340 			       void *accel_priv,
341 			       select_queue_fallback_t fallback)
342 {
343 	struct net_device_context *ndc = netdev_priv(ndev);
344 	struct net_device *vf_netdev;
345 	u16 txq;
346 
347 	rcu_read_lock();
348 	vf_netdev = rcu_dereference(ndc->vf_netdev);
349 	if (vf_netdev) {
350 		const struct net_device_ops *vf_ops = vf_netdev->netdev_ops;
351 
352 		if (vf_ops->ndo_select_queue)
353 			txq = vf_ops->ndo_select_queue(vf_netdev, skb,
354 						       accel_priv, fallback);
355 		else
356 			txq = fallback(vf_netdev, skb);
357 
358 		/* Record the queue selected by VF so that it can be
359 		 * used for common case where VF has more queues than
360 		 * the synthetic device.
361 		 */
362 		qdisc_skb_cb(skb)->slave_dev_queue_mapping = txq;
363 	} else {
364 		txq = netvsc_pick_tx(ndev, skb);
365 	}
366 	rcu_read_unlock();
367 
368 	while (unlikely(txq >= ndev->real_num_tx_queues))
369 		txq -= ndev->real_num_tx_queues;
370 
371 	return txq;
372 }
373 
fill_pg_buf(struct page * page,u32 offset,u32 len,struct hv_page_buffer * pb)374 static u32 fill_pg_buf(struct page *page, u32 offset, u32 len,
375 		       struct hv_page_buffer *pb)
376 {
377 	int j = 0;
378 
379 	/* Deal with compund pages by ignoring unused part
380 	 * of the page.
381 	 */
382 	page += (offset >> PAGE_SHIFT);
383 	offset &= ~PAGE_MASK;
384 
385 	while (len > 0) {
386 		unsigned long bytes;
387 
388 		bytes = PAGE_SIZE - offset;
389 		if (bytes > len)
390 			bytes = len;
391 		pb[j].pfn = page_to_pfn(page);
392 		pb[j].offset = offset;
393 		pb[j].len = bytes;
394 
395 		offset += bytes;
396 		len -= bytes;
397 
398 		if (offset == PAGE_SIZE && len) {
399 			page++;
400 			offset = 0;
401 			j++;
402 		}
403 	}
404 
405 	return j + 1;
406 }
407 
init_page_array(void * hdr,u32 len,struct sk_buff * skb,struct hv_netvsc_packet * packet,struct hv_page_buffer * pb)408 static u32 init_page_array(void *hdr, u32 len, struct sk_buff *skb,
409 			   struct hv_netvsc_packet *packet,
410 			   struct hv_page_buffer *pb)
411 {
412 	u32 slots_used = 0;
413 	char *data = skb->data;
414 	int frags = skb_shinfo(skb)->nr_frags;
415 	int i;
416 
417 	/* The packet is laid out thus:
418 	 * 1. hdr: RNDIS header and PPI
419 	 * 2. skb linear data
420 	 * 3. skb fragment data
421 	 */
422 	slots_used += fill_pg_buf(virt_to_page(hdr),
423 				  offset_in_page(hdr),
424 				  len, &pb[slots_used]);
425 
426 	packet->rmsg_size = len;
427 	packet->rmsg_pgcnt = slots_used;
428 
429 	slots_used += fill_pg_buf(virt_to_page(data),
430 				offset_in_page(data),
431 				skb_headlen(skb), &pb[slots_used]);
432 
433 	for (i = 0; i < frags; i++) {
434 		skb_frag_t *frag = skb_shinfo(skb)->frags + i;
435 
436 		slots_used += fill_pg_buf(skb_frag_page(frag),
437 					frag->page_offset,
438 					skb_frag_size(frag), &pb[slots_used]);
439 	}
440 	return slots_used;
441 }
442 
count_skb_frag_slots(struct sk_buff * skb)443 static int count_skb_frag_slots(struct sk_buff *skb)
444 {
445 	int i, frags = skb_shinfo(skb)->nr_frags;
446 	int pages = 0;
447 
448 	for (i = 0; i < frags; i++) {
449 		skb_frag_t *frag = skb_shinfo(skb)->frags + i;
450 		unsigned long size = skb_frag_size(frag);
451 		unsigned long offset = frag->page_offset;
452 
453 		/* Skip unused frames from start of page */
454 		offset &= ~PAGE_MASK;
455 		pages += PFN_UP(offset + size);
456 	}
457 	return pages;
458 }
459 
netvsc_get_slots(struct sk_buff * skb)460 static int netvsc_get_slots(struct sk_buff *skb)
461 {
462 	char *data = skb->data;
463 	unsigned int offset = offset_in_page(data);
464 	unsigned int len = skb_headlen(skb);
465 	int slots;
466 	int frag_slots;
467 
468 	slots = DIV_ROUND_UP(offset + len, PAGE_SIZE);
469 	frag_slots = count_skb_frag_slots(skb);
470 	return slots + frag_slots;
471 }
472 
net_checksum_info(struct sk_buff * skb)473 static u32 net_checksum_info(struct sk_buff *skb)
474 {
475 	if (skb->protocol == htons(ETH_P_IP)) {
476 		struct iphdr *ip = ip_hdr(skb);
477 
478 		if (ip->protocol == IPPROTO_TCP)
479 			return TRANSPORT_INFO_IPV4_TCP;
480 		else if (ip->protocol == IPPROTO_UDP)
481 			return TRANSPORT_INFO_IPV4_UDP;
482 	} else {
483 		struct ipv6hdr *ip6 = ipv6_hdr(skb);
484 
485 		if (ip6->nexthdr == IPPROTO_TCP)
486 			return TRANSPORT_INFO_IPV6_TCP;
487 		else if (ip6->nexthdr == IPPROTO_UDP)
488 			return TRANSPORT_INFO_IPV6_UDP;
489 	}
490 
491 	return TRANSPORT_INFO_NOT_IP;
492 }
493 
494 /* Send skb on the slave VF device. */
netvsc_vf_xmit(struct net_device * net,struct net_device * vf_netdev,struct sk_buff * skb)495 static int netvsc_vf_xmit(struct net_device *net, struct net_device *vf_netdev,
496 			  struct sk_buff *skb)
497 {
498 	struct net_device_context *ndev_ctx = netdev_priv(net);
499 	unsigned int len = skb->len;
500 	int rc;
501 
502 	skb->dev = vf_netdev;
503 	skb->queue_mapping = qdisc_skb_cb(skb)->slave_dev_queue_mapping;
504 
505 	rc = dev_queue_xmit(skb);
506 	if (likely(rc == NET_XMIT_SUCCESS || rc == NET_XMIT_CN)) {
507 		struct netvsc_vf_pcpu_stats *pcpu_stats
508 			= this_cpu_ptr(ndev_ctx->vf_stats);
509 
510 		u64_stats_update_begin(&pcpu_stats->syncp);
511 		pcpu_stats->tx_packets++;
512 		pcpu_stats->tx_bytes += len;
513 		u64_stats_update_end(&pcpu_stats->syncp);
514 	} else {
515 		this_cpu_inc(ndev_ctx->vf_stats->tx_dropped);
516 	}
517 
518 	return rc;
519 }
520 
netvsc_start_xmit(struct sk_buff * skb,struct net_device * net)521 static int netvsc_start_xmit(struct sk_buff *skb, struct net_device *net)
522 {
523 	struct net_device_context *net_device_ctx = netdev_priv(net);
524 	struct hv_netvsc_packet *packet = NULL;
525 	int ret;
526 	unsigned int num_data_pgs;
527 	struct rndis_message *rndis_msg;
528 	struct rndis_packet *rndis_pkt;
529 	struct net_device *vf_netdev;
530 	u32 rndis_msg_size;
531 	struct rndis_per_packet_info *ppi;
532 	u32 hash;
533 	struct hv_page_buffer pb[MAX_PAGE_BUFFER_COUNT];
534 
535 	/* if VF is present and up then redirect packets
536 	 * already called with rcu_read_lock_bh
537 	 */
538 	vf_netdev = rcu_dereference_bh(net_device_ctx->vf_netdev);
539 	if (vf_netdev && netif_running(vf_netdev) &&
540 	    !netpoll_tx_running(net))
541 		return netvsc_vf_xmit(net, vf_netdev, skb);
542 
543 	/* We will atmost need two pages to describe the rndis
544 	 * header. We can only transmit MAX_PAGE_BUFFER_COUNT number
545 	 * of pages in a single packet. If skb is scattered around
546 	 * more pages we try linearizing it.
547 	 */
548 
549 	num_data_pgs = netvsc_get_slots(skb) + 2;
550 
551 	if (unlikely(num_data_pgs > MAX_PAGE_BUFFER_COUNT)) {
552 		++net_device_ctx->eth_stats.tx_scattered;
553 
554 		if (skb_linearize(skb))
555 			goto no_memory;
556 
557 		num_data_pgs = netvsc_get_slots(skb) + 2;
558 		if (num_data_pgs > MAX_PAGE_BUFFER_COUNT) {
559 			++net_device_ctx->eth_stats.tx_too_big;
560 			goto drop;
561 		}
562 	}
563 
564 	/*
565 	 * Place the rndis header in the skb head room and
566 	 * the skb->cb will be used for hv_netvsc_packet
567 	 * structure.
568 	 */
569 	ret = skb_cow_head(skb, RNDIS_AND_PPI_SIZE);
570 	if (ret)
571 		goto no_memory;
572 
573 	/* Use the skb control buffer for building up the packet */
574 	BUILD_BUG_ON(sizeof(struct hv_netvsc_packet) >
575 			FIELD_SIZEOF(struct sk_buff, cb));
576 	packet = (struct hv_netvsc_packet *)skb->cb;
577 
578 	packet->q_idx = skb_get_queue_mapping(skb);
579 
580 	packet->total_data_buflen = skb->len;
581 	packet->total_bytes = skb->len;
582 	packet->total_packets = 1;
583 
584 	rndis_msg = (struct rndis_message *)skb->head;
585 
586 	memset(rndis_msg, 0, RNDIS_AND_PPI_SIZE);
587 
588 	/* Add the rndis header */
589 	rndis_msg->ndis_msg_type = RNDIS_MSG_PACKET;
590 	rndis_msg->msg_len = packet->total_data_buflen;
591 	rndis_pkt = &rndis_msg->msg.pkt;
592 	rndis_pkt->data_offset = sizeof(struct rndis_packet);
593 	rndis_pkt->data_len = packet->total_data_buflen;
594 	rndis_pkt->per_pkt_info_offset = sizeof(struct rndis_packet);
595 
596 	rndis_msg_size = RNDIS_MESSAGE_SIZE(struct rndis_packet);
597 
598 	hash = skb_get_hash_raw(skb);
599 	if (hash != 0 && net->real_num_tx_queues > 1) {
600 		rndis_msg_size += NDIS_HASH_PPI_SIZE;
601 		ppi = init_ppi_data(rndis_msg, NDIS_HASH_PPI_SIZE,
602 				    NBL_HASH_VALUE);
603 		*(u32 *)((void *)ppi + ppi->ppi_offset) = hash;
604 	}
605 
606 	if (skb_vlan_tag_present(skb)) {
607 		struct ndis_pkt_8021q_info *vlan;
608 
609 		rndis_msg_size += NDIS_VLAN_PPI_SIZE;
610 		ppi = init_ppi_data(rndis_msg, NDIS_VLAN_PPI_SIZE,
611 				    IEEE_8021Q_INFO);
612 
613 		vlan = (void *)ppi + ppi->ppi_offset;
614 		vlan->vlanid = skb->vlan_tci & VLAN_VID_MASK;
615 		vlan->pri = (skb->vlan_tci & VLAN_PRIO_MASK) >>
616 				VLAN_PRIO_SHIFT;
617 	}
618 
619 	if (skb_is_gso(skb)) {
620 		struct ndis_tcp_lso_info *lso_info;
621 
622 		rndis_msg_size += NDIS_LSO_PPI_SIZE;
623 		ppi = init_ppi_data(rndis_msg, NDIS_LSO_PPI_SIZE,
624 				    TCP_LARGESEND_PKTINFO);
625 
626 		lso_info = (void *)ppi + ppi->ppi_offset;
627 
628 		lso_info->lso_v2_transmit.type = NDIS_TCP_LARGE_SEND_OFFLOAD_V2_TYPE;
629 		if (skb->protocol == htons(ETH_P_IP)) {
630 			lso_info->lso_v2_transmit.ip_version =
631 				NDIS_TCP_LARGE_SEND_OFFLOAD_IPV4;
632 			ip_hdr(skb)->tot_len = 0;
633 			ip_hdr(skb)->check = 0;
634 			tcp_hdr(skb)->check =
635 				~csum_tcpudp_magic(ip_hdr(skb)->saddr,
636 						   ip_hdr(skb)->daddr, 0, IPPROTO_TCP, 0);
637 		} else {
638 			lso_info->lso_v2_transmit.ip_version =
639 				NDIS_TCP_LARGE_SEND_OFFLOAD_IPV6;
640 			ipv6_hdr(skb)->payload_len = 0;
641 			tcp_hdr(skb)->check =
642 				~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
643 						 &ipv6_hdr(skb)->daddr, 0, IPPROTO_TCP, 0);
644 		}
645 		lso_info->lso_v2_transmit.tcp_header_offset = skb_transport_offset(skb);
646 		lso_info->lso_v2_transmit.mss = skb_shinfo(skb)->gso_size;
647 	} else if (skb->ip_summed == CHECKSUM_PARTIAL) {
648 		if (net_checksum_info(skb) & net_device_ctx->tx_checksum_mask) {
649 			struct ndis_tcp_ip_checksum_info *csum_info;
650 
651 			rndis_msg_size += NDIS_CSUM_PPI_SIZE;
652 			ppi = init_ppi_data(rndis_msg, NDIS_CSUM_PPI_SIZE,
653 					    TCPIP_CHKSUM_PKTINFO);
654 
655 			csum_info = (struct ndis_tcp_ip_checksum_info *)((void *)ppi +
656 									 ppi->ppi_offset);
657 
658 			csum_info->transmit.tcp_header_offset = skb_transport_offset(skb);
659 
660 			if (skb->protocol == htons(ETH_P_IP)) {
661 				csum_info->transmit.is_ipv4 = 1;
662 
663 				if (ip_hdr(skb)->protocol == IPPROTO_TCP)
664 					csum_info->transmit.tcp_checksum = 1;
665 				else
666 					csum_info->transmit.udp_checksum = 1;
667 			} else {
668 				csum_info->transmit.is_ipv6 = 1;
669 
670 				if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
671 					csum_info->transmit.tcp_checksum = 1;
672 				else
673 					csum_info->transmit.udp_checksum = 1;
674 			}
675 		} else {
676 			/* Can't do offload of this type of checksum */
677 			if (skb_checksum_help(skb))
678 				goto drop;
679 		}
680 	}
681 
682 	/* Start filling in the page buffers with the rndis hdr */
683 	rndis_msg->msg_len += rndis_msg_size;
684 	packet->total_data_buflen = rndis_msg->msg_len;
685 	packet->page_buf_cnt = init_page_array(rndis_msg, rndis_msg_size,
686 					       skb, packet, pb);
687 
688 	/* timestamp packet in software */
689 	skb_tx_timestamp(skb);
690 
691 	ret = netvsc_send(net, packet, rndis_msg, pb, skb);
692 	if (likely(ret == 0))
693 		return NETDEV_TX_OK;
694 
695 	if (ret == -EAGAIN) {
696 		++net_device_ctx->eth_stats.tx_busy;
697 		return NETDEV_TX_BUSY;
698 	}
699 
700 	if (ret == -ENOSPC)
701 		++net_device_ctx->eth_stats.tx_no_space;
702 
703 drop:
704 	dev_kfree_skb_any(skb);
705 	net->stats.tx_dropped++;
706 
707 	return NETDEV_TX_OK;
708 
709 no_memory:
710 	++net_device_ctx->eth_stats.tx_no_memory;
711 	goto drop;
712 }
713 
714 /*
715  * netvsc_linkstatus_callback - Link up/down notification
716  */
netvsc_linkstatus_callback(struct hv_device * device_obj,struct rndis_message * resp)717 void netvsc_linkstatus_callback(struct hv_device *device_obj,
718 				struct rndis_message *resp)
719 {
720 	struct rndis_indicate_status *indicate = &resp->msg.indicate_status;
721 	struct net_device *net;
722 	struct net_device_context *ndev_ctx;
723 	struct netvsc_reconfig *event;
724 	unsigned long flags;
725 
726 	net = hv_get_drvdata(device_obj);
727 
728 	if (!net)
729 		return;
730 
731 	ndev_ctx = netdev_priv(net);
732 
733 	/* Update the physical link speed when changing to another vSwitch */
734 	if (indicate->status == RNDIS_STATUS_LINK_SPEED_CHANGE) {
735 		u32 speed;
736 
737 		speed = *(u32 *)((void *)indicate
738 				 + indicate->status_buf_offset) / 10000;
739 		ndev_ctx->speed = speed;
740 		return;
741 	}
742 
743 	/* Handle these link change statuses below */
744 	if (indicate->status != RNDIS_STATUS_NETWORK_CHANGE &&
745 	    indicate->status != RNDIS_STATUS_MEDIA_CONNECT &&
746 	    indicate->status != RNDIS_STATUS_MEDIA_DISCONNECT)
747 		return;
748 
749 	if (net->reg_state != NETREG_REGISTERED)
750 		return;
751 
752 	event = kzalloc(sizeof(*event), GFP_ATOMIC);
753 	if (!event)
754 		return;
755 	event->event = indicate->status;
756 
757 	spin_lock_irqsave(&ndev_ctx->lock, flags);
758 	list_add_tail(&event->list, &ndev_ctx->reconfig_events);
759 	spin_unlock_irqrestore(&ndev_ctx->lock, flags);
760 
761 	schedule_delayed_work(&ndev_ctx->dwork, 0);
762 }
763 
netvsc_comp_ipcsum(struct sk_buff * skb)764 static void netvsc_comp_ipcsum(struct sk_buff *skb)
765 {
766 	struct iphdr *iph = (struct iphdr *)skb->data;
767 
768 	iph->check = 0;
769 	iph->check = ip_fast_csum(iph, iph->ihl);
770 }
771 
netvsc_alloc_recv_skb(struct net_device * net,struct napi_struct * napi,const struct ndis_tcp_ip_checksum_info * csum_info,const struct ndis_pkt_8021q_info * vlan,void * data,u32 buflen)772 static struct sk_buff *netvsc_alloc_recv_skb(struct net_device *net,
773 					     struct napi_struct *napi,
774 					     const struct ndis_tcp_ip_checksum_info *csum_info,
775 					     const struct ndis_pkt_8021q_info *vlan,
776 					     void *data, u32 buflen)
777 {
778 	struct sk_buff *skb;
779 
780 	skb = napi_alloc_skb(napi, buflen);
781 	if (!skb)
782 		return skb;
783 
784 	/*
785 	 * Copy to skb. This copy is needed here since the memory pointed by
786 	 * hv_netvsc_packet cannot be deallocated
787 	 */
788 	skb_put_data(skb, data, buflen);
789 
790 	skb->protocol = eth_type_trans(skb, net);
791 
792 	/* skb is already created with CHECKSUM_NONE */
793 	skb_checksum_none_assert(skb);
794 
795 	/* Incoming packets may have IP header checksum verified by the host.
796 	 * They may not have IP header checksum computed after coalescing.
797 	 * We compute it here if the flags are set, because on Linux, the IP
798 	 * checksum is always checked.
799 	 */
800 	if (csum_info && csum_info->receive.ip_checksum_value_invalid &&
801 	    csum_info->receive.ip_checksum_succeeded &&
802 	    skb->protocol == htons(ETH_P_IP))
803 		netvsc_comp_ipcsum(skb);
804 
805 	/* Do L4 checksum offload if enabled and present. */
806 	if (csum_info && (net->features & NETIF_F_RXCSUM)) {
807 		if (csum_info->receive.tcp_checksum_succeeded ||
808 		    csum_info->receive.udp_checksum_succeeded)
809 			skb->ip_summed = CHECKSUM_UNNECESSARY;
810 	}
811 
812 	if (vlan) {
813 		u16 vlan_tci = vlan->vlanid | (vlan->pri << VLAN_PRIO_SHIFT);
814 
815 		__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
816 				       vlan_tci);
817 	}
818 
819 	return skb;
820 }
821 
822 /*
823  * netvsc_recv_callback -  Callback when we receive a packet from the
824  * "wire" on the specified device.
825  */
netvsc_recv_callback(struct net_device * net,struct vmbus_channel * channel,void * data,u32 len,const struct ndis_tcp_ip_checksum_info * csum_info,const struct ndis_pkt_8021q_info * vlan)826 int netvsc_recv_callback(struct net_device *net,
827 			 struct vmbus_channel *channel,
828 			 void  *data, u32 len,
829 			 const struct ndis_tcp_ip_checksum_info *csum_info,
830 			 const struct ndis_pkt_8021q_info *vlan)
831 {
832 	struct net_device_context *net_device_ctx = netdev_priv(net);
833 	struct netvsc_device *net_device;
834 	u16 q_idx = channel->offermsg.offer.sub_channel_index;
835 	struct netvsc_channel *nvchan;
836 	struct sk_buff *skb;
837 	struct netvsc_stats *rx_stats;
838 
839 	if (net->reg_state != NETREG_REGISTERED)
840 		return NVSP_STAT_FAIL;
841 
842 	rcu_read_lock();
843 	net_device = rcu_dereference(net_device_ctx->nvdev);
844 	if (unlikely(!net_device))
845 		goto drop;
846 
847 	nvchan = &net_device->chan_table[q_idx];
848 
849 	/* Allocate a skb - TODO direct I/O to pages? */
850 	skb = netvsc_alloc_recv_skb(net, &nvchan->napi,
851 				    csum_info, vlan, data, len);
852 	if (unlikely(!skb)) {
853 drop:
854 		++net->stats.rx_dropped;
855 		rcu_read_unlock();
856 		return NVSP_STAT_FAIL;
857 	}
858 
859 	skb_record_rx_queue(skb, q_idx);
860 
861 	/*
862 	 * Even if injecting the packet, record the statistics
863 	 * on the synthetic device because modifying the VF device
864 	 * statistics will not work correctly.
865 	 */
866 	rx_stats = &nvchan->rx_stats;
867 	u64_stats_update_begin(&rx_stats->syncp);
868 	rx_stats->packets++;
869 	rx_stats->bytes += len;
870 
871 	if (skb->pkt_type == PACKET_BROADCAST)
872 		++rx_stats->broadcast;
873 	else if (skb->pkt_type == PACKET_MULTICAST)
874 		++rx_stats->multicast;
875 	u64_stats_update_end(&rx_stats->syncp);
876 
877 	napi_gro_receive(&nvchan->napi, skb);
878 	rcu_read_unlock();
879 
880 	return 0;
881 }
882 
netvsc_get_drvinfo(struct net_device * net,struct ethtool_drvinfo * info)883 static void netvsc_get_drvinfo(struct net_device *net,
884 			       struct ethtool_drvinfo *info)
885 {
886 	strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
887 	strlcpy(info->fw_version, "N/A", sizeof(info->fw_version));
888 }
889 
netvsc_get_channels(struct net_device * net,struct ethtool_channels * channel)890 static void netvsc_get_channels(struct net_device *net,
891 				struct ethtool_channels *channel)
892 {
893 	struct net_device_context *net_device_ctx = netdev_priv(net);
894 	struct netvsc_device *nvdev = rtnl_dereference(net_device_ctx->nvdev);
895 
896 	if (nvdev) {
897 		channel->max_combined	= nvdev->max_chn;
898 		channel->combined_count = nvdev->num_chn;
899 	}
900 }
901 
netvsc_detach(struct net_device * ndev,struct netvsc_device * nvdev)902 static int netvsc_detach(struct net_device *ndev,
903 			 struct netvsc_device *nvdev)
904 {
905 	struct net_device_context *ndev_ctx = netdev_priv(ndev);
906 	struct hv_device *hdev = ndev_ctx->device_ctx;
907 	int ret;
908 
909 	/* Don't try continuing to try and setup sub channels */
910 	if (cancel_work_sync(&nvdev->subchan_work))
911 		nvdev->num_chn = 1;
912 
913 	/* If device was up (receiving) then shutdown */
914 	if (netif_running(ndev)) {
915 		netvsc_tx_disable(nvdev, ndev);
916 
917 		ret = rndis_filter_close(nvdev);
918 		if (ret) {
919 			netdev_err(ndev,
920 				   "unable to close device (ret %d).\n", ret);
921 			return ret;
922 		}
923 
924 		ret = netvsc_wait_until_empty(nvdev);
925 		if (ret) {
926 			netdev_err(ndev,
927 				   "Ring buffer not empty after closing rndis\n");
928 			return ret;
929 		}
930 	}
931 
932 	netif_device_detach(ndev);
933 
934 	rndis_filter_device_remove(hdev, nvdev);
935 
936 	return 0;
937 }
938 
netvsc_attach(struct net_device * ndev,struct netvsc_device_info * dev_info)939 static int netvsc_attach(struct net_device *ndev,
940 			 struct netvsc_device_info *dev_info)
941 {
942 	struct net_device_context *ndev_ctx = netdev_priv(ndev);
943 	struct hv_device *hdev = ndev_ctx->device_ctx;
944 	struct netvsc_device *nvdev;
945 	struct rndis_device *rdev;
946 	int ret;
947 
948 	nvdev = rndis_filter_device_add(hdev, dev_info);
949 	if (IS_ERR(nvdev))
950 		return PTR_ERR(nvdev);
951 
952 	if (nvdev->num_chn > 1) {
953 		ret = rndis_set_subchannel(ndev, nvdev);
954 
955 		/* if unavailable, just proceed with one queue */
956 		if (ret) {
957 			nvdev->max_chn = 1;
958 			nvdev->num_chn = 1;
959 		}
960 	}
961 
962 	/* In any case device is now ready */
963 	netif_device_attach(ndev);
964 
965 	/* Note: enable and attach happen when sub-channels setup */
966 	netif_carrier_off(ndev);
967 
968 	if (netif_running(ndev)) {
969 		ret = rndis_filter_open(nvdev);
970 		if (ret)
971 			goto err;
972 
973 		rdev = nvdev->extension;
974 		if (!rdev->link_state)
975 			netif_carrier_on(ndev);
976 	}
977 
978 	return 0;
979 
980 err:
981 	netif_device_detach(ndev);
982 
983 	rndis_filter_device_remove(hdev, nvdev);
984 
985 	return ret;
986 }
987 
netvsc_set_channels(struct net_device * net,struct ethtool_channels * channels)988 static int netvsc_set_channels(struct net_device *net,
989 			       struct ethtool_channels *channels)
990 {
991 	struct net_device_context *net_device_ctx = netdev_priv(net);
992 	struct netvsc_device *nvdev = rtnl_dereference(net_device_ctx->nvdev);
993 	unsigned int orig, count = channels->combined_count;
994 	struct netvsc_device_info device_info;
995 	int ret;
996 
997 	/* We do not support separate count for rx, tx, or other */
998 	if (count == 0 ||
999 	    channels->rx_count || channels->tx_count || channels->other_count)
1000 		return -EINVAL;
1001 
1002 	if (!nvdev || nvdev->destroy)
1003 		return -ENODEV;
1004 
1005 	if (nvdev->nvsp_version < NVSP_PROTOCOL_VERSION_5)
1006 		return -EINVAL;
1007 
1008 	if (count > nvdev->max_chn)
1009 		return -EINVAL;
1010 
1011 	orig = nvdev->num_chn;
1012 
1013 	memset(&device_info, 0, sizeof(device_info));
1014 	device_info.num_chn = count;
1015 	device_info.ring_size = ring_size;
1016 	device_info.send_sections = nvdev->send_section_cnt;
1017 	device_info.send_section_size = nvdev->send_section_size;
1018 	device_info.recv_sections = nvdev->recv_section_cnt;
1019 	device_info.recv_section_size = nvdev->recv_section_size;
1020 
1021 	ret = netvsc_detach(net, nvdev);
1022 	if (ret)
1023 		return ret;
1024 
1025 	ret = netvsc_attach(net, &device_info);
1026 	if (ret) {
1027 		device_info.num_chn = orig;
1028 		if (netvsc_attach(net, &device_info))
1029 			netdev_err(net, "restoring channel setting failed\n");
1030 	}
1031 
1032 	return ret;
1033 }
1034 
1035 static bool
netvsc_validate_ethtool_ss_cmd(const struct ethtool_link_ksettings * cmd)1036 netvsc_validate_ethtool_ss_cmd(const struct ethtool_link_ksettings *cmd)
1037 {
1038 	struct ethtool_link_ksettings diff1 = *cmd;
1039 	struct ethtool_link_ksettings diff2 = {};
1040 
1041 	diff1.base.speed = 0;
1042 	diff1.base.duplex = 0;
1043 	/* advertising and cmd are usually set */
1044 	ethtool_link_ksettings_zero_link_mode(&diff1, advertising);
1045 	diff1.base.cmd = 0;
1046 	/* We set port to PORT_OTHER */
1047 	diff2.base.port = PORT_OTHER;
1048 
1049 	return !memcmp(&diff1, &diff2, sizeof(diff1));
1050 }
1051 
netvsc_init_settings(struct net_device * dev)1052 static void netvsc_init_settings(struct net_device *dev)
1053 {
1054 	struct net_device_context *ndc = netdev_priv(dev);
1055 
1056 	ndc->udp4_l4_hash = true;
1057 	ndc->udp6_l4_hash = true;
1058 
1059 	ndc->speed = SPEED_UNKNOWN;
1060 	ndc->duplex = DUPLEX_FULL;
1061 }
1062 
netvsc_get_link_ksettings(struct net_device * dev,struct ethtool_link_ksettings * cmd)1063 static int netvsc_get_link_ksettings(struct net_device *dev,
1064 				     struct ethtool_link_ksettings *cmd)
1065 {
1066 	struct net_device_context *ndc = netdev_priv(dev);
1067 
1068 	cmd->base.speed = ndc->speed;
1069 	cmd->base.duplex = ndc->duplex;
1070 	cmd->base.port = PORT_OTHER;
1071 
1072 	return 0;
1073 }
1074 
netvsc_set_link_ksettings(struct net_device * dev,const struct ethtool_link_ksettings * cmd)1075 static int netvsc_set_link_ksettings(struct net_device *dev,
1076 				     const struct ethtool_link_ksettings *cmd)
1077 {
1078 	struct net_device_context *ndc = netdev_priv(dev);
1079 	u32 speed;
1080 
1081 	speed = cmd->base.speed;
1082 	if (!ethtool_validate_speed(speed) ||
1083 	    !ethtool_validate_duplex(cmd->base.duplex) ||
1084 	    !netvsc_validate_ethtool_ss_cmd(cmd))
1085 		return -EINVAL;
1086 
1087 	ndc->speed = speed;
1088 	ndc->duplex = cmd->base.duplex;
1089 
1090 	return 0;
1091 }
1092 
netvsc_change_mtu(struct net_device * ndev,int mtu)1093 static int netvsc_change_mtu(struct net_device *ndev, int mtu)
1094 {
1095 	struct net_device_context *ndevctx = netdev_priv(ndev);
1096 	struct net_device *vf_netdev = rtnl_dereference(ndevctx->vf_netdev);
1097 	struct netvsc_device *nvdev = rtnl_dereference(ndevctx->nvdev);
1098 	int orig_mtu = ndev->mtu;
1099 	struct netvsc_device_info device_info;
1100 	int ret = 0;
1101 
1102 	if (!nvdev || nvdev->destroy)
1103 		return -ENODEV;
1104 
1105 	/* Change MTU of underlying VF netdev first. */
1106 	if (vf_netdev) {
1107 		ret = dev_set_mtu(vf_netdev, mtu);
1108 		if (ret)
1109 			return ret;
1110 	}
1111 
1112 	memset(&device_info, 0, sizeof(device_info));
1113 	device_info.ring_size = ring_size;
1114 	device_info.num_chn = nvdev->num_chn;
1115 	device_info.send_sections = nvdev->send_section_cnt;
1116 	device_info.send_section_size = nvdev->send_section_size;
1117 	device_info.recv_sections = nvdev->recv_section_cnt;
1118 	device_info.recv_section_size = nvdev->recv_section_size;
1119 
1120 	ret = netvsc_detach(ndev, nvdev);
1121 	if (ret)
1122 		goto rollback_vf;
1123 
1124 	ndev->mtu = mtu;
1125 
1126 	ret = netvsc_attach(ndev, &device_info);
1127 	if (ret)
1128 		goto rollback;
1129 
1130 	return 0;
1131 
1132 rollback:
1133 	/* Attempt rollback to original MTU */
1134 	ndev->mtu = orig_mtu;
1135 
1136 	if (netvsc_attach(ndev, &device_info))
1137 		netdev_err(ndev, "restoring mtu failed\n");
1138 rollback_vf:
1139 	if (vf_netdev)
1140 		dev_set_mtu(vf_netdev, orig_mtu);
1141 
1142 	return ret;
1143 }
1144 
netvsc_get_vf_stats(struct net_device * net,struct netvsc_vf_pcpu_stats * tot)1145 static void netvsc_get_vf_stats(struct net_device *net,
1146 				struct netvsc_vf_pcpu_stats *tot)
1147 {
1148 	struct net_device_context *ndev_ctx = netdev_priv(net);
1149 	int i;
1150 
1151 	memset(tot, 0, sizeof(*tot));
1152 
1153 	for_each_possible_cpu(i) {
1154 		const struct netvsc_vf_pcpu_stats *stats
1155 			= per_cpu_ptr(ndev_ctx->vf_stats, i);
1156 		u64 rx_packets, rx_bytes, tx_packets, tx_bytes;
1157 		unsigned int start;
1158 
1159 		do {
1160 			start = u64_stats_fetch_begin_irq(&stats->syncp);
1161 			rx_packets = stats->rx_packets;
1162 			tx_packets = stats->tx_packets;
1163 			rx_bytes = stats->rx_bytes;
1164 			tx_bytes = stats->tx_bytes;
1165 		} while (u64_stats_fetch_retry_irq(&stats->syncp, start));
1166 
1167 		tot->rx_packets += rx_packets;
1168 		tot->tx_packets += tx_packets;
1169 		tot->rx_bytes   += rx_bytes;
1170 		tot->tx_bytes   += tx_bytes;
1171 		tot->tx_dropped += stats->tx_dropped;
1172 	}
1173 }
1174 
netvsc_get_stats64(struct net_device * net,struct rtnl_link_stats64 * t)1175 static void netvsc_get_stats64(struct net_device *net,
1176 			       struct rtnl_link_stats64 *t)
1177 {
1178 	struct net_device_context *ndev_ctx = netdev_priv(net);
1179 	struct netvsc_device *nvdev;
1180 	struct netvsc_vf_pcpu_stats vf_tot;
1181 	int i;
1182 
1183 	rcu_read_lock();
1184 
1185 	nvdev = rcu_dereference(ndev_ctx->nvdev);
1186 	if (!nvdev)
1187 		goto out;
1188 
1189 	netdev_stats_to_stats64(t, &net->stats);
1190 
1191 	netvsc_get_vf_stats(net, &vf_tot);
1192 	t->rx_packets += vf_tot.rx_packets;
1193 	t->tx_packets += vf_tot.tx_packets;
1194 	t->rx_bytes   += vf_tot.rx_bytes;
1195 	t->tx_bytes   += vf_tot.tx_bytes;
1196 	t->tx_dropped += vf_tot.tx_dropped;
1197 
1198 	for (i = 0; i < nvdev->num_chn; i++) {
1199 		const struct netvsc_channel *nvchan = &nvdev->chan_table[i];
1200 		const struct netvsc_stats *stats;
1201 		u64 packets, bytes, multicast;
1202 		unsigned int start;
1203 
1204 		stats = &nvchan->tx_stats;
1205 		do {
1206 			start = u64_stats_fetch_begin_irq(&stats->syncp);
1207 			packets = stats->packets;
1208 			bytes = stats->bytes;
1209 		} while (u64_stats_fetch_retry_irq(&stats->syncp, start));
1210 
1211 		t->tx_bytes	+= bytes;
1212 		t->tx_packets	+= packets;
1213 
1214 		stats = &nvchan->rx_stats;
1215 		do {
1216 			start = u64_stats_fetch_begin_irq(&stats->syncp);
1217 			packets = stats->packets;
1218 			bytes = stats->bytes;
1219 			multicast = stats->multicast + stats->broadcast;
1220 		} while (u64_stats_fetch_retry_irq(&stats->syncp, start));
1221 
1222 		t->rx_bytes	+= bytes;
1223 		t->rx_packets	+= packets;
1224 		t->multicast	+= multicast;
1225 	}
1226 out:
1227 	rcu_read_unlock();
1228 }
1229 
netvsc_set_mac_addr(struct net_device * ndev,void * p)1230 static int netvsc_set_mac_addr(struct net_device *ndev, void *p)
1231 {
1232 	struct net_device_context *ndc = netdev_priv(ndev);
1233 	struct net_device *vf_netdev = rtnl_dereference(ndc->vf_netdev);
1234 	struct netvsc_device *nvdev = rtnl_dereference(ndc->nvdev);
1235 	struct sockaddr *addr = p;
1236 	int err;
1237 
1238 	err = eth_prepare_mac_addr_change(ndev, p);
1239 	if (err)
1240 		return err;
1241 
1242 	if (!nvdev)
1243 		return -ENODEV;
1244 
1245 	if (vf_netdev) {
1246 		err = dev_set_mac_address(vf_netdev, addr);
1247 		if (err)
1248 			return err;
1249 	}
1250 
1251 	err = rndis_filter_set_device_mac(nvdev, addr->sa_data);
1252 	if (!err) {
1253 		eth_commit_mac_addr_change(ndev, p);
1254 	} else if (vf_netdev) {
1255 		/* rollback change on VF */
1256 		memcpy(addr->sa_data, ndev->dev_addr, ETH_ALEN);
1257 		dev_set_mac_address(vf_netdev, addr);
1258 	}
1259 
1260 	return err;
1261 }
1262 
1263 static const struct {
1264 	char name[ETH_GSTRING_LEN];
1265 	u16 offset;
1266 } netvsc_stats[] = {
1267 	{ "tx_scattered", offsetof(struct netvsc_ethtool_stats, tx_scattered) },
1268 	{ "tx_no_memory",  offsetof(struct netvsc_ethtool_stats, tx_no_memory) },
1269 	{ "tx_no_space",  offsetof(struct netvsc_ethtool_stats, tx_no_space) },
1270 	{ "tx_too_big",	  offsetof(struct netvsc_ethtool_stats, tx_too_big) },
1271 	{ "tx_busy",	  offsetof(struct netvsc_ethtool_stats, tx_busy) },
1272 	{ "tx_send_full", offsetof(struct netvsc_ethtool_stats, tx_send_full) },
1273 	{ "rx_comp_busy", offsetof(struct netvsc_ethtool_stats, rx_comp_busy) },
1274 }, vf_stats[] = {
1275 	{ "vf_rx_packets", offsetof(struct netvsc_vf_pcpu_stats, rx_packets) },
1276 	{ "vf_rx_bytes",   offsetof(struct netvsc_vf_pcpu_stats, rx_bytes) },
1277 	{ "vf_tx_packets", offsetof(struct netvsc_vf_pcpu_stats, tx_packets) },
1278 	{ "vf_tx_bytes",   offsetof(struct netvsc_vf_pcpu_stats, tx_bytes) },
1279 	{ "vf_tx_dropped", offsetof(struct netvsc_vf_pcpu_stats, tx_dropped) },
1280 };
1281 
1282 #define NETVSC_GLOBAL_STATS_LEN	ARRAY_SIZE(netvsc_stats)
1283 #define NETVSC_VF_STATS_LEN	ARRAY_SIZE(vf_stats)
1284 
1285 /* 4 statistics per queue (rx/tx packets/bytes) */
1286 #define NETVSC_QUEUE_STATS_LEN(dev) ((dev)->num_chn * 4)
1287 
netvsc_get_sset_count(struct net_device * dev,int string_set)1288 static int netvsc_get_sset_count(struct net_device *dev, int string_set)
1289 {
1290 	struct net_device_context *ndc = netdev_priv(dev);
1291 	struct netvsc_device *nvdev = rtnl_dereference(ndc->nvdev);
1292 
1293 	if (!nvdev)
1294 		return -ENODEV;
1295 
1296 	switch (string_set) {
1297 	case ETH_SS_STATS:
1298 		return NETVSC_GLOBAL_STATS_LEN
1299 			+ NETVSC_VF_STATS_LEN
1300 			+ NETVSC_QUEUE_STATS_LEN(nvdev);
1301 	default:
1302 		return -EINVAL;
1303 	}
1304 }
1305 
netvsc_get_ethtool_stats(struct net_device * dev,struct ethtool_stats * stats,u64 * data)1306 static void netvsc_get_ethtool_stats(struct net_device *dev,
1307 				     struct ethtool_stats *stats, u64 *data)
1308 {
1309 	struct net_device_context *ndc = netdev_priv(dev);
1310 	struct netvsc_device *nvdev = rtnl_dereference(ndc->nvdev);
1311 	const void *nds = &ndc->eth_stats;
1312 	const struct netvsc_stats *qstats;
1313 	struct netvsc_vf_pcpu_stats sum;
1314 	unsigned int start;
1315 	u64 packets, bytes;
1316 	int i, j;
1317 
1318 	if (!nvdev)
1319 		return;
1320 
1321 	for (i = 0; i < NETVSC_GLOBAL_STATS_LEN; i++)
1322 		data[i] = *(unsigned long *)(nds + netvsc_stats[i].offset);
1323 
1324 	netvsc_get_vf_stats(dev, &sum);
1325 	for (j = 0; j < NETVSC_VF_STATS_LEN; j++)
1326 		data[i++] = *(u64 *)((void *)&sum + vf_stats[j].offset);
1327 
1328 	for (j = 0; j < nvdev->num_chn; j++) {
1329 		qstats = &nvdev->chan_table[j].tx_stats;
1330 
1331 		do {
1332 			start = u64_stats_fetch_begin_irq(&qstats->syncp);
1333 			packets = qstats->packets;
1334 			bytes = qstats->bytes;
1335 		} while (u64_stats_fetch_retry_irq(&qstats->syncp, start));
1336 		data[i++] = packets;
1337 		data[i++] = bytes;
1338 
1339 		qstats = &nvdev->chan_table[j].rx_stats;
1340 		do {
1341 			start = u64_stats_fetch_begin_irq(&qstats->syncp);
1342 			packets = qstats->packets;
1343 			bytes = qstats->bytes;
1344 		} while (u64_stats_fetch_retry_irq(&qstats->syncp, start));
1345 		data[i++] = packets;
1346 		data[i++] = bytes;
1347 	}
1348 }
1349 
netvsc_get_strings(struct net_device * dev,u32 stringset,u8 * data)1350 static void netvsc_get_strings(struct net_device *dev, u32 stringset, u8 *data)
1351 {
1352 	struct net_device_context *ndc = netdev_priv(dev);
1353 	struct netvsc_device *nvdev = rtnl_dereference(ndc->nvdev);
1354 	u8 *p = data;
1355 	int i;
1356 
1357 	if (!nvdev)
1358 		return;
1359 
1360 	switch (stringset) {
1361 	case ETH_SS_STATS:
1362 		for (i = 0; i < ARRAY_SIZE(netvsc_stats); i++) {
1363 			memcpy(p, netvsc_stats[i].name, ETH_GSTRING_LEN);
1364 			p += ETH_GSTRING_LEN;
1365 		}
1366 
1367 		for (i = 0; i < ARRAY_SIZE(vf_stats); i++) {
1368 			memcpy(p, vf_stats[i].name, ETH_GSTRING_LEN);
1369 			p += ETH_GSTRING_LEN;
1370 		}
1371 
1372 		for (i = 0; i < nvdev->num_chn; i++) {
1373 			sprintf(p, "tx_queue_%u_packets", i);
1374 			p += ETH_GSTRING_LEN;
1375 			sprintf(p, "tx_queue_%u_bytes", i);
1376 			p += ETH_GSTRING_LEN;
1377 			sprintf(p, "rx_queue_%u_packets", i);
1378 			p += ETH_GSTRING_LEN;
1379 			sprintf(p, "rx_queue_%u_bytes", i);
1380 			p += ETH_GSTRING_LEN;
1381 		}
1382 
1383 		break;
1384 	}
1385 }
1386 
1387 static int
netvsc_get_rss_hash_opts(struct net_device_context * ndc,struct ethtool_rxnfc * info)1388 netvsc_get_rss_hash_opts(struct net_device_context *ndc,
1389 			 struct ethtool_rxnfc *info)
1390 {
1391 	info->data = RXH_IP_SRC | RXH_IP_DST;
1392 
1393 	switch (info->flow_type) {
1394 	case TCP_V4_FLOW:
1395 	case TCP_V6_FLOW:
1396 		info->data |= RXH_L4_B_0_1 | RXH_L4_B_2_3;
1397 		break;
1398 
1399 	case UDP_V4_FLOW:
1400 		if (ndc->udp4_l4_hash)
1401 			info->data |= RXH_L4_B_0_1 | RXH_L4_B_2_3;
1402 
1403 		break;
1404 
1405 	case UDP_V6_FLOW:
1406 		if (ndc->udp6_l4_hash)
1407 			info->data |= RXH_L4_B_0_1 | RXH_L4_B_2_3;
1408 
1409 		break;
1410 
1411 	case IPV4_FLOW:
1412 	case IPV6_FLOW:
1413 		break;
1414 	default:
1415 		info->data = 0;
1416 		break;
1417 	}
1418 
1419 	return 0;
1420 }
1421 
1422 static int
netvsc_get_rxnfc(struct net_device * dev,struct ethtool_rxnfc * info,u32 * rules)1423 netvsc_get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info,
1424 		 u32 *rules)
1425 {
1426 	struct net_device_context *ndc = netdev_priv(dev);
1427 	struct netvsc_device *nvdev = rtnl_dereference(ndc->nvdev);
1428 
1429 	if (!nvdev)
1430 		return -ENODEV;
1431 
1432 	switch (info->cmd) {
1433 	case ETHTOOL_GRXRINGS:
1434 		info->data = nvdev->num_chn;
1435 		return 0;
1436 
1437 	case ETHTOOL_GRXFH:
1438 		return netvsc_get_rss_hash_opts(ndc, info);
1439 	}
1440 	return -EOPNOTSUPP;
1441 }
1442 
netvsc_set_rss_hash_opts(struct net_device_context * ndc,struct ethtool_rxnfc * info)1443 static int netvsc_set_rss_hash_opts(struct net_device_context *ndc,
1444 				    struct ethtool_rxnfc *info)
1445 {
1446 	if (info->data == (RXH_IP_SRC | RXH_IP_DST |
1447 			   RXH_L4_B_0_1 | RXH_L4_B_2_3)) {
1448 		if (info->flow_type == UDP_V4_FLOW)
1449 			ndc->udp4_l4_hash = true;
1450 		else if (info->flow_type == UDP_V6_FLOW)
1451 			ndc->udp6_l4_hash = true;
1452 		else
1453 			return -EOPNOTSUPP;
1454 
1455 		return 0;
1456 	}
1457 
1458 	if (info->data == (RXH_IP_SRC | RXH_IP_DST)) {
1459 		if (info->flow_type == UDP_V4_FLOW)
1460 			ndc->udp4_l4_hash = false;
1461 		else if (info->flow_type == UDP_V6_FLOW)
1462 			ndc->udp6_l4_hash = false;
1463 		else
1464 			return -EOPNOTSUPP;
1465 
1466 		return 0;
1467 	}
1468 
1469 	return -EOPNOTSUPP;
1470 }
1471 
1472 static int
netvsc_set_rxnfc(struct net_device * ndev,struct ethtool_rxnfc * info)1473 netvsc_set_rxnfc(struct net_device *ndev, struct ethtool_rxnfc *info)
1474 {
1475 	struct net_device_context *ndc = netdev_priv(ndev);
1476 
1477 	if (info->cmd == ETHTOOL_SRXFH)
1478 		return netvsc_set_rss_hash_opts(ndc, info);
1479 
1480 	return -EOPNOTSUPP;
1481 }
1482 
1483 #ifdef CONFIG_NET_POLL_CONTROLLER
netvsc_poll_controller(struct net_device * dev)1484 static void netvsc_poll_controller(struct net_device *dev)
1485 {
1486 	struct net_device_context *ndc = netdev_priv(dev);
1487 	struct netvsc_device *ndev;
1488 	int i;
1489 
1490 	rcu_read_lock();
1491 	ndev = rcu_dereference(ndc->nvdev);
1492 	if (ndev) {
1493 		for (i = 0; i < ndev->num_chn; i++) {
1494 			struct netvsc_channel *nvchan = &ndev->chan_table[i];
1495 
1496 			napi_schedule(&nvchan->napi);
1497 		}
1498 	}
1499 	rcu_read_unlock();
1500 }
1501 #endif
1502 
netvsc_get_rxfh_key_size(struct net_device * dev)1503 static u32 netvsc_get_rxfh_key_size(struct net_device *dev)
1504 {
1505 	return NETVSC_HASH_KEYLEN;
1506 }
1507 
netvsc_rss_indir_size(struct net_device * dev)1508 static u32 netvsc_rss_indir_size(struct net_device *dev)
1509 {
1510 	return ITAB_NUM;
1511 }
1512 
netvsc_get_rxfh(struct net_device * dev,u32 * indir,u8 * key,u8 * hfunc)1513 static int netvsc_get_rxfh(struct net_device *dev, u32 *indir, u8 *key,
1514 			   u8 *hfunc)
1515 {
1516 	struct net_device_context *ndc = netdev_priv(dev);
1517 	struct netvsc_device *ndev = rtnl_dereference(ndc->nvdev);
1518 	struct rndis_device *rndis_dev;
1519 	int i;
1520 
1521 	if (!ndev)
1522 		return -ENODEV;
1523 
1524 	if (hfunc)
1525 		*hfunc = ETH_RSS_HASH_TOP;	/* Toeplitz */
1526 
1527 	rndis_dev = ndev->extension;
1528 	if (indir) {
1529 		for (i = 0; i < ITAB_NUM; i++)
1530 			indir[i] = ndc->rx_table[i];
1531 	}
1532 
1533 	if (key)
1534 		memcpy(key, rndis_dev->rss_key, NETVSC_HASH_KEYLEN);
1535 
1536 	return 0;
1537 }
1538 
netvsc_set_rxfh(struct net_device * dev,const u32 * indir,const u8 * key,const u8 hfunc)1539 static int netvsc_set_rxfh(struct net_device *dev, const u32 *indir,
1540 			   const u8 *key, const u8 hfunc)
1541 {
1542 	struct net_device_context *ndc = netdev_priv(dev);
1543 	struct netvsc_device *ndev = rtnl_dereference(ndc->nvdev);
1544 	struct rndis_device *rndis_dev;
1545 	int i;
1546 
1547 	if (!ndev)
1548 		return -ENODEV;
1549 
1550 	if (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP)
1551 		return -EOPNOTSUPP;
1552 
1553 	rndis_dev = ndev->extension;
1554 	if (indir) {
1555 		for (i = 0; i < ITAB_NUM; i++)
1556 			if (indir[i] >= ndev->num_chn)
1557 				return -EINVAL;
1558 
1559 		for (i = 0; i < ITAB_NUM; i++)
1560 			ndc->rx_table[i] = indir[i];
1561 	}
1562 
1563 	if (!key) {
1564 		if (!indir)
1565 			return 0;
1566 
1567 		key = rndis_dev->rss_key;
1568 	}
1569 
1570 	return rndis_filter_set_rss_param(rndis_dev, key);
1571 }
1572 
1573 /* Hyper-V RNDIS protocol does not have ring in the HW sense.
1574  * It does have pre-allocated receive area which is divided into sections.
1575  */
__netvsc_get_ringparam(struct netvsc_device * nvdev,struct ethtool_ringparam * ring)1576 static void __netvsc_get_ringparam(struct netvsc_device *nvdev,
1577 				   struct ethtool_ringparam *ring)
1578 {
1579 	u32 max_buf_size;
1580 
1581 	ring->rx_pending = nvdev->recv_section_cnt;
1582 	ring->tx_pending = nvdev->send_section_cnt;
1583 
1584 	if (nvdev->nvsp_version <= NVSP_PROTOCOL_VERSION_2)
1585 		max_buf_size = NETVSC_RECEIVE_BUFFER_SIZE_LEGACY;
1586 	else
1587 		max_buf_size = NETVSC_RECEIVE_BUFFER_SIZE;
1588 
1589 	ring->rx_max_pending = max_buf_size / nvdev->recv_section_size;
1590 	ring->tx_max_pending = NETVSC_SEND_BUFFER_SIZE
1591 		/ nvdev->send_section_size;
1592 }
1593 
netvsc_get_ringparam(struct net_device * ndev,struct ethtool_ringparam * ring)1594 static void netvsc_get_ringparam(struct net_device *ndev,
1595 				 struct ethtool_ringparam *ring)
1596 {
1597 	struct net_device_context *ndevctx = netdev_priv(ndev);
1598 	struct netvsc_device *nvdev = rtnl_dereference(ndevctx->nvdev);
1599 
1600 	if (!nvdev)
1601 		return;
1602 
1603 	__netvsc_get_ringparam(nvdev, ring);
1604 }
1605 
netvsc_set_ringparam(struct net_device * ndev,struct ethtool_ringparam * ring)1606 static int netvsc_set_ringparam(struct net_device *ndev,
1607 				struct ethtool_ringparam *ring)
1608 {
1609 	struct net_device_context *ndevctx = netdev_priv(ndev);
1610 	struct netvsc_device *nvdev = rtnl_dereference(ndevctx->nvdev);
1611 	struct netvsc_device_info device_info;
1612 	struct ethtool_ringparam orig;
1613 	u32 new_tx, new_rx;
1614 	int ret = 0;
1615 
1616 	if (!nvdev || nvdev->destroy)
1617 		return -ENODEV;
1618 
1619 	memset(&orig, 0, sizeof(orig));
1620 	__netvsc_get_ringparam(nvdev, &orig);
1621 
1622 	new_tx = clamp_t(u32, ring->tx_pending,
1623 			 NETVSC_MIN_TX_SECTIONS, orig.tx_max_pending);
1624 	new_rx = clamp_t(u32, ring->rx_pending,
1625 			 NETVSC_MIN_RX_SECTIONS, orig.rx_max_pending);
1626 
1627 	if (new_tx == orig.tx_pending &&
1628 	    new_rx == orig.rx_pending)
1629 		return 0;	 /* no change */
1630 
1631 	memset(&device_info, 0, sizeof(device_info));
1632 	device_info.num_chn = nvdev->num_chn;
1633 	device_info.ring_size = ring_size;
1634 	device_info.send_sections = new_tx;
1635 	device_info.send_section_size = nvdev->send_section_size;
1636 	device_info.recv_sections = new_rx;
1637 	device_info.recv_section_size = nvdev->recv_section_size;
1638 
1639 	ret = netvsc_detach(ndev, nvdev);
1640 	if (ret)
1641 		return ret;
1642 
1643 	ret = netvsc_attach(ndev, &device_info);
1644 	if (ret) {
1645 		device_info.send_sections = orig.tx_pending;
1646 		device_info.recv_sections = orig.rx_pending;
1647 
1648 		if (netvsc_attach(ndev, &device_info))
1649 			netdev_err(ndev, "restoring ringparam failed");
1650 	}
1651 
1652 	return ret;
1653 }
1654 
1655 static const struct ethtool_ops ethtool_ops = {
1656 	.get_drvinfo	= netvsc_get_drvinfo,
1657 	.get_link	= ethtool_op_get_link,
1658 	.get_ethtool_stats = netvsc_get_ethtool_stats,
1659 	.get_sset_count = netvsc_get_sset_count,
1660 	.get_strings	= netvsc_get_strings,
1661 	.get_channels   = netvsc_get_channels,
1662 	.set_channels   = netvsc_set_channels,
1663 	.get_ts_info	= ethtool_op_get_ts_info,
1664 	.get_rxnfc	= netvsc_get_rxnfc,
1665 	.set_rxnfc	= netvsc_set_rxnfc,
1666 	.get_rxfh_key_size = netvsc_get_rxfh_key_size,
1667 	.get_rxfh_indir_size = netvsc_rss_indir_size,
1668 	.get_rxfh	= netvsc_get_rxfh,
1669 	.set_rxfh	= netvsc_set_rxfh,
1670 	.get_link_ksettings = netvsc_get_link_ksettings,
1671 	.set_link_ksettings = netvsc_set_link_ksettings,
1672 	.get_ringparam	= netvsc_get_ringparam,
1673 	.set_ringparam	= netvsc_set_ringparam,
1674 };
1675 
1676 static const struct net_device_ops device_ops = {
1677 	.ndo_open =			netvsc_open,
1678 	.ndo_stop =			netvsc_close,
1679 	.ndo_start_xmit =		netvsc_start_xmit,
1680 	.ndo_change_rx_flags =		netvsc_change_rx_flags,
1681 	.ndo_set_rx_mode =		netvsc_set_rx_mode,
1682 	.ndo_change_mtu =		netvsc_change_mtu,
1683 	.ndo_validate_addr =		eth_validate_addr,
1684 	.ndo_set_mac_address =		netvsc_set_mac_addr,
1685 	.ndo_select_queue =		netvsc_select_queue,
1686 	.ndo_get_stats64 =		netvsc_get_stats64,
1687 #ifdef CONFIG_NET_POLL_CONTROLLER
1688 	.ndo_poll_controller =		netvsc_poll_controller,
1689 #endif
1690 };
1691 
1692 /*
1693  * Handle link status changes. For RNDIS_STATUS_NETWORK_CHANGE emulate link
1694  * down/up sequence. In case of RNDIS_STATUS_MEDIA_CONNECT when carrier is
1695  * present send GARP packet to network peers with netif_notify_peers().
1696  */
netvsc_link_change(struct work_struct * w)1697 static void netvsc_link_change(struct work_struct *w)
1698 {
1699 	struct net_device_context *ndev_ctx =
1700 		container_of(w, struct net_device_context, dwork.work);
1701 	struct hv_device *device_obj = ndev_ctx->device_ctx;
1702 	struct net_device *net = hv_get_drvdata(device_obj);
1703 	struct netvsc_device *net_device;
1704 	struct rndis_device *rdev;
1705 	struct netvsc_reconfig *event = NULL;
1706 	bool notify = false, reschedule = false;
1707 	unsigned long flags, next_reconfig, delay;
1708 
1709 	/* if changes are happening, comeback later */
1710 	if (!rtnl_trylock()) {
1711 		schedule_delayed_work(&ndev_ctx->dwork, LINKCHANGE_INT);
1712 		return;
1713 	}
1714 
1715 	net_device = rtnl_dereference(ndev_ctx->nvdev);
1716 	if (!net_device)
1717 		goto out_unlock;
1718 
1719 	rdev = net_device->extension;
1720 
1721 	next_reconfig = ndev_ctx->last_reconfig + LINKCHANGE_INT;
1722 	if (time_is_after_jiffies(next_reconfig)) {
1723 		/* link_watch only sends one notification with current state
1724 		 * per second, avoid doing reconfig more frequently. Handle
1725 		 * wrap around.
1726 		 */
1727 		delay = next_reconfig - jiffies;
1728 		delay = delay < LINKCHANGE_INT ? delay : LINKCHANGE_INT;
1729 		schedule_delayed_work(&ndev_ctx->dwork, delay);
1730 		goto out_unlock;
1731 	}
1732 	ndev_ctx->last_reconfig = jiffies;
1733 
1734 	spin_lock_irqsave(&ndev_ctx->lock, flags);
1735 	if (!list_empty(&ndev_ctx->reconfig_events)) {
1736 		event = list_first_entry(&ndev_ctx->reconfig_events,
1737 					 struct netvsc_reconfig, list);
1738 		list_del(&event->list);
1739 		reschedule = !list_empty(&ndev_ctx->reconfig_events);
1740 	}
1741 	spin_unlock_irqrestore(&ndev_ctx->lock, flags);
1742 
1743 	if (!event)
1744 		goto out_unlock;
1745 
1746 	switch (event->event) {
1747 		/* Only the following events are possible due to the check in
1748 		 * netvsc_linkstatus_callback()
1749 		 */
1750 	case RNDIS_STATUS_MEDIA_CONNECT:
1751 		if (rdev->link_state) {
1752 			rdev->link_state = false;
1753 			netif_carrier_on(net);
1754 			netvsc_tx_enable(net_device, net);
1755 		} else {
1756 			notify = true;
1757 		}
1758 		kfree(event);
1759 		break;
1760 	case RNDIS_STATUS_MEDIA_DISCONNECT:
1761 		if (!rdev->link_state) {
1762 			rdev->link_state = true;
1763 			netif_carrier_off(net);
1764 			netvsc_tx_disable(net_device, net);
1765 		}
1766 		kfree(event);
1767 		break;
1768 	case RNDIS_STATUS_NETWORK_CHANGE:
1769 		/* Only makes sense if carrier is present */
1770 		if (!rdev->link_state) {
1771 			rdev->link_state = true;
1772 			netif_carrier_off(net);
1773 			netvsc_tx_disable(net_device, net);
1774 			event->event = RNDIS_STATUS_MEDIA_CONNECT;
1775 			spin_lock_irqsave(&ndev_ctx->lock, flags);
1776 			list_add(&event->list, &ndev_ctx->reconfig_events);
1777 			spin_unlock_irqrestore(&ndev_ctx->lock, flags);
1778 			reschedule = true;
1779 		}
1780 		break;
1781 	}
1782 
1783 	rtnl_unlock();
1784 
1785 	if (notify)
1786 		netdev_notify_peers(net);
1787 
1788 	/* link_watch only sends one notification with current state per
1789 	 * second, handle next reconfig event in 2 seconds.
1790 	 */
1791 	if (reschedule)
1792 		schedule_delayed_work(&ndev_ctx->dwork, LINKCHANGE_INT);
1793 
1794 	return;
1795 
1796 out_unlock:
1797 	rtnl_unlock();
1798 }
1799 
get_netvsc_bymac(const u8 * mac)1800 static struct net_device *get_netvsc_bymac(const u8 *mac)
1801 {
1802 	struct net_device_context *ndev_ctx;
1803 
1804 	list_for_each_entry(ndev_ctx, &netvsc_dev_list, list) {
1805 		struct net_device *dev = hv_get_drvdata(ndev_ctx->device_ctx);
1806 
1807 		if (ether_addr_equal(mac, dev->perm_addr))
1808 			return dev;
1809 	}
1810 
1811 	return NULL;
1812 }
1813 
get_netvsc_byref(struct net_device * vf_netdev)1814 static struct net_device *get_netvsc_byref(struct net_device *vf_netdev)
1815 {
1816 	struct net_device_context *net_device_ctx;
1817 	struct net_device *dev;
1818 
1819 	dev = netdev_master_upper_dev_get(vf_netdev);
1820 	if (!dev || dev->netdev_ops != &device_ops)
1821 		return NULL;	/* not a netvsc device */
1822 
1823 	net_device_ctx = netdev_priv(dev);
1824 	if (!rtnl_dereference(net_device_ctx->nvdev))
1825 		return NULL;	/* device is removed */
1826 
1827 	return dev;
1828 }
1829 
1830 /* Called when VF is injecting data into network stack.
1831  * Change the associated network device from VF to netvsc.
1832  * note: already called with rcu_read_lock
1833  */
netvsc_vf_handle_frame(struct sk_buff ** pskb)1834 static rx_handler_result_t netvsc_vf_handle_frame(struct sk_buff **pskb)
1835 {
1836 	struct sk_buff *skb = *pskb;
1837 	struct net_device *ndev = rcu_dereference(skb->dev->rx_handler_data);
1838 	struct net_device_context *ndev_ctx = netdev_priv(ndev);
1839 	struct netvsc_vf_pcpu_stats *pcpu_stats
1840 		 = this_cpu_ptr(ndev_ctx->vf_stats);
1841 
1842 	skb = skb_share_check(skb, GFP_ATOMIC);
1843 	if (unlikely(!skb))
1844 		return RX_HANDLER_CONSUMED;
1845 
1846 	*pskb = skb;
1847 
1848 	skb->dev = ndev;
1849 
1850 	u64_stats_update_begin(&pcpu_stats->syncp);
1851 	pcpu_stats->rx_packets++;
1852 	pcpu_stats->rx_bytes += skb->len;
1853 	u64_stats_update_end(&pcpu_stats->syncp);
1854 
1855 	return RX_HANDLER_ANOTHER;
1856 }
1857 
netvsc_vf_join(struct net_device * vf_netdev,struct net_device * ndev)1858 static int netvsc_vf_join(struct net_device *vf_netdev,
1859 			  struct net_device *ndev)
1860 {
1861 	struct net_device_context *ndev_ctx = netdev_priv(ndev);
1862 	int ret;
1863 
1864 	ret = netdev_rx_handler_register(vf_netdev,
1865 					 netvsc_vf_handle_frame, ndev);
1866 	if (ret != 0) {
1867 		netdev_err(vf_netdev,
1868 			   "can not register netvsc VF receive handler (err = %d)\n",
1869 			   ret);
1870 		goto rx_handler_failed;
1871 	}
1872 
1873 	ret = netdev_master_upper_dev_link(vf_netdev, ndev,
1874 					   NULL, NULL);
1875 	if (ret != 0) {
1876 		netdev_err(vf_netdev,
1877 			   "can not set master device %s (err = %d)\n",
1878 			   ndev->name, ret);
1879 		goto upper_link_failed;
1880 	}
1881 
1882 	/* set slave flag before open to prevent IPv6 addrconf */
1883 	vf_netdev->flags |= IFF_SLAVE;
1884 
1885 	schedule_delayed_work(&ndev_ctx->vf_takeover, VF_TAKEOVER_INT);
1886 
1887 	call_netdevice_notifiers(NETDEV_JOIN, vf_netdev);
1888 
1889 	netdev_info(vf_netdev, "joined to %s\n", ndev->name);
1890 	return 0;
1891 
1892 upper_link_failed:
1893 	netdev_rx_handler_unregister(vf_netdev);
1894 rx_handler_failed:
1895 	return ret;
1896 }
1897 
__netvsc_vf_setup(struct net_device * ndev,struct net_device * vf_netdev)1898 static void __netvsc_vf_setup(struct net_device *ndev,
1899 			      struct net_device *vf_netdev)
1900 {
1901 	int ret;
1902 
1903 	/* Align MTU of VF with master */
1904 	ret = dev_set_mtu(vf_netdev, ndev->mtu);
1905 	if (ret)
1906 		netdev_warn(vf_netdev,
1907 			    "unable to change mtu to %u\n", ndev->mtu);
1908 
1909 	/* set multicast etc flags on VF */
1910 	dev_change_flags(vf_netdev, ndev->flags | IFF_SLAVE);
1911 
1912 	/* sync address list from ndev to VF */
1913 	netif_addr_lock_bh(ndev);
1914 	dev_uc_sync(vf_netdev, ndev);
1915 	dev_mc_sync(vf_netdev, ndev);
1916 	netif_addr_unlock_bh(ndev);
1917 
1918 	if (netif_running(ndev)) {
1919 		ret = dev_open(vf_netdev);
1920 		if (ret)
1921 			netdev_warn(vf_netdev,
1922 				    "unable to open: %d\n", ret);
1923 	}
1924 }
1925 
1926 /* Setup VF as slave of the synthetic device.
1927  * Runs in workqueue to avoid recursion in netlink callbacks.
1928  */
netvsc_vf_setup(struct work_struct * w)1929 static void netvsc_vf_setup(struct work_struct *w)
1930 {
1931 	struct net_device_context *ndev_ctx
1932 		= container_of(w, struct net_device_context, vf_takeover.work);
1933 	struct net_device *ndev = hv_get_drvdata(ndev_ctx->device_ctx);
1934 	struct net_device *vf_netdev;
1935 
1936 	if (!rtnl_trylock()) {
1937 		schedule_delayed_work(&ndev_ctx->vf_takeover, 0);
1938 		return;
1939 	}
1940 
1941 	vf_netdev = rtnl_dereference(ndev_ctx->vf_netdev);
1942 	if (vf_netdev)
1943 		__netvsc_vf_setup(ndev, vf_netdev);
1944 
1945 	rtnl_unlock();
1946 }
1947 
netvsc_register_vf(struct net_device * vf_netdev)1948 static int netvsc_register_vf(struct net_device *vf_netdev)
1949 {
1950 	struct net_device *ndev;
1951 	struct net_device_context *net_device_ctx;
1952 	struct device *pdev = vf_netdev->dev.parent;
1953 	struct netvsc_device *netvsc_dev;
1954 
1955 	if (vf_netdev->addr_len != ETH_ALEN)
1956 		return NOTIFY_DONE;
1957 
1958 	if (!pdev || !dev_is_pci(pdev) || dev_is_pf(pdev))
1959 		return NOTIFY_DONE;
1960 
1961 	/*
1962 	 * We will use the MAC address to locate the synthetic interface to
1963 	 * associate with the VF interface. If we don't find a matching
1964 	 * synthetic interface, move on.
1965 	 */
1966 	ndev = get_netvsc_bymac(vf_netdev->perm_addr);
1967 	if (!ndev)
1968 		return NOTIFY_DONE;
1969 
1970 	net_device_ctx = netdev_priv(ndev);
1971 	netvsc_dev = rtnl_dereference(net_device_ctx->nvdev);
1972 	if (!netvsc_dev || rtnl_dereference(net_device_ctx->vf_netdev))
1973 		return NOTIFY_DONE;
1974 
1975 	if (netvsc_vf_join(vf_netdev, ndev) != 0)
1976 		return NOTIFY_DONE;
1977 
1978 	netdev_info(ndev, "VF registering: %s\n", vf_netdev->name);
1979 
1980 	dev_hold(vf_netdev);
1981 	rcu_assign_pointer(net_device_ctx->vf_netdev, vf_netdev);
1982 	return NOTIFY_OK;
1983 }
1984 
1985 /* VF up/down change detected, schedule to change data path */
netvsc_vf_changed(struct net_device * vf_netdev)1986 static int netvsc_vf_changed(struct net_device *vf_netdev)
1987 {
1988 	struct net_device_context *net_device_ctx;
1989 	struct netvsc_device *netvsc_dev;
1990 	struct net_device *ndev;
1991 	bool vf_is_up = netif_running(vf_netdev);
1992 
1993 	ndev = get_netvsc_byref(vf_netdev);
1994 	if (!ndev)
1995 		return NOTIFY_DONE;
1996 
1997 	net_device_ctx = netdev_priv(ndev);
1998 	netvsc_dev = rtnl_dereference(net_device_ctx->nvdev);
1999 	if (!netvsc_dev)
2000 		return NOTIFY_DONE;
2001 
2002 	netvsc_switch_datapath(ndev, vf_is_up);
2003 	netdev_info(ndev, "Data path switched %s VF: %s\n",
2004 		    vf_is_up ? "to" : "from", vf_netdev->name);
2005 
2006 	return NOTIFY_OK;
2007 }
2008 
netvsc_unregister_vf(struct net_device * vf_netdev)2009 static int netvsc_unregister_vf(struct net_device *vf_netdev)
2010 {
2011 	struct net_device *ndev;
2012 	struct net_device_context *net_device_ctx;
2013 
2014 	ndev = get_netvsc_byref(vf_netdev);
2015 	if (!ndev)
2016 		return NOTIFY_DONE;
2017 
2018 	net_device_ctx = netdev_priv(ndev);
2019 	cancel_delayed_work_sync(&net_device_ctx->vf_takeover);
2020 
2021 	netdev_info(ndev, "VF unregistering: %s\n", vf_netdev->name);
2022 
2023 	netdev_rx_handler_unregister(vf_netdev);
2024 	netdev_upper_dev_unlink(vf_netdev, ndev);
2025 	RCU_INIT_POINTER(net_device_ctx->vf_netdev, NULL);
2026 	dev_put(vf_netdev);
2027 
2028 	return NOTIFY_OK;
2029 }
2030 
netvsc_probe(struct hv_device * dev,const struct hv_vmbus_device_id * dev_id)2031 static int netvsc_probe(struct hv_device *dev,
2032 			const struct hv_vmbus_device_id *dev_id)
2033 {
2034 	struct net_device *net = NULL;
2035 	struct net_device_context *net_device_ctx;
2036 	struct netvsc_device_info device_info;
2037 	struct netvsc_device *nvdev;
2038 	int ret = -ENOMEM;
2039 
2040 	net = alloc_etherdev_mq(sizeof(struct net_device_context),
2041 				VRSS_CHANNEL_MAX);
2042 	if (!net)
2043 		goto no_net;
2044 
2045 	netif_carrier_off(net);
2046 
2047 	netvsc_init_settings(net);
2048 
2049 	net_device_ctx = netdev_priv(net);
2050 	net_device_ctx->device_ctx = dev;
2051 	net_device_ctx->msg_enable = netif_msg_init(debug, default_msg);
2052 	if (netif_msg_probe(net_device_ctx))
2053 		netdev_dbg(net, "netvsc msg_enable: %d\n",
2054 			   net_device_ctx->msg_enable);
2055 
2056 	hv_set_drvdata(dev, net);
2057 
2058 	INIT_DELAYED_WORK(&net_device_ctx->dwork, netvsc_link_change);
2059 
2060 	spin_lock_init(&net_device_ctx->lock);
2061 	INIT_LIST_HEAD(&net_device_ctx->reconfig_events);
2062 	INIT_DELAYED_WORK(&net_device_ctx->vf_takeover, netvsc_vf_setup);
2063 
2064 	net_device_ctx->vf_stats
2065 		= netdev_alloc_pcpu_stats(struct netvsc_vf_pcpu_stats);
2066 	if (!net_device_ctx->vf_stats)
2067 		goto no_stats;
2068 
2069 	net->netdev_ops = &device_ops;
2070 	net->ethtool_ops = &ethtool_ops;
2071 	SET_NETDEV_DEV(net, &dev->device);
2072 
2073 	/* We always need headroom for rndis header */
2074 	net->needed_headroom = RNDIS_AND_PPI_SIZE;
2075 
2076 	/* Initialize the number of queues to be 1, we may change it if more
2077 	 * channels are offered later.
2078 	 */
2079 	netif_set_real_num_tx_queues(net, 1);
2080 	netif_set_real_num_rx_queues(net, 1);
2081 
2082 	/* Notify the netvsc driver of the new device */
2083 	memset(&device_info, 0, sizeof(device_info));
2084 	device_info.ring_size = ring_size;
2085 	device_info.num_chn = VRSS_CHANNEL_DEFAULT;
2086 	device_info.send_sections = NETVSC_DEFAULT_TX;
2087 	device_info.send_section_size = NETVSC_SEND_SECTION_SIZE;
2088 	device_info.recv_sections = NETVSC_DEFAULT_RX;
2089 	device_info.recv_section_size = NETVSC_RECV_SECTION_SIZE;
2090 
2091 	nvdev = rndis_filter_device_add(dev, &device_info);
2092 	if (IS_ERR(nvdev)) {
2093 		ret = PTR_ERR(nvdev);
2094 		netdev_err(net, "unable to add netvsc device (ret %d)\n", ret);
2095 		goto rndis_failed;
2096 	}
2097 
2098 	memcpy(net->dev_addr, device_info.mac_adr, ETH_ALEN);
2099 
2100 	/* We must get rtnl lock before scheduling nvdev->subchan_work,
2101 	 * otherwise netvsc_subchan_work() can get rtnl lock first and wait
2102 	 * all subchannels to show up, but that may not happen because
2103 	 * netvsc_probe() can't get rtnl lock and as a result vmbus_onoffer()
2104 	 * -> ... -> device_add() -> ... -> __device_attach() can't get
2105 	 * the device lock, so all the subchannels can't be processed --
2106 	 * finally netvsc_subchan_work() hangs for ever.
2107 	 */
2108 	rtnl_lock();
2109 
2110 	if (nvdev->num_chn > 1)
2111 		schedule_work(&nvdev->subchan_work);
2112 
2113 	/* hw_features computed in rndis_netdev_set_hwcaps() */
2114 	net->features = net->hw_features |
2115 		NETIF_F_HIGHDMA | NETIF_F_SG |
2116 		NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX;
2117 	net->vlan_features = net->features;
2118 
2119 	netdev_lockdep_set_classes(net);
2120 
2121 	/* MTU range: 68 - 1500 or 65521 */
2122 	net->min_mtu = NETVSC_MTU_MIN;
2123 	if (nvdev->nvsp_version >= NVSP_PROTOCOL_VERSION_2)
2124 		net->max_mtu = NETVSC_MTU - ETH_HLEN;
2125 	else
2126 		net->max_mtu = ETH_DATA_LEN;
2127 
2128 	ret = register_netdevice(net);
2129 	if (ret != 0) {
2130 		pr_err("Unable to register netdev.\n");
2131 		goto register_failed;
2132 	}
2133 
2134 	list_add(&net_device_ctx->list, &netvsc_dev_list);
2135 	rtnl_unlock();
2136 	return 0;
2137 
2138 register_failed:
2139 	rtnl_unlock();
2140 	rndis_filter_device_remove(dev, nvdev);
2141 rndis_failed:
2142 	free_percpu(net_device_ctx->vf_stats);
2143 no_stats:
2144 	hv_set_drvdata(dev, NULL);
2145 	free_netdev(net);
2146 no_net:
2147 	return ret;
2148 }
2149 
netvsc_remove(struct hv_device * dev)2150 static int netvsc_remove(struct hv_device *dev)
2151 {
2152 	struct net_device_context *ndev_ctx;
2153 	struct net_device *vf_netdev, *net;
2154 	struct netvsc_device *nvdev;
2155 
2156 	net = hv_get_drvdata(dev);
2157 	if (net == NULL) {
2158 		dev_err(&dev->device, "No net device to remove\n");
2159 		return 0;
2160 	}
2161 
2162 	ndev_ctx = netdev_priv(net);
2163 
2164 	cancel_delayed_work_sync(&ndev_ctx->dwork);
2165 
2166 	rtnl_lock();
2167 	nvdev = rtnl_dereference(ndev_ctx->nvdev);
2168 	if (nvdev)
2169 		cancel_work_sync(&nvdev->subchan_work);
2170 
2171 	/*
2172 	 * Call to the vsc driver to let it know that the device is being
2173 	 * removed. Also blocks mtu and channel changes.
2174 	 */
2175 	vf_netdev = rtnl_dereference(ndev_ctx->vf_netdev);
2176 	if (vf_netdev)
2177 		netvsc_unregister_vf(vf_netdev);
2178 
2179 	if (nvdev)
2180 		rndis_filter_device_remove(dev, nvdev);
2181 
2182 	unregister_netdevice(net);
2183 	list_del(&ndev_ctx->list);
2184 
2185 	rtnl_unlock();
2186 
2187 	hv_set_drvdata(dev, NULL);
2188 
2189 	free_percpu(ndev_ctx->vf_stats);
2190 	free_netdev(net);
2191 	return 0;
2192 }
2193 
2194 static const struct hv_vmbus_device_id id_table[] = {
2195 	/* Network guid */
2196 	{ HV_NIC_GUID, },
2197 	{ },
2198 };
2199 
2200 MODULE_DEVICE_TABLE(vmbus, id_table);
2201 
2202 /* The one and only one */
2203 static struct  hv_driver netvsc_drv = {
2204 	.name = KBUILD_MODNAME,
2205 	.id_table = id_table,
2206 	.probe = netvsc_probe,
2207 	.remove = netvsc_remove,
2208 };
2209 
2210 /*
2211  * On Hyper-V, every VF interface is matched with a corresponding
2212  * synthetic interface. The synthetic interface is presented first
2213  * to the guest. When the corresponding VF instance is registered,
2214  * we will take care of switching the data path.
2215  */
netvsc_netdev_event(struct notifier_block * this,unsigned long event,void * ptr)2216 static int netvsc_netdev_event(struct notifier_block *this,
2217 			       unsigned long event, void *ptr)
2218 {
2219 	struct net_device *event_dev = netdev_notifier_info_to_dev(ptr);
2220 
2221 	/* Skip our own events */
2222 	if (event_dev->netdev_ops == &device_ops)
2223 		return NOTIFY_DONE;
2224 
2225 	/* Avoid non-Ethernet type devices */
2226 	if (event_dev->type != ARPHRD_ETHER)
2227 		return NOTIFY_DONE;
2228 
2229 	/* Avoid Vlan dev with same MAC registering as VF */
2230 	if (is_vlan_dev(event_dev))
2231 		return NOTIFY_DONE;
2232 
2233 	/* Avoid Bonding master dev with same MAC registering as VF */
2234 	if ((event_dev->priv_flags & IFF_BONDING) &&
2235 	    (event_dev->flags & IFF_MASTER))
2236 		return NOTIFY_DONE;
2237 
2238 	switch (event) {
2239 	case NETDEV_REGISTER:
2240 		return netvsc_register_vf(event_dev);
2241 	case NETDEV_UNREGISTER:
2242 		return netvsc_unregister_vf(event_dev);
2243 	case NETDEV_UP:
2244 	case NETDEV_DOWN:
2245 		return netvsc_vf_changed(event_dev);
2246 	default:
2247 		return NOTIFY_DONE;
2248 	}
2249 }
2250 
2251 static struct notifier_block netvsc_netdev_notifier = {
2252 	.notifier_call = netvsc_netdev_event,
2253 };
2254 
netvsc_drv_exit(void)2255 static void __exit netvsc_drv_exit(void)
2256 {
2257 	unregister_netdevice_notifier(&netvsc_netdev_notifier);
2258 	vmbus_driver_unregister(&netvsc_drv);
2259 }
2260 
netvsc_drv_init(void)2261 static int __init netvsc_drv_init(void)
2262 {
2263 	int ret;
2264 
2265 	if (ring_size < RING_SIZE_MIN) {
2266 		ring_size = RING_SIZE_MIN;
2267 		pr_info("Increased ring_size to %d (min allowed)\n",
2268 			ring_size);
2269 	}
2270 	ret = vmbus_driver_register(&netvsc_drv);
2271 
2272 	if (ret)
2273 		return ret;
2274 
2275 	register_netdevice_notifier(&netvsc_netdev_notifier);
2276 	return 0;
2277 }
2278 
2279 MODULE_LICENSE("GPL");
2280 MODULE_DESCRIPTION("Microsoft Hyper-V network driver");
2281 
2282 module_init(netvsc_drv_init);
2283 module_exit(netvsc_drv_exit);
2284