• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (C) 2009 Red Hat, Inc.
3  * Author: Michael S. Tsirkin <mst@redhat.com>
4  *
5  * virtio-net server in host kernel.
6  */
7 
8 #include <linux/compat.h>
9 #include <linux/eventfd.h>
10 #include <linux/vhost.h>
11 #include <linux/virtio_net.h>
12 #include <linux/miscdevice.h>
13 #include <linux/module.h>
14 #include <linux/moduleparam.h>
15 #include <linux/mutex.h>
16 #include <linux/workqueue.h>
17 #include <linux/file.h>
18 #include <linux/slab.h>
19 #include <linux/sched/clock.h>
20 #include <linux/sched/signal.h>
21 #include <linux/vmalloc.h>
22 
23 #include <linux/net.h>
24 #include <linux/if_packet.h>
25 #include <linux/if_arp.h>
26 #include <linux/if_tun.h>
27 #include <linux/if_macvlan.h>
28 #include <linux/if_tap.h>
29 #include <linux/if_vlan.h>
30 #include <linux/skb_array.h>
31 #include <linux/skbuff.h>
32 
33 #include <net/sock.h>
34 #include <net/xdp.h>
35 
36 #include "vhost.h"
37 
38 static int experimental_zcopytx = 0;
39 module_param(experimental_zcopytx, int, 0444);
40 MODULE_PARM_DESC(experimental_zcopytx, "Enable Zero Copy TX;"
41 		                       " 1 -Enable; 0 - Disable");
42 
43 /* Max number of bytes transferred before requeueing the job.
44  * Using this limit prevents one virtqueue from starving others. */
45 #define VHOST_NET_WEIGHT 0x80000
46 
47 /* Max number of packets transferred before requeueing the job.
48  * Using this limit prevents one virtqueue from starving others with small
49  * pkts.
50  */
51 #define VHOST_NET_PKT_WEIGHT 256
52 
53 /* MAX number of TX used buffers for outstanding zerocopy */
54 #define VHOST_MAX_PEND 128
55 #define VHOST_GOODCOPY_LEN 256
56 
57 /*
58  * For transmit, used buffer len is unused; we override it to track buffer
59  * status internally; used for zerocopy tx only.
60  */
61 /* Lower device DMA failed */
62 #define VHOST_DMA_FAILED_LEN	((__force __virtio32)3)
63 /* Lower device DMA done */
64 #define VHOST_DMA_DONE_LEN	((__force __virtio32)2)
65 /* Lower device DMA in progress */
66 #define VHOST_DMA_IN_PROGRESS	((__force __virtio32)1)
67 /* Buffer unused */
68 #define VHOST_DMA_CLEAR_LEN	((__force __virtio32)0)
69 
70 #define VHOST_DMA_IS_DONE(len) ((__force u32)(len) >= (__force u32)VHOST_DMA_DONE_LEN)
71 
72 enum {
73 	VHOST_NET_FEATURES = VHOST_FEATURES |
74 			 (1ULL << VHOST_NET_F_VIRTIO_NET_HDR) |
75 			 (1ULL << VIRTIO_NET_F_MRG_RXBUF) |
76 			 (1ULL << VIRTIO_F_ACCESS_PLATFORM)
77 };
78 
79 enum {
80 	VHOST_NET_BACKEND_FEATURES = (1ULL << VHOST_BACKEND_F_IOTLB_MSG_V2)
81 };
82 
83 enum {
84 	VHOST_NET_VQ_RX = 0,
85 	VHOST_NET_VQ_TX = 1,
86 	VHOST_NET_VQ_MAX = 2,
87 };
88 
89 struct vhost_net_ubuf_ref {
90 	/* refcount follows semantics similar to kref:
91 	 *  0: object is released
92 	 *  1: no outstanding ubufs
93 	 * >1: outstanding ubufs
94 	 */
95 	atomic_t refcount;
96 	wait_queue_head_t wait;
97 	struct vhost_virtqueue *vq;
98 };
99 
100 #define VHOST_NET_BATCH 64
101 struct vhost_net_buf {
102 	void **queue;
103 	int tail;
104 	int head;
105 };
106 
107 struct vhost_net_virtqueue {
108 	struct vhost_virtqueue vq;
109 	size_t vhost_hlen;
110 	size_t sock_hlen;
111 	/* vhost zerocopy support fields below: */
112 	/* last used idx for outstanding DMA zerocopy buffers */
113 	int upend_idx;
114 	/* For TX, first used idx for DMA done zerocopy buffers
115 	 * For RX, number of batched heads
116 	 */
117 	int done_idx;
118 	/* Number of XDP frames batched */
119 	int batched_xdp;
120 	/* an array of userspace buffers info */
121 	struct ubuf_info *ubuf_info;
122 	/* Reference counting for outstanding ubufs.
123 	 * Protected by vq mutex. Writers must also take device mutex. */
124 	struct vhost_net_ubuf_ref *ubufs;
125 	struct ptr_ring *rx_ring;
126 	struct vhost_net_buf rxq;
127 	/* Batched XDP buffs */
128 	struct xdp_buff *xdp;
129 };
130 
131 struct vhost_net {
132 	struct vhost_dev dev;
133 	struct vhost_net_virtqueue vqs[VHOST_NET_VQ_MAX];
134 	struct vhost_poll poll[VHOST_NET_VQ_MAX];
135 	/* Number of TX recently submitted.
136 	 * Protected by tx vq lock. */
137 	unsigned tx_packets;
138 	/* Number of times zerocopy TX recently failed.
139 	 * Protected by tx vq lock. */
140 	unsigned tx_zcopy_err;
141 	/* Flush in progress. Protected by tx vq lock. */
142 	bool tx_flush;
143 	/* Private page frag */
144 	struct page_frag page_frag;
145 	/* Refcount bias of page frag */
146 	int refcnt_bias;
147 };
148 
149 static unsigned vhost_net_zcopy_mask __read_mostly;
150 
vhost_net_buf_get_ptr(struct vhost_net_buf * rxq)151 static void *vhost_net_buf_get_ptr(struct vhost_net_buf *rxq)
152 {
153 	if (rxq->tail != rxq->head)
154 		return rxq->queue[rxq->head];
155 	else
156 		return NULL;
157 }
158 
vhost_net_buf_get_size(struct vhost_net_buf * rxq)159 static int vhost_net_buf_get_size(struct vhost_net_buf *rxq)
160 {
161 	return rxq->tail - rxq->head;
162 }
163 
vhost_net_buf_is_empty(struct vhost_net_buf * rxq)164 static int vhost_net_buf_is_empty(struct vhost_net_buf *rxq)
165 {
166 	return rxq->tail == rxq->head;
167 }
168 
vhost_net_buf_consume(struct vhost_net_buf * rxq)169 static void *vhost_net_buf_consume(struct vhost_net_buf *rxq)
170 {
171 	void *ret = vhost_net_buf_get_ptr(rxq);
172 	++rxq->head;
173 	return ret;
174 }
175 
vhost_net_buf_produce(struct vhost_net_virtqueue * nvq)176 static int vhost_net_buf_produce(struct vhost_net_virtqueue *nvq)
177 {
178 	struct vhost_net_buf *rxq = &nvq->rxq;
179 
180 	rxq->head = 0;
181 	rxq->tail = ptr_ring_consume_batched(nvq->rx_ring, rxq->queue,
182 					      VHOST_NET_BATCH);
183 	return rxq->tail;
184 }
185 
vhost_net_buf_unproduce(struct vhost_net_virtqueue * nvq)186 static void vhost_net_buf_unproduce(struct vhost_net_virtqueue *nvq)
187 {
188 	struct vhost_net_buf *rxq = &nvq->rxq;
189 
190 	if (nvq->rx_ring && !vhost_net_buf_is_empty(rxq)) {
191 		ptr_ring_unconsume(nvq->rx_ring, rxq->queue + rxq->head,
192 				   vhost_net_buf_get_size(rxq),
193 				   tun_ptr_free);
194 		rxq->head = rxq->tail = 0;
195 	}
196 }
197 
vhost_net_buf_peek_len(void * ptr)198 static int vhost_net_buf_peek_len(void *ptr)
199 {
200 	if (tun_is_xdp_frame(ptr)) {
201 		struct xdp_frame *xdpf = tun_ptr_to_xdp(ptr);
202 
203 		return xdpf->len;
204 	}
205 
206 	return __skb_array_len_with_tag(ptr);
207 }
208 
vhost_net_buf_peek(struct vhost_net_virtqueue * nvq)209 static int vhost_net_buf_peek(struct vhost_net_virtqueue *nvq)
210 {
211 	struct vhost_net_buf *rxq = &nvq->rxq;
212 
213 	if (!vhost_net_buf_is_empty(rxq))
214 		goto out;
215 
216 	if (!vhost_net_buf_produce(nvq))
217 		return 0;
218 
219 out:
220 	return vhost_net_buf_peek_len(vhost_net_buf_get_ptr(rxq));
221 }
222 
vhost_net_buf_init(struct vhost_net_buf * rxq)223 static void vhost_net_buf_init(struct vhost_net_buf *rxq)
224 {
225 	rxq->head = rxq->tail = 0;
226 }
227 
vhost_net_enable_zcopy(int vq)228 static void vhost_net_enable_zcopy(int vq)
229 {
230 	vhost_net_zcopy_mask |= 0x1 << vq;
231 }
232 
233 static struct vhost_net_ubuf_ref *
vhost_net_ubuf_alloc(struct vhost_virtqueue * vq,bool zcopy)234 vhost_net_ubuf_alloc(struct vhost_virtqueue *vq, bool zcopy)
235 {
236 	struct vhost_net_ubuf_ref *ubufs;
237 	/* No zero copy backend? Nothing to count. */
238 	if (!zcopy)
239 		return NULL;
240 	ubufs = kmalloc(sizeof(*ubufs), GFP_KERNEL);
241 	if (!ubufs)
242 		return ERR_PTR(-ENOMEM);
243 	atomic_set(&ubufs->refcount, 1);
244 	init_waitqueue_head(&ubufs->wait);
245 	ubufs->vq = vq;
246 	return ubufs;
247 }
248 
vhost_net_ubuf_put(struct vhost_net_ubuf_ref * ubufs)249 static int vhost_net_ubuf_put(struct vhost_net_ubuf_ref *ubufs)
250 {
251 	int r = atomic_sub_return(1, &ubufs->refcount);
252 	if (unlikely(!r))
253 		wake_up(&ubufs->wait);
254 	return r;
255 }
256 
vhost_net_ubuf_put_and_wait(struct vhost_net_ubuf_ref * ubufs)257 static void vhost_net_ubuf_put_and_wait(struct vhost_net_ubuf_ref *ubufs)
258 {
259 	vhost_net_ubuf_put(ubufs);
260 	wait_event(ubufs->wait, !atomic_read(&ubufs->refcount));
261 }
262 
vhost_net_ubuf_put_wait_and_free(struct vhost_net_ubuf_ref * ubufs)263 static void vhost_net_ubuf_put_wait_and_free(struct vhost_net_ubuf_ref *ubufs)
264 {
265 	vhost_net_ubuf_put_and_wait(ubufs);
266 	kfree(ubufs);
267 }
268 
vhost_net_clear_ubuf_info(struct vhost_net * n)269 static void vhost_net_clear_ubuf_info(struct vhost_net *n)
270 {
271 	int i;
272 
273 	for (i = 0; i < VHOST_NET_VQ_MAX; ++i) {
274 		kfree(n->vqs[i].ubuf_info);
275 		n->vqs[i].ubuf_info = NULL;
276 	}
277 }
278 
vhost_net_set_ubuf_info(struct vhost_net * n)279 static int vhost_net_set_ubuf_info(struct vhost_net *n)
280 {
281 	bool zcopy;
282 	int i;
283 
284 	for (i = 0; i < VHOST_NET_VQ_MAX; ++i) {
285 		zcopy = vhost_net_zcopy_mask & (0x1 << i);
286 		if (!zcopy)
287 			continue;
288 		n->vqs[i].ubuf_info =
289 			kmalloc_array(UIO_MAXIOV,
290 				      sizeof(*n->vqs[i].ubuf_info),
291 				      GFP_KERNEL);
292 		if  (!n->vqs[i].ubuf_info)
293 			goto err;
294 	}
295 	return 0;
296 
297 err:
298 	vhost_net_clear_ubuf_info(n);
299 	return -ENOMEM;
300 }
301 
vhost_net_vq_reset(struct vhost_net * n)302 static void vhost_net_vq_reset(struct vhost_net *n)
303 {
304 	int i;
305 
306 	vhost_net_clear_ubuf_info(n);
307 
308 	for (i = 0; i < VHOST_NET_VQ_MAX; i++) {
309 		n->vqs[i].done_idx = 0;
310 		n->vqs[i].upend_idx = 0;
311 		n->vqs[i].ubufs = NULL;
312 		n->vqs[i].vhost_hlen = 0;
313 		n->vqs[i].sock_hlen = 0;
314 		vhost_net_buf_init(&n->vqs[i].rxq);
315 	}
316 
317 }
318 
vhost_net_tx_packet(struct vhost_net * net)319 static void vhost_net_tx_packet(struct vhost_net *net)
320 {
321 	++net->tx_packets;
322 	if (net->tx_packets < 1024)
323 		return;
324 	net->tx_packets = 0;
325 	net->tx_zcopy_err = 0;
326 }
327 
vhost_net_tx_err(struct vhost_net * net)328 static void vhost_net_tx_err(struct vhost_net *net)
329 {
330 	++net->tx_zcopy_err;
331 }
332 
vhost_net_tx_select_zcopy(struct vhost_net * net)333 static bool vhost_net_tx_select_zcopy(struct vhost_net *net)
334 {
335 	/* TX flush waits for outstanding DMAs to be done.
336 	 * Don't start new DMAs.
337 	 */
338 	return !net->tx_flush &&
339 		net->tx_packets / 64 >= net->tx_zcopy_err;
340 }
341 
vhost_sock_zcopy(struct socket * sock)342 static bool vhost_sock_zcopy(struct socket *sock)
343 {
344 	return unlikely(experimental_zcopytx) &&
345 		sock_flag(sock->sk, SOCK_ZEROCOPY);
346 }
347 
vhost_sock_xdp(struct socket * sock)348 static bool vhost_sock_xdp(struct socket *sock)
349 {
350 	return sock_flag(sock->sk, SOCK_XDP);
351 }
352 
353 /* In case of DMA done not in order in lower device driver for some reason.
354  * upend_idx is used to track end of used idx, done_idx is used to track head
355  * of used idx. Once lower device DMA done contiguously, we will signal KVM
356  * guest used idx.
357  */
vhost_zerocopy_signal_used(struct vhost_net * net,struct vhost_virtqueue * vq)358 static void vhost_zerocopy_signal_used(struct vhost_net *net,
359 				       struct vhost_virtqueue *vq)
360 {
361 	struct vhost_net_virtqueue *nvq =
362 		container_of(vq, struct vhost_net_virtqueue, vq);
363 	int i, add;
364 	int j = 0;
365 
366 	for (i = nvq->done_idx; i != nvq->upend_idx; i = (i + 1) % UIO_MAXIOV) {
367 		if (vq->heads[i].len == VHOST_DMA_FAILED_LEN)
368 			vhost_net_tx_err(net);
369 		if (VHOST_DMA_IS_DONE(vq->heads[i].len)) {
370 			vq->heads[i].len = VHOST_DMA_CLEAR_LEN;
371 			++j;
372 		} else
373 			break;
374 	}
375 	while (j) {
376 		add = min(UIO_MAXIOV - nvq->done_idx, j);
377 		vhost_add_used_and_signal_n(vq->dev, vq,
378 					    &vq->heads[nvq->done_idx], add);
379 		nvq->done_idx = (nvq->done_idx + add) % UIO_MAXIOV;
380 		j -= add;
381 	}
382 }
383 
vhost_zerocopy_callback(struct sk_buff * skb,struct ubuf_info * ubuf,bool success)384 static void vhost_zerocopy_callback(struct sk_buff *skb,
385 				    struct ubuf_info *ubuf, bool success)
386 {
387 	struct vhost_net_ubuf_ref *ubufs = ubuf->ctx;
388 	struct vhost_virtqueue *vq = ubufs->vq;
389 	int cnt;
390 
391 	rcu_read_lock_bh();
392 
393 	/* set len to mark this desc buffers done DMA */
394 	vq->heads[ubuf->desc].len = success ?
395 		VHOST_DMA_DONE_LEN : VHOST_DMA_FAILED_LEN;
396 	cnt = vhost_net_ubuf_put(ubufs);
397 
398 	/*
399 	 * Trigger polling thread if guest stopped submitting new buffers:
400 	 * in this case, the refcount after decrement will eventually reach 1.
401 	 * We also trigger polling periodically after each 16 packets
402 	 * (the value 16 here is more or less arbitrary, it's tuned to trigger
403 	 * less than 10% of times).
404 	 */
405 	if (cnt <= 1 || !(cnt % 16))
406 		vhost_poll_queue(&vq->poll);
407 
408 	rcu_read_unlock_bh();
409 }
410 
busy_clock(void)411 static inline unsigned long busy_clock(void)
412 {
413 	return local_clock() >> 10;
414 }
415 
vhost_can_busy_poll(unsigned long endtime)416 static bool vhost_can_busy_poll(unsigned long endtime)
417 {
418 	return likely(!need_resched() && !time_after(busy_clock(), endtime) &&
419 		      !signal_pending(current));
420 }
421 
vhost_net_disable_vq(struct vhost_net * n,struct vhost_virtqueue * vq)422 static void vhost_net_disable_vq(struct vhost_net *n,
423 				 struct vhost_virtqueue *vq)
424 {
425 	struct vhost_net_virtqueue *nvq =
426 		container_of(vq, struct vhost_net_virtqueue, vq);
427 	struct vhost_poll *poll = n->poll + (nvq - n->vqs);
428 	if (!vhost_vq_get_backend(vq))
429 		return;
430 	vhost_poll_stop(poll);
431 }
432 
vhost_net_enable_vq(struct vhost_net * n,struct vhost_virtqueue * vq)433 static int vhost_net_enable_vq(struct vhost_net *n,
434 				struct vhost_virtqueue *vq)
435 {
436 	struct vhost_net_virtqueue *nvq =
437 		container_of(vq, struct vhost_net_virtqueue, vq);
438 	struct vhost_poll *poll = n->poll + (nvq - n->vqs);
439 	struct socket *sock;
440 
441 	sock = vhost_vq_get_backend(vq);
442 	if (!sock)
443 		return 0;
444 
445 	return vhost_poll_start(poll, sock->file);
446 }
447 
vhost_net_signal_used(struct vhost_net_virtqueue * nvq)448 static void vhost_net_signal_used(struct vhost_net_virtqueue *nvq)
449 {
450 	struct vhost_virtqueue *vq = &nvq->vq;
451 	struct vhost_dev *dev = vq->dev;
452 
453 	if (!nvq->done_idx)
454 		return;
455 
456 	vhost_add_used_and_signal_n(dev, vq, vq->heads, nvq->done_idx);
457 	nvq->done_idx = 0;
458 }
459 
vhost_tx_batch(struct vhost_net * net,struct vhost_net_virtqueue * nvq,struct socket * sock,struct msghdr * msghdr)460 static void vhost_tx_batch(struct vhost_net *net,
461 			   struct vhost_net_virtqueue *nvq,
462 			   struct socket *sock,
463 			   struct msghdr *msghdr)
464 {
465 	struct tun_msg_ctl ctl = {
466 		.type = TUN_MSG_PTR,
467 		.num = nvq->batched_xdp,
468 		.ptr = nvq->xdp,
469 	};
470 	int i, err;
471 
472 	if (nvq->batched_xdp == 0)
473 		goto signal_used;
474 
475 	msghdr->msg_control = &ctl;
476 	msghdr->msg_controllen = sizeof(ctl);
477 	err = sock->ops->sendmsg(sock, msghdr, 0);
478 	if (unlikely(err < 0)) {
479 		vq_err(&nvq->vq, "Fail to batch sending packets\n");
480 
481 		/* free pages owned by XDP; since this is an unlikely error path,
482 		 * keep it simple and avoid more complex bulk update for the
483 		 * used pages
484 		 */
485 		for (i = 0; i < nvq->batched_xdp; ++i)
486 			put_page(virt_to_head_page(nvq->xdp[i].data));
487 		nvq->batched_xdp = 0;
488 		nvq->done_idx = 0;
489 		return;
490 	}
491 
492 signal_used:
493 	vhost_net_signal_used(nvq);
494 	nvq->batched_xdp = 0;
495 }
496 
sock_has_rx_data(struct socket * sock)497 static int sock_has_rx_data(struct socket *sock)
498 {
499 	if (unlikely(!sock))
500 		return 0;
501 
502 	if (sock->ops->peek_len)
503 		return sock->ops->peek_len(sock);
504 
505 	return skb_queue_empty(&sock->sk->sk_receive_queue);
506 }
507 
vhost_net_busy_poll_try_queue(struct vhost_net * net,struct vhost_virtqueue * vq)508 static void vhost_net_busy_poll_try_queue(struct vhost_net *net,
509 					  struct vhost_virtqueue *vq)
510 {
511 	if (!vhost_vq_avail_empty(&net->dev, vq)) {
512 		vhost_poll_queue(&vq->poll);
513 	} else if (unlikely(vhost_enable_notify(&net->dev, vq))) {
514 		vhost_disable_notify(&net->dev, vq);
515 		vhost_poll_queue(&vq->poll);
516 	}
517 }
518 
vhost_net_busy_poll(struct vhost_net * net,struct vhost_virtqueue * rvq,struct vhost_virtqueue * tvq,bool * busyloop_intr,bool poll_rx)519 static void vhost_net_busy_poll(struct vhost_net *net,
520 				struct vhost_virtqueue *rvq,
521 				struct vhost_virtqueue *tvq,
522 				bool *busyloop_intr,
523 				bool poll_rx)
524 {
525 	unsigned long busyloop_timeout;
526 	unsigned long endtime;
527 	struct socket *sock;
528 	struct vhost_virtqueue *vq = poll_rx ? tvq : rvq;
529 
530 	/* Try to hold the vq mutex of the paired virtqueue. We can't
531 	 * use mutex_lock() here since we could not guarantee a
532 	 * consistenet lock ordering.
533 	 */
534 	if (!mutex_trylock(&vq->mutex))
535 		return;
536 
537 	vhost_disable_notify(&net->dev, vq);
538 	sock = vhost_vq_get_backend(rvq);
539 
540 	busyloop_timeout = poll_rx ? rvq->busyloop_timeout:
541 				     tvq->busyloop_timeout;
542 
543 	preempt_disable();
544 	endtime = busy_clock() + busyloop_timeout;
545 
546 	while (vhost_can_busy_poll(endtime)) {
547 		if (vhost_has_work(&net->dev)) {
548 			*busyloop_intr = true;
549 			break;
550 		}
551 
552 		if ((sock_has_rx_data(sock) &&
553 		     !vhost_vq_avail_empty(&net->dev, rvq)) ||
554 		    !vhost_vq_avail_empty(&net->dev, tvq))
555 			break;
556 
557 		cpu_relax();
558 	}
559 
560 	preempt_enable();
561 
562 	if (poll_rx || sock_has_rx_data(sock))
563 		vhost_net_busy_poll_try_queue(net, vq);
564 	else if (!poll_rx) /* On tx here, sock has no rx data. */
565 		vhost_enable_notify(&net->dev, rvq);
566 
567 	mutex_unlock(&vq->mutex);
568 }
569 
vhost_net_tx_get_vq_desc(struct vhost_net * net,struct vhost_net_virtqueue * tnvq,unsigned int * out_num,unsigned int * in_num,struct msghdr * msghdr,bool * busyloop_intr)570 static int vhost_net_tx_get_vq_desc(struct vhost_net *net,
571 				    struct vhost_net_virtqueue *tnvq,
572 				    unsigned int *out_num, unsigned int *in_num,
573 				    struct msghdr *msghdr, bool *busyloop_intr)
574 {
575 	struct vhost_net_virtqueue *rnvq = &net->vqs[VHOST_NET_VQ_RX];
576 	struct vhost_virtqueue *rvq = &rnvq->vq;
577 	struct vhost_virtqueue *tvq = &tnvq->vq;
578 
579 	int r = vhost_get_vq_desc(tvq, tvq->iov, ARRAY_SIZE(tvq->iov),
580 				  out_num, in_num, NULL, NULL);
581 
582 	if (r == tvq->num && tvq->busyloop_timeout) {
583 		/* Flush batched packets first */
584 		if (!vhost_sock_zcopy(vhost_vq_get_backend(tvq)))
585 			vhost_tx_batch(net, tnvq,
586 				       vhost_vq_get_backend(tvq),
587 				       msghdr);
588 
589 		vhost_net_busy_poll(net, rvq, tvq, busyloop_intr, false);
590 
591 		r = vhost_get_vq_desc(tvq, tvq->iov, ARRAY_SIZE(tvq->iov),
592 				      out_num, in_num, NULL, NULL);
593 	}
594 
595 	return r;
596 }
597 
vhost_exceeds_maxpend(struct vhost_net * net)598 static bool vhost_exceeds_maxpend(struct vhost_net *net)
599 {
600 	struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_TX];
601 	struct vhost_virtqueue *vq = &nvq->vq;
602 
603 	return (nvq->upend_idx + UIO_MAXIOV - nvq->done_idx) % UIO_MAXIOV >
604 	       min_t(unsigned int, VHOST_MAX_PEND, vq->num >> 2);
605 }
606 
init_iov_iter(struct vhost_virtqueue * vq,struct iov_iter * iter,size_t hdr_size,int out)607 static size_t init_iov_iter(struct vhost_virtqueue *vq, struct iov_iter *iter,
608 			    size_t hdr_size, int out)
609 {
610 	/* Skip header. TODO: support TSO. */
611 	size_t len = iov_length(vq->iov, out);
612 
613 	iov_iter_init(iter, WRITE, vq->iov, out, len);
614 	iov_iter_advance(iter, hdr_size);
615 
616 	return iov_iter_count(iter);
617 }
618 
get_tx_bufs(struct vhost_net * net,struct vhost_net_virtqueue * nvq,struct msghdr * msg,unsigned int * out,unsigned int * in,size_t * len,bool * busyloop_intr)619 static int get_tx_bufs(struct vhost_net *net,
620 		       struct vhost_net_virtqueue *nvq,
621 		       struct msghdr *msg,
622 		       unsigned int *out, unsigned int *in,
623 		       size_t *len, bool *busyloop_intr)
624 {
625 	struct vhost_virtqueue *vq = &nvq->vq;
626 	int ret;
627 
628 	ret = vhost_net_tx_get_vq_desc(net, nvq, out, in, msg, busyloop_intr);
629 
630 	if (ret < 0 || ret == vq->num)
631 		return ret;
632 
633 	if (*in) {
634 		vq_err(vq, "Unexpected descriptor format for TX: out %d, int %d\n",
635 			*out, *in);
636 		return -EFAULT;
637 	}
638 
639 	/* Sanity check */
640 	*len = init_iov_iter(vq, &msg->msg_iter, nvq->vhost_hlen, *out);
641 	if (*len == 0) {
642 		vq_err(vq, "Unexpected header len for TX: %zd expected %zd\n",
643 			*len, nvq->vhost_hlen);
644 		return -EFAULT;
645 	}
646 
647 	return ret;
648 }
649 
tx_can_batch(struct vhost_virtqueue * vq,size_t total_len)650 static bool tx_can_batch(struct vhost_virtqueue *vq, size_t total_len)
651 {
652 	return total_len < VHOST_NET_WEIGHT &&
653 	       !vhost_vq_avail_empty(vq->dev, vq);
654 }
655 
vhost_net_page_frag_refill(struct vhost_net * net,unsigned int sz,struct page_frag * pfrag,gfp_t gfp)656 static bool vhost_net_page_frag_refill(struct vhost_net *net, unsigned int sz,
657 				       struct page_frag *pfrag, gfp_t gfp)
658 {
659 	if (pfrag->page) {
660 		if (pfrag->offset + sz <= pfrag->size)
661 			return true;
662 		__page_frag_cache_drain(pfrag->page, net->refcnt_bias);
663 	}
664 
665 	pfrag->offset = 0;
666 	net->refcnt_bias = 0;
667 	if (SKB_FRAG_PAGE_ORDER) {
668 		/* Avoid direct reclaim but allow kswapd to wake */
669 		pfrag->page = alloc_pages((gfp & ~__GFP_DIRECT_RECLAIM) |
670 					  __GFP_COMP | __GFP_NOWARN |
671 					  __GFP_NORETRY,
672 					  SKB_FRAG_PAGE_ORDER);
673 		if (likely(pfrag->page)) {
674 			pfrag->size = PAGE_SIZE << SKB_FRAG_PAGE_ORDER;
675 			goto done;
676 		}
677 	}
678 	pfrag->page = alloc_page(gfp);
679 	if (likely(pfrag->page)) {
680 		pfrag->size = PAGE_SIZE;
681 		goto done;
682 	}
683 	return false;
684 
685 done:
686 	net->refcnt_bias = USHRT_MAX;
687 	page_ref_add(pfrag->page, USHRT_MAX - 1);
688 	return true;
689 }
690 
691 #define VHOST_NET_RX_PAD (NET_IP_ALIGN + NET_SKB_PAD)
692 
vhost_net_build_xdp(struct vhost_net_virtqueue * nvq,struct iov_iter * from)693 static int vhost_net_build_xdp(struct vhost_net_virtqueue *nvq,
694 			       struct iov_iter *from)
695 {
696 	struct vhost_virtqueue *vq = &nvq->vq;
697 	struct vhost_net *net = container_of(vq->dev, struct vhost_net,
698 					     dev);
699 	struct socket *sock = vhost_vq_get_backend(vq);
700 	struct page_frag *alloc_frag = &net->page_frag;
701 	struct virtio_net_hdr *gso;
702 	struct xdp_buff *xdp = &nvq->xdp[nvq->batched_xdp];
703 	struct tun_xdp_hdr *hdr;
704 	size_t len = iov_iter_count(from);
705 	int headroom = vhost_sock_xdp(sock) ? XDP_PACKET_HEADROOM : 0;
706 	int buflen = SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
707 	int pad = SKB_DATA_ALIGN(VHOST_NET_RX_PAD + headroom + nvq->sock_hlen);
708 	int sock_hlen = nvq->sock_hlen;
709 	void *buf;
710 	int copied;
711 
712 	if (unlikely(len < nvq->sock_hlen))
713 		return -EFAULT;
714 
715 	if (SKB_DATA_ALIGN(len + pad) +
716 	    SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) > PAGE_SIZE)
717 		return -ENOSPC;
718 
719 	buflen += SKB_DATA_ALIGN(len + pad);
720 	alloc_frag->offset = ALIGN((u64)alloc_frag->offset, SMP_CACHE_BYTES);
721 	if (unlikely(!vhost_net_page_frag_refill(net, buflen,
722 						 alloc_frag, GFP_KERNEL)))
723 		return -ENOMEM;
724 
725 	buf = (char *)page_address(alloc_frag->page) + alloc_frag->offset;
726 	copied = copy_page_from_iter(alloc_frag->page,
727 				     alloc_frag->offset +
728 				     offsetof(struct tun_xdp_hdr, gso),
729 				     sock_hlen, from);
730 	if (copied != sock_hlen)
731 		return -EFAULT;
732 
733 	hdr = buf;
734 	gso = &hdr->gso;
735 
736 	if ((gso->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) &&
737 	    vhost16_to_cpu(vq, gso->csum_start) +
738 	    vhost16_to_cpu(vq, gso->csum_offset) + 2 >
739 	    vhost16_to_cpu(vq, gso->hdr_len)) {
740 		gso->hdr_len = cpu_to_vhost16(vq,
741 			       vhost16_to_cpu(vq, gso->csum_start) +
742 			       vhost16_to_cpu(vq, gso->csum_offset) + 2);
743 
744 		if (vhost16_to_cpu(vq, gso->hdr_len) > len)
745 			return -EINVAL;
746 	}
747 
748 	len -= sock_hlen;
749 	copied = copy_page_from_iter(alloc_frag->page,
750 				     alloc_frag->offset + pad,
751 				     len, from);
752 	if (copied != len)
753 		return -EFAULT;
754 
755 	xdp_init_buff(xdp, buflen, NULL);
756 	xdp_prepare_buff(xdp, buf, pad, len, true);
757 	hdr->buflen = buflen;
758 
759 	--net->refcnt_bias;
760 	alloc_frag->offset += buflen;
761 
762 	++nvq->batched_xdp;
763 
764 	return 0;
765 }
766 
handle_tx_copy(struct vhost_net * net,struct socket * sock)767 static void handle_tx_copy(struct vhost_net *net, struct socket *sock)
768 {
769 	struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_TX];
770 	struct vhost_virtqueue *vq = &nvq->vq;
771 	unsigned out, in;
772 	int head;
773 	struct msghdr msg = {
774 		.msg_name = NULL,
775 		.msg_namelen = 0,
776 		.msg_control = NULL,
777 		.msg_controllen = 0,
778 		.msg_flags = MSG_DONTWAIT,
779 	};
780 	size_t len, total_len = 0;
781 	int err;
782 	int sent_pkts = 0;
783 	bool sock_can_batch = (sock->sk->sk_sndbuf == INT_MAX);
784 
785 	do {
786 		bool busyloop_intr = false;
787 
788 		if (nvq->done_idx == VHOST_NET_BATCH)
789 			vhost_tx_batch(net, nvq, sock, &msg);
790 
791 		head = get_tx_bufs(net, nvq, &msg, &out, &in, &len,
792 				   &busyloop_intr);
793 		/* On error, stop handling until the next kick. */
794 		if (unlikely(head < 0))
795 			break;
796 		/* Nothing new?  Wait for eventfd to tell us they refilled. */
797 		if (head == vq->num) {
798 			if (unlikely(busyloop_intr)) {
799 				vhost_poll_queue(&vq->poll);
800 			} else if (unlikely(vhost_enable_notify(&net->dev,
801 								vq))) {
802 				vhost_disable_notify(&net->dev, vq);
803 				continue;
804 			}
805 			break;
806 		}
807 
808 		total_len += len;
809 
810 		/* For simplicity, TX batching is only enabled if
811 		 * sndbuf is unlimited.
812 		 */
813 		if (sock_can_batch) {
814 			err = vhost_net_build_xdp(nvq, &msg.msg_iter);
815 			if (!err) {
816 				goto done;
817 			} else if (unlikely(err != -ENOSPC)) {
818 				vhost_tx_batch(net, nvq, sock, &msg);
819 				vhost_discard_vq_desc(vq, 1);
820 				vhost_net_enable_vq(net, vq);
821 				break;
822 			}
823 
824 			/* We can't build XDP buff, go for single
825 			 * packet path but let's flush batched
826 			 * packets.
827 			 */
828 			vhost_tx_batch(net, nvq, sock, &msg);
829 			msg.msg_control = NULL;
830 		} else {
831 			if (tx_can_batch(vq, total_len))
832 				msg.msg_flags |= MSG_MORE;
833 			else
834 				msg.msg_flags &= ~MSG_MORE;
835 		}
836 
837 		err = sock->ops->sendmsg(sock, &msg, len);
838 		if (unlikely(err < 0)) {
839 			if (err == -EAGAIN || err == -ENOMEM || err == -ENOBUFS) {
840 				vhost_discard_vq_desc(vq, 1);
841 				vhost_net_enable_vq(net, vq);
842 				break;
843 			}
844 			pr_debug("Fail to send packet: err %d", err);
845 		} else if (unlikely(err != len))
846 			pr_debug("Truncated TX packet: len %d != %zd\n",
847 				 err, len);
848 done:
849 		vq->heads[nvq->done_idx].id = cpu_to_vhost32(vq, head);
850 		vq->heads[nvq->done_idx].len = 0;
851 		++nvq->done_idx;
852 	} while (likely(!vhost_exceeds_weight(vq, ++sent_pkts, total_len)));
853 
854 	vhost_tx_batch(net, nvq, sock, &msg);
855 }
856 
handle_tx_zerocopy(struct vhost_net * net,struct socket * sock)857 static void handle_tx_zerocopy(struct vhost_net *net, struct socket *sock)
858 {
859 	struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_TX];
860 	struct vhost_virtqueue *vq = &nvq->vq;
861 	unsigned out, in;
862 	int head;
863 	struct msghdr msg = {
864 		.msg_name = NULL,
865 		.msg_namelen = 0,
866 		.msg_control = NULL,
867 		.msg_controllen = 0,
868 		.msg_flags = MSG_DONTWAIT,
869 	};
870 	struct tun_msg_ctl ctl;
871 	size_t len, total_len = 0;
872 	int err;
873 	struct vhost_net_ubuf_ref *ubufs;
874 	struct ubuf_info *ubuf;
875 	bool zcopy_used;
876 	int sent_pkts = 0;
877 
878 	do {
879 		bool busyloop_intr;
880 
881 		/* Release DMAs done buffers first */
882 		vhost_zerocopy_signal_used(net, vq);
883 
884 		busyloop_intr = false;
885 		head = get_tx_bufs(net, nvq, &msg, &out, &in, &len,
886 				   &busyloop_intr);
887 		/* On error, stop handling until the next kick. */
888 		if (unlikely(head < 0))
889 			break;
890 		/* Nothing new?  Wait for eventfd to tell us they refilled. */
891 		if (head == vq->num) {
892 			if (unlikely(busyloop_intr)) {
893 				vhost_poll_queue(&vq->poll);
894 			} else if (unlikely(vhost_enable_notify(&net->dev, vq))) {
895 				vhost_disable_notify(&net->dev, vq);
896 				continue;
897 			}
898 			break;
899 		}
900 
901 		zcopy_used = len >= VHOST_GOODCOPY_LEN
902 			     && !vhost_exceeds_maxpend(net)
903 			     && vhost_net_tx_select_zcopy(net);
904 
905 		/* use msg_control to pass vhost zerocopy ubuf info to skb */
906 		if (zcopy_used) {
907 			ubuf = nvq->ubuf_info + nvq->upend_idx;
908 			vq->heads[nvq->upend_idx].id = cpu_to_vhost32(vq, head);
909 			vq->heads[nvq->upend_idx].len = VHOST_DMA_IN_PROGRESS;
910 			ubuf->callback = vhost_zerocopy_callback;
911 			ubuf->ctx = nvq->ubufs;
912 			ubuf->desc = nvq->upend_idx;
913 			ubuf->flags = SKBFL_ZEROCOPY_FRAG;
914 			refcount_set(&ubuf->refcnt, 1);
915 			msg.msg_control = &ctl;
916 			ctl.type = TUN_MSG_UBUF;
917 			ctl.ptr = ubuf;
918 			msg.msg_controllen = sizeof(ctl);
919 			ubufs = nvq->ubufs;
920 			atomic_inc(&ubufs->refcount);
921 			nvq->upend_idx = (nvq->upend_idx + 1) % UIO_MAXIOV;
922 		} else {
923 			msg.msg_control = NULL;
924 			ubufs = NULL;
925 		}
926 		total_len += len;
927 		if (tx_can_batch(vq, total_len) &&
928 		    likely(!vhost_exceeds_maxpend(net))) {
929 			msg.msg_flags |= MSG_MORE;
930 		} else {
931 			msg.msg_flags &= ~MSG_MORE;
932 		}
933 
934 		err = sock->ops->sendmsg(sock, &msg, len);
935 		if (unlikely(err < 0)) {
936 			bool retry = err == -EAGAIN || err == -ENOMEM || err == -ENOBUFS;
937 
938 			if (zcopy_used) {
939 				if (vq->heads[ubuf->desc].len == VHOST_DMA_IN_PROGRESS)
940 					vhost_net_ubuf_put(ubufs);
941 				if (retry)
942 					nvq->upend_idx = ((unsigned)nvq->upend_idx - 1)
943 						% UIO_MAXIOV;
944 				else
945 					vq->heads[ubuf->desc].len = VHOST_DMA_DONE_LEN;
946 			}
947 			if (retry) {
948 				vhost_discard_vq_desc(vq, 1);
949 				vhost_net_enable_vq(net, vq);
950 				break;
951 			}
952 			pr_debug("Fail to send packet: err %d", err);
953 		} else if (unlikely(err != len))
954 			pr_debug("Truncated TX packet: "
955 				 " len %d != %zd\n", err, len);
956 		if (!zcopy_used)
957 			vhost_add_used_and_signal(&net->dev, vq, head, 0);
958 		else
959 			vhost_zerocopy_signal_used(net, vq);
960 		vhost_net_tx_packet(net);
961 	} while (likely(!vhost_exceeds_weight(vq, ++sent_pkts, total_len)));
962 }
963 
964 /* Expects to be always run from workqueue - which acts as
965  * read-size critical section for our kind of RCU. */
handle_tx(struct vhost_net * net)966 static void handle_tx(struct vhost_net *net)
967 {
968 	struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_TX];
969 	struct vhost_virtqueue *vq = &nvq->vq;
970 	struct socket *sock;
971 
972 	mutex_lock_nested(&vq->mutex, VHOST_NET_VQ_TX);
973 	sock = vhost_vq_get_backend(vq);
974 	if (!sock)
975 		goto out;
976 
977 	if (!vq_meta_prefetch(vq))
978 		goto out;
979 
980 	vhost_disable_notify(&net->dev, vq);
981 	vhost_net_disable_vq(net, vq);
982 
983 	if (vhost_sock_zcopy(sock))
984 		handle_tx_zerocopy(net, sock);
985 	else
986 		handle_tx_copy(net, sock);
987 
988 out:
989 	mutex_unlock(&vq->mutex);
990 }
991 
peek_head_len(struct vhost_net_virtqueue * rvq,struct sock * sk)992 static int peek_head_len(struct vhost_net_virtqueue *rvq, struct sock *sk)
993 {
994 	struct sk_buff *head;
995 	int len = 0;
996 	unsigned long flags;
997 
998 	if (rvq->rx_ring)
999 		return vhost_net_buf_peek(rvq);
1000 
1001 	spin_lock_irqsave(&sk->sk_receive_queue.lock, flags);
1002 	head = skb_peek(&sk->sk_receive_queue);
1003 	if (likely(head)) {
1004 		len = head->len;
1005 		if (skb_vlan_tag_present(head))
1006 			len += VLAN_HLEN;
1007 	}
1008 
1009 	spin_unlock_irqrestore(&sk->sk_receive_queue.lock, flags);
1010 	return len;
1011 }
1012 
vhost_net_rx_peek_head_len(struct vhost_net * net,struct sock * sk,bool * busyloop_intr)1013 static int vhost_net_rx_peek_head_len(struct vhost_net *net, struct sock *sk,
1014 				      bool *busyloop_intr)
1015 {
1016 	struct vhost_net_virtqueue *rnvq = &net->vqs[VHOST_NET_VQ_RX];
1017 	struct vhost_net_virtqueue *tnvq = &net->vqs[VHOST_NET_VQ_TX];
1018 	struct vhost_virtqueue *rvq = &rnvq->vq;
1019 	struct vhost_virtqueue *tvq = &tnvq->vq;
1020 	int len = peek_head_len(rnvq, sk);
1021 
1022 	if (!len && rvq->busyloop_timeout) {
1023 		/* Flush batched heads first */
1024 		vhost_net_signal_used(rnvq);
1025 		/* Both tx vq and rx socket were polled here */
1026 		vhost_net_busy_poll(net, rvq, tvq, busyloop_intr, true);
1027 
1028 		len = peek_head_len(rnvq, sk);
1029 	}
1030 
1031 	return len;
1032 }
1033 
1034 /* This is a multi-buffer version of vhost_get_desc, that works if
1035  *	vq has read descriptors only.
1036  * @vq		- the relevant virtqueue
1037  * @datalen	- data length we'll be reading
1038  * @iovcount	- returned count of io vectors we fill
1039  * @log		- vhost log
1040  * @log_num	- log offset
1041  * @quota       - headcount quota, 1 for big buffer
1042  *	returns number of buffer heads allocated, negative on error
1043  */
get_rx_bufs(struct vhost_virtqueue * vq,struct vring_used_elem * heads,int datalen,unsigned * iovcount,struct vhost_log * log,unsigned * log_num,unsigned int quota)1044 static int get_rx_bufs(struct vhost_virtqueue *vq,
1045 		       struct vring_used_elem *heads,
1046 		       int datalen,
1047 		       unsigned *iovcount,
1048 		       struct vhost_log *log,
1049 		       unsigned *log_num,
1050 		       unsigned int quota)
1051 {
1052 	unsigned int out, in;
1053 	int seg = 0;
1054 	int headcount = 0;
1055 	unsigned d;
1056 	int r, nlogs = 0;
1057 	/* len is always initialized before use since we are always called with
1058 	 * datalen > 0.
1059 	 */
1060 	u32 len;
1061 
1062 	while (datalen > 0 && headcount < quota) {
1063 		if (unlikely(seg >= UIO_MAXIOV)) {
1064 			r = -ENOBUFS;
1065 			goto err;
1066 		}
1067 		r = vhost_get_vq_desc(vq, vq->iov + seg,
1068 				      ARRAY_SIZE(vq->iov) - seg, &out,
1069 				      &in, log, log_num);
1070 		if (unlikely(r < 0))
1071 			goto err;
1072 
1073 		d = r;
1074 		if (d == vq->num) {
1075 			r = 0;
1076 			goto err;
1077 		}
1078 		if (unlikely(out || in <= 0)) {
1079 			vq_err(vq, "unexpected descriptor format for RX: "
1080 				"out %d, in %d\n", out, in);
1081 			r = -EINVAL;
1082 			goto err;
1083 		}
1084 		if (unlikely(log)) {
1085 			nlogs += *log_num;
1086 			log += *log_num;
1087 		}
1088 		heads[headcount].id = cpu_to_vhost32(vq, d);
1089 		len = iov_length(vq->iov + seg, in);
1090 		heads[headcount].len = cpu_to_vhost32(vq, len);
1091 		datalen -= len;
1092 		++headcount;
1093 		seg += in;
1094 	}
1095 	heads[headcount - 1].len = cpu_to_vhost32(vq, len + datalen);
1096 	*iovcount = seg;
1097 	if (unlikely(log))
1098 		*log_num = nlogs;
1099 
1100 	/* Detect overrun */
1101 	if (unlikely(datalen > 0)) {
1102 		r = UIO_MAXIOV + 1;
1103 		goto err;
1104 	}
1105 	return headcount;
1106 err:
1107 	vhost_discard_vq_desc(vq, headcount);
1108 	return r;
1109 }
1110 
1111 /* Expects to be always run from workqueue - which acts as
1112  * read-size critical section for our kind of RCU. */
handle_rx(struct vhost_net * net)1113 static void handle_rx(struct vhost_net *net)
1114 {
1115 	struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_RX];
1116 	struct vhost_virtqueue *vq = &nvq->vq;
1117 	unsigned in, log;
1118 	struct vhost_log *vq_log;
1119 	struct msghdr msg = {
1120 		.msg_name = NULL,
1121 		.msg_namelen = 0,
1122 		.msg_control = NULL, /* FIXME: get and handle RX aux data. */
1123 		.msg_controllen = 0,
1124 		.msg_flags = MSG_DONTWAIT,
1125 	};
1126 	struct virtio_net_hdr hdr = {
1127 		.flags = 0,
1128 		.gso_type = VIRTIO_NET_HDR_GSO_NONE
1129 	};
1130 	size_t total_len = 0;
1131 	int err, mergeable;
1132 	s16 headcount;
1133 	size_t vhost_hlen, sock_hlen;
1134 	size_t vhost_len, sock_len;
1135 	bool busyloop_intr = false;
1136 	struct socket *sock;
1137 	struct iov_iter fixup;
1138 	__virtio16 num_buffers;
1139 	int recv_pkts = 0;
1140 
1141 	mutex_lock_nested(&vq->mutex, VHOST_NET_VQ_RX);
1142 	sock = vhost_vq_get_backend(vq);
1143 	if (!sock)
1144 		goto out;
1145 
1146 	if (!vq_meta_prefetch(vq))
1147 		goto out;
1148 
1149 	vhost_disable_notify(&net->dev, vq);
1150 	vhost_net_disable_vq(net, vq);
1151 
1152 	vhost_hlen = nvq->vhost_hlen;
1153 	sock_hlen = nvq->sock_hlen;
1154 
1155 	vq_log = unlikely(vhost_has_feature(vq, VHOST_F_LOG_ALL)) ?
1156 		vq->log : NULL;
1157 	mergeable = vhost_has_feature(vq, VIRTIO_NET_F_MRG_RXBUF);
1158 
1159 	do {
1160 		sock_len = vhost_net_rx_peek_head_len(net, sock->sk,
1161 						      &busyloop_intr);
1162 		if (!sock_len)
1163 			break;
1164 		sock_len += sock_hlen;
1165 		vhost_len = sock_len + vhost_hlen;
1166 		headcount = get_rx_bufs(vq, vq->heads + nvq->done_idx,
1167 					vhost_len, &in, vq_log, &log,
1168 					likely(mergeable) ? UIO_MAXIOV : 1);
1169 		/* On error, stop handling until the next kick. */
1170 		if (unlikely(headcount < 0))
1171 			goto out;
1172 		/* OK, now we need to know about added descriptors. */
1173 		if (!headcount) {
1174 			if (unlikely(busyloop_intr)) {
1175 				vhost_poll_queue(&vq->poll);
1176 			} else if (unlikely(vhost_enable_notify(&net->dev, vq))) {
1177 				/* They have slipped one in as we were
1178 				 * doing that: check again. */
1179 				vhost_disable_notify(&net->dev, vq);
1180 				continue;
1181 			}
1182 			/* Nothing new?  Wait for eventfd to tell us
1183 			 * they refilled. */
1184 			goto out;
1185 		}
1186 		busyloop_intr = false;
1187 		if (nvq->rx_ring)
1188 			msg.msg_control = vhost_net_buf_consume(&nvq->rxq);
1189 		/* On overrun, truncate and discard */
1190 		if (unlikely(headcount > UIO_MAXIOV)) {
1191 			iov_iter_init(&msg.msg_iter, READ, vq->iov, 1, 1);
1192 			err = sock->ops->recvmsg(sock, &msg,
1193 						 1, MSG_DONTWAIT | MSG_TRUNC);
1194 			pr_debug("Discarded rx packet: len %zd\n", sock_len);
1195 			continue;
1196 		}
1197 		/* We don't need to be notified again. */
1198 		iov_iter_init(&msg.msg_iter, READ, vq->iov, in, vhost_len);
1199 		fixup = msg.msg_iter;
1200 		if (unlikely((vhost_hlen))) {
1201 			/* We will supply the header ourselves
1202 			 * TODO: support TSO.
1203 			 */
1204 			iov_iter_advance(&msg.msg_iter, vhost_hlen);
1205 		}
1206 		err = sock->ops->recvmsg(sock, &msg,
1207 					 sock_len, MSG_DONTWAIT | MSG_TRUNC);
1208 		/* Userspace might have consumed the packet meanwhile:
1209 		 * it's not supposed to do this usually, but might be hard
1210 		 * to prevent. Discard data we got (if any) and keep going. */
1211 		if (unlikely(err != sock_len)) {
1212 			pr_debug("Discarded rx packet: "
1213 				 " len %d, expected %zd\n", err, sock_len);
1214 			vhost_discard_vq_desc(vq, headcount);
1215 			continue;
1216 		}
1217 		/* Supply virtio_net_hdr if VHOST_NET_F_VIRTIO_NET_HDR */
1218 		if (unlikely(vhost_hlen)) {
1219 			if (copy_to_iter(&hdr, sizeof(hdr),
1220 					 &fixup) != sizeof(hdr)) {
1221 				vq_err(vq, "Unable to write vnet_hdr "
1222 				       "at addr %p\n", vq->iov->iov_base);
1223 				goto out;
1224 			}
1225 		} else {
1226 			/* Header came from socket; we'll need to patch
1227 			 * ->num_buffers over if VIRTIO_NET_F_MRG_RXBUF
1228 			 */
1229 			iov_iter_advance(&fixup, sizeof(hdr));
1230 		}
1231 		/* TODO: Should check and handle checksum. */
1232 
1233 		num_buffers = cpu_to_vhost16(vq, headcount);
1234 		if (likely(mergeable) &&
1235 		    copy_to_iter(&num_buffers, sizeof num_buffers,
1236 				 &fixup) != sizeof num_buffers) {
1237 			vq_err(vq, "Failed num_buffers write");
1238 			vhost_discard_vq_desc(vq, headcount);
1239 			goto out;
1240 		}
1241 		nvq->done_idx += headcount;
1242 		if (nvq->done_idx > VHOST_NET_BATCH)
1243 			vhost_net_signal_used(nvq);
1244 		if (unlikely(vq_log))
1245 			vhost_log_write(vq, vq_log, log, vhost_len,
1246 					vq->iov, in);
1247 		total_len += vhost_len;
1248 	} while (likely(!vhost_exceeds_weight(vq, ++recv_pkts, total_len)));
1249 
1250 	if (unlikely(busyloop_intr))
1251 		vhost_poll_queue(&vq->poll);
1252 	else if (!sock_len)
1253 		vhost_net_enable_vq(net, vq);
1254 out:
1255 	vhost_net_signal_used(nvq);
1256 	mutex_unlock(&vq->mutex);
1257 }
1258 
handle_tx_kick(struct vhost_work * work)1259 static void handle_tx_kick(struct vhost_work *work)
1260 {
1261 	struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,
1262 						  poll.work);
1263 	struct vhost_net *net = container_of(vq->dev, struct vhost_net, dev);
1264 
1265 	handle_tx(net);
1266 }
1267 
handle_rx_kick(struct vhost_work * work)1268 static void handle_rx_kick(struct vhost_work *work)
1269 {
1270 	struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,
1271 						  poll.work);
1272 	struct vhost_net *net = container_of(vq->dev, struct vhost_net, dev);
1273 
1274 	handle_rx(net);
1275 }
1276 
handle_tx_net(struct vhost_work * work)1277 static void handle_tx_net(struct vhost_work *work)
1278 {
1279 	struct vhost_net *net = container_of(work, struct vhost_net,
1280 					     poll[VHOST_NET_VQ_TX].work);
1281 	handle_tx(net);
1282 }
1283 
handle_rx_net(struct vhost_work * work)1284 static void handle_rx_net(struct vhost_work *work)
1285 {
1286 	struct vhost_net *net = container_of(work, struct vhost_net,
1287 					     poll[VHOST_NET_VQ_RX].work);
1288 	handle_rx(net);
1289 }
1290 
vhost_net_open(struct inode * inode,struct file * f)1291 static int vhost_net_open(struct inode *inode, struct file *f)
1292 {
1293 	struct vhost_net *n;
1294 	struct vhost_dev *dev;
1295 	struct vhost_virtqueue **vqs;
1296 	void **queue;
1297 	struct xdp_buff *xdp;
1298 	int i;
1299 
1300 	n = kvmalloc(sizeof *n, GFP_KERNEL | __GFP_RETRY_MAYFAIL);
1301 	if (!n)
1302 		return -ENOMEM;
1303 	vqs = kmalloc_array(VHOST_NET_VQ_MAX, sizeof(*vqs), GFP_KERNEL);
1304 	if (!vqs) {
1305 		kvfree(n);
1306 		return -ENOMEM;
1307 	}
1308 
1309 	queue = kmalloc_array(VHOST_NET_BATCH, sizeof(void *),
1310 			      GFP_KERNEL);
1311 	if (!queue) {
1312 		kfree(vqs);
1313 		kvfree(n);
1314 		return -ENOMEM;
1315 	}
1316 	n->vqs[VHOST_NET_VQ_RX].rxq.queue = queue;
1317 
1318 	xdp = kmalloc_array(VHOST_NET_BATCH, sizeof(*xdp), GFP_KERNEL);
1319 	if (!xdp) {
1320 		kfree(vqs);
1321 		kvfree(n);
1322 		kfree(queue);
1323 		return -ENOMEM;
1324 	}
1325 	n->vqs[VHOST_NET_VQ_TX].xdp = xdp;
1326 
1327 	dev = &n->dev;
1328 	vqs[VHOST_NET_VQ_TX] = &n->vqs[VHOST_NET_VQ_TX].vq;
1329 	vqs[VHOST_NET_VQ_RX] = &n->vqs[VHOST_NET_VQ_RX].vq;
1330 	n->vqs[VHOST_NET_VQ_TX].vq.handle_kick = handle_tx_kick;
1331 	n->vqs[VHOST_NET_VQ_RX].vq.handle_kick = handle_rx_kick;
1332 	for (i = 0; i < VHOST_NET_VQ_MAX; i++) {
1333 		n->vqs[i].ubufs = NULL;
1334 		n->vqs[i].ubuf_info = NULL;
1335 		n->vqs[i].upend_idx = 0;
1336 		n->vqs[i].done_idx = 0;
1337 		n->vqs[i].batched_xdp = 0;
1338 		n->vqs[i].vhost_hlen = 0;
1339 		n->vqs[i].sock_hlen = 0;
1340 		n->vqs[i].rx_ring = NULL;
1341 		vhost_net_buf_init(&n->vqs[i].rxq);
1342 	}
1343 	vhost_dev_init(dev, vqs, VHOST_NET_VQ_MAX,
1344 		       UIO_MAXIOV + VHOST_NET_BATCH,
1345 		       VHOST_NET_PKT_WEIGHT, VHOST_NET_WEIGHT, true,
1346 		       NULL);
1347 
1348 	vhost_poll_init(n->poll + VHOST_NET_VQ_TX, handle_tx_net, EPOLLOUT, dev);
1349 	vhost_poll_init(n->poll + VHOST_NET_VQ_RX, handle_rx_net, EPOLLIN, dev);
1350 
1351 	f->private_data = n;
1352 	n->page_frag.page = NULL;
1353 	n->refcnt_bias = 0;
1354 
1355 	return 0;
1356 }
1357 
vhost_net_stop_vq(struct vhost_net * n,struct vhost_virtqueue * vq)1358 static struct socket *vhost_net_stop_vq(struct vhost_net *n,
1359 					struct vhost_virtqueue *vq)
1360 {
1361 	struct socket *sock;
1362 	struct vhost_net_virtqueue *nvq =
1363 		container_of(vq, struct vhost_net_virtqueue, vq);
1364 
1365 	mutex_lock(&vq->mutex);
1366 	sock = vhost_vq_get_backend(vq);
1367 	vhost_net_disable_vq(n, vq);
1368 	vhost_vq_set_backend(vq, NULL);
1369 	vhost_net_buf_unproduce(nvq);
1370 	nvq->rx_ring = NULL;
1371 	mutex_unlock(&vq->mutex);
1372 	return sock;
1373 }
1374 
vhost_net_stop(struct vhost_net * n,struct socket ** tx_sock,struct socket ** rx_sock)1375 static void vhost_net_stop(struct vhost_net *n, struct socket **tx_sock,
1376 			   struct socket **rx_sock)
1377 {
1378 	*tx_sock = vhost_net_stop_vq(n, &n->vqs[VHOST_NET_VQ_TX].vq);
1379 	*rx_sock = vhost_net_stop_vq(n, &n->vqs[VHOST_NET_VQ_RX].vq);
1380 }
1381 
vhost_net_flush_vq(struct vhost_net * n,int index)1382 static void vhost_net_flush_vq(struct vhost_net *n, int index)
1383 {
1384 	vhost_poll_flush(n->poll + index);
1385 	vhost_poll_flush(&n->vqs[index].vq.poll);
1386 }
1387 
vhost_net_flush(struct vhost_net * n)1388 static void vhost_net_flush(struct vhost_net *n)
1389 {
1390 	vhost_net_flush_vq(n, VHOST_NET_VQ_TX);
1391 	vhost_net_flush_vq(n, VHOST_NET_VQ_RX);
1392 	if (n->vqs[VHOST_NET_VQ_TX].ubufs) {
1393 		mutex_lock(&n->vqs[VHOST_NET_VQ_TX].vq.mutex);
1394 		n->tx_flush = true;
1395 		mutex_unlock(&n->vqs[VHOST_NET_VQ_TX].vq.mutex);
1396 		/* Wait for all lower device DMAs done. */
1397 		vhost_net_ubuf_put_and_wait(n->vqs[VHOST_NET_VQ_TX].ubufs);
1398 		mutex_lock(&n->vqs[VHOST_NET_VQ_TX].vq.mutex);
1399 		n->tx_flush = false;
1400 		atomic_set(&n->vqs[VHOST_NET_VQ_TX].ubufs->refcount, 1);
1401 		mutex_unlock(&n->vqs[VHOST_NET_VQ_TX].vq.mutex);
1402 	}
1403 }
1404 
vhost_net_release(struct inode * inode,struct file * f)1405 static int vhost_net_release(struct inode *inode, struct file *f)
1406 {
1407 	struct vhost_net *n = f->private_data;
1408 	struct socket *tx_sock;
1409 	struct socket *rx_sock;
1410 
1411 	vhost_net_stop(n, &tx_sock, &rx_sock);
1412 	vhost_net_flush(n);
1413 	vhost_dev_stop(&n->dev);
1414 	vhost_dev_cleanup(&n->dev);
1415 	vhost_net_vq_reset(n);
1416 	if (tx_sock)
1417 		sockfd_put(tx_sock);
1418 	if (rx_sock)
1419 		sockfd_put(rx_sock);
1420 	/* Make sure no callbacks are outstanding */
1421 	synchronize_rcu();
1422 	/* We do an extra flush before freeing memory,
1423 	 * since jobs can re-queue themselves. */
1424 	vhost_net_flush(n);
1425 	kfree(n->vqs[VHOST_NET_VQ_RX].rxq.queue);
1426 	kfree(n->vqs[VHOST_NET_VQ_TX].xdp);
1427 	kfree(n->dev.vqs);
1428 	if (n->page_frag.page)
1429 		__page_frag_cache_drain(n->page_frag.page, n->refcnt_bias);
1430 	kvfree(n);
1431 	return 0;
1432 }
1433 
get_raw_socket(int fd)1434 static struct socket *get_raw_socket(int fd)
1435 {
1436 	int r;
1437 	struct socket *sock = sockfd_lookup(fd, &r);
1438 
1439 	if (!sock)
1440 		return ERR_PTR(-ENOTSOCK);
1441 
1442 	/* Parameter checking */
1443 	if (sock->sk->sk_type != SOCK_RAW) {
1444 		r = -ESOCKTNOSUPPORT;
1445 		goto err;
1446 	}
1447 
1448 	if (sock->sk->sk_family != AF_PACKET) {
1449 		r = -EPFNOSUPPORT;
1450 		goto err;
1451 	}
1452 	return sock;
1453 err:
1454 	sockfd_put(sock);
1455 	return ERR_PTR(r);
1456 }
1457 
get_tap_ptr_ring(struct file * file)1458 static struct ptr_ring *get_tap_ptr_ring(struct file *file)
1459 {
1460 	struct ptr_ring *ring;
1461 	ring = tun_get_tx_ring(file);
1462 	if (!IS_ERR(ring))
1463 		goto out;
1464 	ring = tap_get_ptr_ring(file);
1465 	if (!IS_ERR(ring))
1466 		goto out;
1467 	ring = NULL;
1468 out:
1469 	return ring;
1470 }
1471 
get_tap_socket(int fd)1472 static struct socket *get_tap_socket(int fd)
1473 {
1474 	struct file *file = fget(fd);
1475 	struct socket *sock;
1476 
1477 	if (!file)
1478 		return ERR_PTR(-EBADF);
1479 	sock = tun_get_socket(file);
1480 	if (!IS_ERR(sock))
1481 		return sock;
1482 	sock = tap_get_socket(file);
1483 	if (IS_ERR(sock))
1484 		fput(file);
1485 	return sock;
1486 }
1487 
get_socket(int fd)1488 static struct socket *get_socket(int fd)
1489 {
1490 	struct socket *sock;
1491 
1492 	/* special case to disable backend */
1493 	if (fd == -1)
1494 		return NULL;
1495 	sock = get_raw_socket(fd);
1496 	if (!IS_ERR(sock))
1497 		return sock;
1498 	sock = get_tap_socket(fd);
1499 	if (!IS_ERR(sock))
1500 		return sock;
1501 	return ERR_PTR(-ENOTSOCK);
1502 }
1503 
vhost_net_set_backend(struct vhost_net * n,unsigned index,int fd)1504 static long vhost_net_set_backend(struct vhost_net *n, unsigned index, int fd)
1505 {
1506 	struct socket *sock, *oldsock;
1507 	struct vhost_virtqueue *vq;
1508 	struct vhost_net_virtqueue *nvq;
1509 	struct vhost_net_ubuf_ref *ubufs, *oldubufs = NULL;
1510 	int r;
1511 
1512 	mutex_lock(&n->dev.mutex);
1513 	r = vhost_dev_check_owner(&n->dev);
1514 	if (r)
1515 		goto err;
1516 
1517 	if (index >= VHOST_NET_VQ_MAX) {
1518 		r = -ENOBUFS;
1519 		goto err;
1520 	}
1521 	vq = &n->vqs[index].vq;
1522 	nvq = &n->vqs[index];
1523 	mutex_lock(&vq->mutex);
1524 
1525 	if (fd == -1)
1526 		vhost_clear_msg(&n->dev);
1527 
1528 	/* Verify that ring has been setup correctly. */
1529 	if (!vhost_vq_access_ok(vq)) {
1530 		r = -EFAULT;
1531 		goto err_vq;
1532 	}
1533 	sock = get_socket(fd);
1534 	if (IS_ERR(sock)) {
1535 		r = PTR_ERR(sock);
1536 		goto err_vq;
1537 	}
1538 
1539 	/* start polling new socket */
1540 	oldsock = vhost_vq_get_backend(vq);
1541 	if (sock != oldsock) {
1542 		ubufs = vhost_net_ubuf_alloc(vq,
1543 					     sock && vhost_sock_zcopy(sock));
1544 		if (IS_ERR(ubufs)) {
1545 			r = PTR_ERR(ubufs);
1546 			goto err_ubufs;
1547 		}
1548 
1549 		vhost_net_disable_vq(n, vq);
1550 		vhost_vq_set_backend(vq, sock);
1551 		vhost_net_buf_unproduce(nvq);
1552 		r = vhost_vq_init_access(vq);
1553 		if (r)
1554 			goto err_used;
1555 		r = vhost_net_enable_vq(n, vq);
1556 		if (r)
1557 			goto err_used;
1558 		if (index == VHOST_NET_VQ_RX) {
1559 			if (sock)
1560 				nvq->rx_ring = get_tap_ptr_ring(sock->file);
1561 			else
1562 				nvq->rx_ring = NULL;
1563 		}
1564 
1565 		oldubufs = nvq->ubufs;
1566 		nvq->ubufs = ubufs;
1567 
1568 		n->tx_packets = 0;
1569 		n->tx_zcopy_err = 0;
1570 		n->tx_flush = false;
1571 	}
1572 
1573 	mutex_unlock(&vq->mutex);
1574 
1575 	if (oldubufs) {
1576 		vhost_net_ubuf_put_wait_and_free(oldubufs);
1577 		mutex_lock(&vq->mutex);
1578 		vhost_zerocopy_signal_used(n, vq);
1579 		mutex_unlock(&vq->mutex);
1580 	}
1581 
1582 	if (oldsock) {
1583 		vhost_net_flush_vq(n, index);
1584 		sockfd_put(oldsock);
1585 	}
1586 
1587 	mutex_unlock(&n->dev.mutex);
1588 	return 0;
1589 
1590 err_used:
1591 	vhost_vq_set_backend(vq, oldsock);
1592 	vhost_net_enable_vq(n, vq);
1593 	if (ubufs)
1594 		vhost_net_ubuf_put_wait_and_free(ubufs);
1595 err_ubufs:
1596 	if (sock)
1597 		sockfd_put(sock);
1598 err_vq:
1599 	mutex_unlock(&vq->mutex);
1600 err:
1601 	mutex_unlock(&n->dev.mutex);
1602 	return r;
1603 }
1604 
vhost_net_reset_owner(struct vhost_net * n)1605 static long vhost_net_reset_owner(struct vhost_net *n)
1606 {
1607 	struct socket *tx_sock = NULL;
1608 	struct socket *rx_sock = NULL;
1609 	long err;
1610 	struct vhost_iotlb *umem;
1611 
1612 	mutex_lock(&n->dev.mutex);
1613 	err = vhost_dev_check_owner(&n->dev);
1614 	if (err)
1615 		goto done;
1616 	umem = vhost_dev_reset_owner_prepare();
1617 	if (!umem) {
1618 		err = -ENOMEM;
1619 		goto done;
1620 	}
1621 	vhost_net_stop(n, &tx_sock, &rx_sock);
1622 	vhost_net_flush(n);
1623 	vhost_dev_stop(&n->dev);
1624 	vhost_dev_reset_owner(&n->dev, umem);
1625 	vhost_net_vq_reset(n);
1626 done:
1627 	mutex_unlock(&n->dev.mutex);
1628 	if (tx_sock)
1629 		sockfd_put(tx_sock);
1630 	if (rx_sock)
1631 		sockfd_put(rx_sock);
1632 	return err;
1633 }
1634 
vhost_net_set_features(struct vhost_net * n,u64 features)1635 static int vhost_net_set_features(struct vhost_net *n, u64 features)
1636 {
1637 	size_t vhost_hlen, sock_hlen, hdr_len;
1638 	int i;
1639 
1640 	hdr_len = (features & ((1ULL << VIRTIO_NET_F_MRG_RXBUF) |
1641 			       (1ULL << VIRTIO_F_VERSION_1))) ?
1642 			sizeof(struct virtio_net_hdr_mrg_rxbuf) :
1643 			sizeof(struct virtio_net_hdr);
1644 	if (features & (1 << VHOST_NET_F_VIRTIO_NET_HDR)) {
1645 		/* vhost provides vnet_hdr */
1646 		vhost_hlen = hdr_len;
1647 		sock_hlen = 0;
1648 	} else {
1649 		/* socket provides vnet_hdr */
1650 		vhost_hlen = 0;
1651 		sock_hlen = hdr_len;
1652 	}
1653 	mutex_lock(&n->dev.mutex);
1654 	if ((features & (1 << VHOST_F_LOG_ALL)) &&
1655 	    !vhost_log_access_ok(&n->dev))
1656 		goto out_unlock;
1657 
1658 	if ((features & (1ULL << VIRTIO_F_ACCESS_PLATFORM))) {
1659 		if (vhost_init_device_iotlb(&n->dev, true))
1660 			goto out_unlock;
1661 	}
1662 
1663 	for (i = 0; i < VHOST_NET_VQ_MAX; ++i) {
1664 		mutex_lock(&n->vqs[i].vq.mutex);
1665 		n->vqs[i].vq.acked_features = features;
1666 		n->vqs[i].vhost_hlen = vhost_hlen;
1667 		n->vqs[i].sock_hlen = sock_hlen;
1668 		mutex_unlock(&n->vqs[i].vq.mutex);
1669 	}
1670 	mutex_unlock(&n->dev.mutex);
1671 	return 0;
1672 
1673 out_unlock:
1674 	mutex_unlock(&n->dev.mutex);
1675 	return -EFAULT;
1676 }
1677 
vhost_net_set_owner(struct vhost_net * n)1678 static long vhost_net_set_owner(struct vhost_net *n)
1679 {
1680 	int r;
1681 
1682 	mutex_lock(&n->dev.mutex);
1683 	if (vhost_dev_has_owner(&n->dev)) {
1684 		r = -EBUSY;
1685 		goto out;
1686 	}
1687 	r = vhost_net_set_ubuf_info(n);
1688 	if (r)
1689 		goto out;
1690 	r = vhost_dev_set_owner(&n->dev);
1691 	if (r)
1692 		vhost_net_clear_ubuf_info(n);
1693 	vhost_net_flush(n);
1694 out:
1695 	mutex_unlock(&n->dev.mutex);
1696 	return r;
1697 }
1698 
vhost_net_ioctl(struct file * f,unsigned int ioctl,unsigned long arg)1699 static long vhost_net_ioctl(struct file *f, unsigned int ioctl,
1700 			    unsigned long arg)
1701 {
1702 	struct vhost_net *n = f->private_data;
1703 	void __user *argp = (void __user *)arg;
1704 	u64 __user *featurep = argp;
1705 	struct vhost_vring_file backend;
1706 	u64 features;
1707 	int r;
1708 
1709 	switch (ioctl) {
1710 	case VHOST_NET_SET_BACKEND:
1711 		if (copy_from_user(&backend, argp, sizeof backend))
1712 			return -EFAULT;
1713 		return vhost_net_set_backend(n, backend.index, backend.fd);
1714 	case VHOST_GET_FEATURES:
1715 		features = VHOST_NET_FEATURES;
1716 		if (copy_to_user(featurep, &features, sizeof features))
1717 			return -EFAULT;
1718 		return 0;
1719 	case VHOST_SET_FEATURES:
1720 		if (copy_from_user(&features, featurep, sizeof features))
1721 			return -EFAULT;
1722 		if (features & ~VHOST_NET_FEATURES)
1723 			return -EOPNOTSUPP;
1724 		return vhost_net_set_features(n, features);
1725 	case VHOST_GET_BACKEND_FEATURES:
1726 		features = VHOST_NET_BACKEND_FEATURES;
1727 		if (copy_to_user(featurep, &features, sizeof(features)))
1728 			return -EFAULT;
1729 		return 0;
1730 	case VHOST_SET_BACKEND_FEATURES:
1731 		if (copy_from_user(&features, featurep, sizeof(features)))
1732 			return -EFAULT;
1733 		if (features & ~VHOST_NET_BACKEND_FEATURES)
1734 			return -EOPNOTSUPP;
1735 		vhost_set_backend_features(&n->dev, features);
1736 		return 0;
1737 	case VHOST_RESET_OWNER:
1738 		return vhost_net_reset_owner(n);
1739 	case VHOST_SET_OWNER:
1740 		return vhost_net_set_owner(n);
1741 	default:
1742 		mutex_lock(&n->dev.mutex);
1743 		r = vhost_dev_ioctl(&n->dev, ioctl, argp);
1744 		if (r == -ENOIOCTLCMD)
1745 			r = vhost_vring_ioctl(&n->dev, ioctl, argp);
1746 		else
1747 			vhost_net_flush(n);
1748 		mutex_unlock(&n->dev.mutex);
1749 		return r;
1750 	}
1751 }
1752 
vhost_net_chr_read_iter(struct kiocb * iocb,struct iov_iter * to)1753 static ssize_t vhost_net_chr_read_iter(struct kiocb *iocb, struct iov_iter *to)
1754 {
1755 	struct file *file = iocb->ki_filp;
1756 	struct vhost_net *n = file->private_data;
1757 	struct vhost_dev *dev = &n->dev;
1758 	int noblock = file->f_flags & O_NONBLOCK;
1759 
1760 	return vhost_chr_read_iter(dev, to, noblock);
1761 }
1762 
vhost_net_chr_write_iter(struct kiocb * iocb,struct iov_iter * from)1763 static ssize_t vhost_net_chr_write_iter(struct kiocb *iocb,
1764 					struct iov_iter *from)
1765 {
1766 	struct file *file = iocb->ki_filp;
1767 	struct vhost_net *n = file->private_data;
1768 	struct vhost_dev *dev = &n->dev;
1769 
1770 	return vhost_chr_write_iter(dev, from);
1771 }
1772 
vhost_net_chr_poll(struct file * file,poll_table * wait)1773 static __poll_t vhost_net_chr_poll(struct file *file, poll_table *wait)
1774 {
1775 	struct vhost_net *n = file->private_data;
1776 	struct vhost_dev *dev = &n->dev;
1777 
1778 	return vhost_chr_poll(file, dev, wait);
1779 }
1780 
1781 static const struct file_operations vhost_net_fops = {
1782 	.owner          = THIS_MODULE,
1783 	.release        = vhost_net_release,
1784 	.read_iter      = vhost_net_chr_read_iter,
1785 	.write_iter     = vhost_net_chr_write_iter,
1786 	.poll           = vhost_net_chr_poll,
1787 	.unlocked_ioctl = vhost_net_ioctl,
1788 	.compat_ioctl   = compat_ptr_ioctl,
1789 	.open           = vhost_net_open,
1790 	.llseek		= noop_llseek,
1791 };
1792 
1793 static struct miscdevice vhost_net_misc = {
1794 	.minor = VHOST_NET_MINOR,
1795 	.name = "vhost-net",
1796 	.fops = &vhost_net_fops,
1797 };
1798 
vhost_net_init(void)1799 static int vhost_net_init(void)
1800 {
1801 	if (experimental_zcopytx)
1802 		vhost_net_enable_zcopy(VHOST_NET_VQ_TX);
1803 	return misc_register(&vhost_net_misc);
1804 }
1805 module_init(vhost_net_init);
1806 
vhost_net_exit(void)1807 static void vhost_net_exit(void)
1808 {
1809 	misc_deregister(&vhost_net_misc);
1810 }
1811 module_exit(vhost_net_exit);
1812 
1813 MODULE_VERSION("0.0.1");
1814 MODULE_LICENSE("GPL v2");
1815 MODULE_AUTHOR("Michael S. Tsirkin");
1816 MODULE_DESCRIPTION("Host kernel accelerator for virtio net");
1817 MODULE_ALIAS_MISCDEV(VHOST_NET_MINOR);
1818 MODULE_ALIAS("devname:vhost-net");
1819