1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (C) 2009 Red Hat, Inc.
3 * Author: Michael S. Tsirkin <mst@redhat.com>
4 *
5 * virtio-net server in host kernel.
6 */
7
8 #include <linux/compat.h>
9 #include <linux/eventfd.h>
10 #include <linux/vhost.h>
11 #include <linux/virtio_net.h>
12 #include <linux/miscdevice.h>
13 #include <linux/module.h>
14 #include <linux/moduleparam.h>
15 #include <linux/mutex.h>
16 #include <linux/workqueue.h>
17 #include <linux/file.h>
18 #include <linux/slab.h>
19 #include <linux/sched/clock.h>
20 #include <linux/sched/signal.h>
21 #include <linux/vmalloc.h>
22
23 #include <linux/net.h>
24 #include <linux/if_packet.h>
25 #include <linux/if_arp.h>
26 #include <linux/if_tun.h>
27 #include <linux/if_macvlan.h>
28 #include <linux/if_tap.h>
29 #include <linux/if_vlan.h>
30 #include <linux/skb_array.h>
31 #include <linux/skbuff.h>
32
33 #include <net/sock.h>
34 #include <net/xdp.h>
35
36 #include "vhost.h"
37
38 static int experimental_zcopytx = 0;
39 module_param(experimental_zcopytx, int, 0444);
40 MODULE_PARM_DESC(experimental_zcopytx, "Enable Zero Copy TX;"
41 " 1 -Enable; 0 - Disable");
42
43 /* Max number of bytes transferred before requeueing the job.
44 * Using this limit prevents one virtqueue from starving others. */
45 #define VHOST_NET_WEIGHT 0x80000
46
47 /* Max number of packets transferred before requeueing the job.
48 * Using this limit prevents one virtqueue from starving others with small
49 * pkts.
50 */
51 #define VHOST_NET_PKT_WEIGHT 256
52
53 /* MAX number of TX used buffers for outstanding zerocopy */
54 #define VHOST_MAX_PEND 128
55 #define VHOST_GOODCOPY_LEN 256
56
57 /*
58 * For transmit, used buffer len is unused; we override it to track buffer
59 * status internally; used for zerocopy tx only.
60 */
61 /* Lower device DMA failed */
62 #define VHOST_DMA_FAILED_LEN ((__force __virtio32)3)
63 /* Lower device DMA done */
64 #define VHOST_DMA_DONE_LEN ((__force __virtio32)2)
65 /* Lower device DMA in progress */
66 #define VHOST_DMA_IN_PROGRESS ((__force __virtio32)1)
67 /* Buffer unused */
68 #define VHOST_DMA_CLEAR_LEN ((__force __virtio32)0)
69
70 #define VHOST_DMA_IS_DONE(len) ((__force u32)(len) >= (__force u32)VHOST_DMA_DONE_LEN)
71
72 enum {
73 VHOST_NET_FEATURES = VHOST_FEATURES |
74 (1ULL << VHOST_NET_F_VIRTIO_NET_HDR) |
75 (1ULL << VIRTIO_NET_F_MRG_RXBUF) |
76 (1ULL << VIRTIO_F_ACCESS_PLATFORM)
77 };
78
79 enum {
80 VHOST_NET_BACKEND_FEATURES = (1ULL << VHOST_BACKEND_F_IOTLB_MSG_V2)
81 };
82
83 enum {
84 VHOST_NET_VQ_RX = 0,
85 VHOST_NET_VQ_TX = 1,
86 VHOST_NET_VQ_MAX = 2,
87 };
88
89 struct vhost_net_ubuf_ref {
90 /* refcount follows semantics similar to kref:
91 * 0: object is released
92 * 1: no outstanding ubufs
93 * >1: outstanding ubufs
94 */
95 atomic_t refcount;
96 wait_queue_head_t wait;
97 struct vhost_virtqueue *vq;
98 };
99
100 #define VHOST_NET_BATCH 64
101 struct vhost_net_buf {
102 void **queue;
103 int tail;
104 int head;
105 };
106
107 struct vhost_net_virtqueue {
108 struct vhost_virtqueue vq;
109 size_t vhost_hlen;
110 size_t sock_hlen;
111 /* vhost zerocopy support fields below: */
112 /* last used idx for outstanding DMA zerocopy buffers */
113 int upend_idx;
114 /* For TX, first used idx for DMA done zerocopy buffers
115 * For RX, number of batched heads
116 */
117 int done_idx;
118 /* Number of XDP frames batched */
119 int batched_xdp;
120 /* an array of userspace buffers info */
121 struct ubuf_info *ubuf_info;
122 /* Reference counting for outstanding ubufs.
123 * Protected by vq mutex. Writers must also take device mutex. */
124 struct vhost_net_ubuf_ref *ubufs;
125 struct ptr_ring *rx_ring;
126 struct vhost_net_buf rxq;
127 /* Batched XDP buffs */
128 struct xdp_buff *xdp;
129 };
130
131 struct vhost_net {
132 struct vhost_dev dev;
133 struct vhost_net_virtqueue vqs[VHOST_NET_VQ_MAX];
134 struct vhost_poll poll[VHOST_NET_VQ_MAX];
135 /* Number of TX recently submitted.
136 * Protected by tx vq lock. */
137 unsigned tx_packets;
138 /* Number of times zerocopy TX recently failed.
139 * Protected by tx vq lock. */
140 unsigned tx_zcopy_err;
141 /* Flush in progress. Protected by tx vq lock. */
142 bool tx_flush;
143 /* Private page frag */
144 struct page_frag page_frag;
145 /* Refcount bias of page frag */
146 int refcnt_bias;
147 };
148
149 static unsigned vhost_net_zcopy_mask __read_mostly;
150
vhost_net_buf_get_ptr(struct vhost_net_buf * rxq)151 static void *vhost_net_buf_get_ptr(struct vhost_net_buf *rxq)
152 {
153 if (rxq->tail != rxq->head)
154 return rxq->queue[rxq->head];
155 else
156 return NULL;
157 }
158
vhost_net_buf_get_size(struct vhost_net_buf * rxq)159 static int vhost_net_buf_get_size(struct vhost_net_buf *rxq)
160 {
161 return rxq->tail - rxq->head;
162 }
163
vhost_net_buf_is_empty(struct vhost_net_buf * rxq)164 static int vhost_net_buf_is_empty(struct vhost_net_buf *rxq)
165 {
166 return rxq->tail == rxq->head;
167 }
168
vhost_net_buf_consume(struct vhost_net_buf * rxq)169 static void *vhost_net_buf_consume(struct vhost_net_buf *rxq)
170 {
171 void *ret = vhost_net_buf_get_ptr(rxq);
172 ++rxq->head;
173 return ret;
174 }
175
vhost_net_buf_produce(struct vhost_net_virtqueue * nvq)176 static int vhost_net_buf_produce(struct vhost_net_virtqueue *nvq)
177 {
178 struct vhost_net_buf *rxq = &nvq->rxq;
179
180 rxq->head = 0;
181 rxq->tail = ptr_ring_consume_batched(nvq->rx_ring, rxq->queue,
182 VHOST_NET_BATCH);
183 return rxq->tail;
184 }
185
vhost_net_buf_unproduce(struct vhost_net_virtqueue * nvq)186 static void vhost_net_buf_unproduce(struct vhost_net_virtqueue *nvq)
187 {
188 struct vhost_net_buf *rxq = &nvq->rxq;
189
190 if (nvq->rx_ring && !vhost_net_buf_is_empty(rxq)) {
191 ptr_ring_unconsume(nvq->rx_ring, rxq->queue + rxq->head,
192 vhost_net_buf_get_size(rxq),
193 tun_ptr_free);
194 rxq->head = rxq->tail = 0;
195 }
196 }
197
vhost_net_buf_peek_len(void * ptr)198 static int vhost_net_buf_peek_len(void *ptr)
199 {
200 if (tun_is_xdp_frame(ptr)) {
201 struct xdp_frame *xdpf = tun_ptr_to_xdp(ptr);
202
203 return xdpf->len;
204 }
205
206 return __skb_array_len_with_tag(ptr);
207 }
208
vhost_net_buf_peek(struct vhost_net_virtqueue * nvq)209 static int vhost_net_buf_peek(struct vhost_net_virtqueue *nvq)
210 {
211 struct vhost_net_buf *rxq = &nvq->rxq;
212
213 if (!vhost_net_buf_is_empty(rxq))
214 goto out;
215
216 if (!vhost_net_buf_produce(nvq))
217 return 0;
218
219 out:
220 return vhost_net_buf_peek_len(vhost_net_buf_get_ptr(rxq));
221 }
222
vhost_net_buf_init(struct vhost_net_buf * rxq)223 static void vhost_net_buf_init(struct vhost_net_buf *rxq)
224 {
225 rxq->head = rxq->tail = 0;
226 }
227
vhost_net_enable_zcopy(int vq)228 static void vhost_net_enable_zcopy(int vq)
229 {
230 vhost_net_zcopy_mask |= 0x1 << vq;
231 }
232
233 static struct vhost_net_ubuf_ref *
vhost_net_ubuf_alloc(struct vhost_virtqueue * vq,bool zcopy)234 vhost_net_ubuf_alloc(struct vhost_virtqueue *vq, bool zcopy)
235 {
236 struct vhost_net_ubuf_ref *ubufs;
237 /* No zero copy backend? Nothing to count. */
238 if (!zcopy)
239 return NULL;
240 ubufs = kmalloc(sizeof(*ubufs), GFP_KERNEL);
241 if (!ubufs)
242 return ERR_PTR(-ENOMEM);
243 atomic_set(&ubufs->refcount, 1);
244 init_waitqueue_head(&ubufs->wait);
245 ubufs->vq = vq;
246 return ubufs;
247 }
248
vhost_net_ubuf_put(struct vhost_net_ubuf_ref * ubufs)249 static int vhost_net_ubuf_put(struct vhost_net_ubuf_ref *ubufs)
250 {
251 int r = atomic_sub_return(1, &ubufs->refcount);
252 if (unlikely(!r))
253 wake_up(&ubufs->wait);
254 return r;
255 }
256
vhost_net_ubuf_put_and_wait(struct vhost_net_ubuf_ref * ubufs)257 static void vhost_net_ubuf_put_and_wait(struct vhost_net_ubuf_ref *ubufs)
258 {
259 vhost_net_ubuf_put(ubufs);
260 wait_event(ubufs->wait, !atomic_read(&ubufs->refcount));
261 }
262
vhost_net_ubuf_put_wait_and_free(struct vhost_net_ubuf_ref * ubufs)263 static void vhost_net_ubuf_put_wait_and_free(struct vhost_net_ubuf_ref *ubufs)
264 {
265 vhost_net_ubuf_put_and_wait(ubufs);
266 kfree(ubufs);
267 }
268
vhost_net_clear_ubuf_info(struct vhost_net * n)269 static void vhost_net_clear_ubuf_info(struct vhost_net *n)
270 {
271 int i;
272
273 for (i = 0; i < VHOST_NET_VQ_MAX; ++i) {
274 kfree(n->vqs[i].ubuf_info);
275 n->vqs[i].ubuf_info = NULL;
276 }
277 }
278
vhost_net_set_ubuf_info(struct vhost_net * n)279 static int vhost_net_set_ubuf_info(struct vhost_net *n)
280 {
281 bool zcopy;
282 int i;
283
284 for (i = 0; i < VHOST_NET_VQ_MAX; ++i) {
285 zcopy = vhost_net_zcopy_mask & (0x1 << i);
286 if (!zcopy)
287 continue;
288 n->vqs[i].ubuf_info =
289 kmalloc_array(UIO_MAXIOV,
290 sizeof(*n->vqs[i].ubuf_info),
291 GFP_KERNEL);
292 if (!n->vqs[i].ubuf_info)
293 goto err;
294 }
295 return 0;
296
297 err:
298 vhost_net_clear_ubuf_info(n);
299 return -ENOMEM;
300 }
301
vhost_net_vq_reset(struct vhost_net * n)302 static void vhost_net_vq_reset(struct vhost_net *n)
303 {
304 int i;
305
306 vhost_net_clear_ubuf_info(n);
307
308 for (i = 0; i < VHOST_NET_VQ_MAX; i++) {
309 n->vqs[i].done_idx = 0;
310 n->vqs[i].upend_idx = 0;
311 n->vqs[i].ubufs = NULL;
312 n->vqs[i].vhost_hlen = 0;
313 n->vqs[i].sock_hlen = 0;
314 vhost_net_buf_init(&n->vqs[i].rxq);
315 }
316
317 }
318
vhost_net_tx_packet(struct vhost_net * net)319 static void vhost_net_tx_packet(struct vhost_net *net)
320 {
321 ++net->tx_packets;
322 if (net->tx_packets < 1024)
323 return;
324 net->tx_packets = 0;
325 net->tx_zcopy_err = 0;
326 }
327
vhost_net_tx_err(struct vhost_net * net)328 static void vhost_net_tx_err(struct vhost_net *net)
329 {
330 ++net->tx_zcopy_err;
331 }
332
vhost_net_tx_select_zcopy(struct vhost_net * net)333 static bool vhost_net_tx_select_zcopy(struct vhost_net *net)
334 {
335 /* TX flush waits for outstanding DMAs to be done.
336 * Don't start new DMAs.
337 */
338 return !net->tx_flush &&
339 net->tx_packets / 64 >= net->tx_zcopy_err;
340 }
341
vhost_sock_zcopy(struct socket * sock)342 static bool vhost_sock_zcopy(struct socket *sock)
343 {
344 return unlikely(experimental_zcopytx) &&
345 sock_flag(sock->sk, SOCK_ZEROCOPY);
346 }
347
vhost_sock_xdp(struct socket * sock)348 static bool vhost_sock_xdp(struct socket *sock)
349 {
350 return sock_flag(sock->sk, SOCK_XDP);
351 }
352
353 /* In case of DMA done not in order in lower device driver for some reason.
354 * upend_idx is used to track end of used idx, done_idx is used to track head
355 * of used idx. Once lower device DMA done contiguously, we will signal KVM
356 * guest used idx.
357 */
vhost_zerocopy_signal_used(struct vhost_net * net,struct vhost_virtqueue * vq)358 static void vhost_zerocopy_signal_used(struct vhost_net *net,
359 struct vhost_virtqueue *vq)
360 {
361 struct vhost_net_virtqueue *nvq =
362 container_of(vq, struct vhost_net_virtqueue, vq);
363 int i, add;
364 int j = 0;
365
366 for (i = nvq->done_idx; i != nvq->upend_idx; i = (i + 1) % UIO_MAXIOV) {
367 if (vq->heads[i].len == VHOST_DMA_FAILED_LEN)
368 vhost_net_tx_err(net);
369 if (VHOST_DMA_IS_DONE(vq->heads[i].len)) {
370 vq->heads[i].len = VHOST_DMA_CLEAR_LEN;
371 ++j;
372 } else
373 break;
374 }
375 while (j) {
376 add = min(UIO_MAXIOV - nvq->done_idx, j);
377 vhost_add_used_and_signal_n(vq->dev, vq,
378 &vq->heads[nvq->done_idx], add);
379 nvq->done_idx = (nvq->done_idx + add) % UIO_MAXIOV;
380 j -= add;
381 }
382 }
383
vhost_zerocopy_callback(struct sk_buff * skb,struct ubuf_info * ubuf,bool success)384 static void vhost_zerocopy_callback(struct sk_buff *skb,
385 struct ubuf_info *ubuf, bool success)
386 {
387 struct vhost_net_ubuf_ref *ubufs = ubuf->ctx;
388 struct vhost_virtqueue *vq = ubufs->vq;
389 int cnt;
390
391 rcu_read_lock_bh();
392
393 /* set len to mark this desc buffers done DMA */
394 vq->heads[ubuf->desc].len = success ?
395 VHOST_DMA_DONE_LEN : VHOST_DMA_FAILED_LEN;
396 cnt = vhost_net_ubuf_put(ubufs);
397
398 /*
399 * Trigger polling thread if guest stopped submitting new buffers:
400 * in this case, the refcount after decrement will eventually reach 1.
401 * We also trigger polling periodically after each 16 packets
402 * (the value 16 here is more or less arbitrary, it's tuned to trigger
403 * less than 10% of times).
404 */
405 if (cnt <= 1 || !(cnt % 16))
406 vhost_poll_queue(&vq->poll);
407
408 rcu_read_unlock_bh();
409 }
410
busy_clock(void)411 static inline unsigned long busy_clock(void)
412 {
413 return local_clock() >> 10;
414 }
415
vhost_can_busy_poll(unsigned long endtime)416 static bool vhost_can_busy_poll(unsigned long endtime)
417 {
418 return likely(!need_resched() && !time_after(busy_clock(), endtime) &&
419 !signal_pending(current));
420 }
421
vhost_net_disable_vq(struct vhost_net * n,struct vhost_virtqueue * vq)422 static void vhost_net_disable_vq(struct vhost_net *n,
423 struct vhost_virtqueue *vq)
424 {
425 struct vhost_net_virtqueue *nvq =
426 container_of(vq, struct vhost_net_virtqueue, vq);
427 struct vhost_poll *poll = n->poll + (nvq - n->vqs);
428 if (!vhost_vq_get_backend(vq))
429 return;
430 vhost_poll_stop(poll);
431 }
432
vhost_net_enable_vq(struct vhost_net * n,struct vhost_virtqueue * vq)433 static int vhost_net_enable_vq(struct vhost_net *n,
434 struct vhost_virtqueue *vq)
435 {
436 struct vhost_net_virtqueue *nvq =
437 container_of(vq, struct vhost_net_virtqueue, vq);
438 struct vhost_poll *poll = n->poll + (nvq - n->vqs);
439 struct socket *sock;
440
441 sock = vhost_vq_get_backend(vq);
442 if (!sock)
443 return 0;
444
445 return vhost_poll_start(poll, sock->file);
446 }
447
vhost_net_signal_used(struct vhost_net_virtqueue * nvq)448 static void vhost_net_signal_used(struct vhost_net_virtqueue *nvq)
449 {
450 struct vhost_virtqueue *vq = &nvq->vq;
451 struct vhost_dev *dev = vq->dev;
452
453 if (!nvq->done_idx)
454 return;
455
456 vhost_add_used_and_signal_n(dev, vq, vq->heads, nvq->done_idx);
457 nvq->done_idx = 0;
458 }
459
vhost_tx_batch(struct vhost_net * net,struct vhost_net_virtqueue * nvq,struct socket * sock,struct msghdr * msghdr)460 static void vhost_tx_batch(struct vhost_net *net,
461 struct vhost_net_virtqueue *nvq,
462 struct socket *sock,
463 struct msghdr *msghdr)
464 {
465 struct tun_msg_ctl ctl = {
466 .type = TUN_MSG_PTR,
467 .num = nvq->batched_xdp,
468 .ptr = nvq->xdp,
469 };
470 int i, err;
471
472 if (nvq->batched_xdp == 0)
473 goto signal_used;
474
475 msghdr->msg_control = &ctl;
476 msghdr->msg_controllen = sizeof(ctl);
477 err = sock->ops->sendmsg(sock, msghdr, 0);
478 if (unlikely(err < 0)) {
479 vq_err(&nvq->vq, "Fail to batch sending packets\n");
480
481 /* free pages owned by XDP; since this is an unlikely error path,
482 * keep it simple and avoid more complex bulk update for the
483 * used pages
484 */
485 for (i = 0; i < nvq->batched_xdp; ++i)
486 put_page(virt_to_head_page(nvq->xdp[i].data));
487 nvq->batched_xdp = 0;
488 nvq->done_idx = 0;
489 return;
490 }
491
492 signal_used:
493 vhost_net_signal_used(nvq);
494 nvq->batched_xdp = 0;
495 }
496
sock_has_rx_data(struct socket * sock)497 static int sock_has_rx_data(struct socket *sock)
498 {
499 if (unlikely(!sock))
500 return 0;
501
502 if (sock->ops->peek_len)
503 return sock->ops->peek_len(sock);
504
505 return skb_queue_empty(&sock->sk->sk_receive_queue);
506 }
507
vhost_net_busy_poll_try_queue(struct vhost_net * net,struct vhost_virtqueue * vq)508 static void vhost_net_busy_poll_try_queue(struct vhost_net *net,
509 struct vhost_virtqueue *vq)
510 {
511 if (!vhost_vq_avail_empty(&net->dev, vq)) {
512 vhost_poll_queue(&vq->poll);
513 } else if (unlikely(vhost_enable_notify(&net->dev, vq))) {
514 vhost_disable_notify(&net->dev, vq);
515 vhost_poll_queue(&vq->poll);
516 }
517 }
518
vhost_net_busy_poll(struct vhost_net * net,struct vhost_virtqueue * rvq,struct vhost_virtqueue * tvq,bool * busyloop_intr,bool poll_rx)519 static void vhost_net_busy_poll(struct vhost_net *net,
520 struct vhost_virtqueue *rvq,
521 struct vhost_virtqueue *tvq,
522 bool *busyloop_intr,
523 bool poll_rx)
524 {
525 unsigned long busyloop_timeout;
526 unsigned long endtime;
527 struct socket *sock;
528 struct vhost_virtqueue *vq = poll_rx ? tvq : rvq;
529
530 /* Try to hold the vq mutex of the paired virtqueue. We can't
531 * use mutex_lock() here since we could not guarantee a
532 * consistenet lock ordering.
533 */
534 if (!mutex_trylock(&vq->mutex))
535 return;
536
537 vhost_disable_notify(&net->dev, vq);
538 sock = vhost_vq_get_backend(rvq);
539
540 busyloop_timeout = poll_rx ? rvq->busyloop_timeout:
541 tvq->busyloop_timeout;
542
543 preempt_disable();
544 endtime = busy_clock() + busyloop_timeout;
545
546 while (vhost_can_busy_poll(endtime)) {
547 if (vhost_has_work(&net->dev)) {
548 *busyloop_intr = true;
549 break;
550 }
551
552 if ((sock_has_rx_data(sock) &&
553 !vhost_vq_avail_empty(&net->dev, rvq)) ||
554 !vhost_vq_avail_empty(&net->dev, tvq))
555 break;
556
557 cpu_relax();
558 }
559
560 preempt_enable();
561
562 if (poll_rx || sock_has_rx_data(sock))
563 vhost_net_busy_poll_try_queue(net, vq);
564 else if (!poll_rx) /* On tx here, sock has no rx data. */
565 vhost_enable_notify(&net->dev, rvq);
566
567 mutex_unlock(&vq->mutex);
568 }
569
vhost_net_tx_get_vq_desc(struct vhost_net * net,struct vhost_net_virtqueue * tnvq,unsigned int * out_num,unsigned int * in_num,struct msghdr * msghdr,bool * busyloop_intr)570 static int vhost_net_tx_get_vq_desc(struct vhost_net *net,
571 struct vhost_net_virtqueue *tnvq,
572 unsigned int *out_num, unsigned int *in_num,
573 struct msghdr *msghdr, bool *busyloop_intr)
574 {
575 struct vhost_net_virtqueue *rnvq = &net->vqs[VHOST_NET_VQ_RX];
576 struct vhost_virtqueue *rvq = &rnvq->vq;
577 struct vhost_virtqueue *tvq = &tnvq->vq;
578
579 int r = vhost_get_vq_desc(tvq, tvq->iov, ARRAY_SIZE(tvq->iov),
580 out_num, in_num, NULL, NULL);
581
582 if (r == tvq->num && tvq->busyloop_timeout) {
583 /* Flush batched packets first */
584 if (!vhost_sock_zcopy(vhost_vq_get_backend(tvq)))
585 vhost_tx_batch(net, tnvq,
586 vhost_vq_get_backend(tvq),
587 msghdr);
588
589 vhost_net_busy_poll(net, rvq, tvq, busyloop_intr, false);
590
591 r = vhost_get_vq_desc(tvq, tvq->iov, ARRAY_SIZE(tvq->iov),
592 out_num, in_num, NULL, NULL);
593 }
594
595 return r;
596 }
597
vhost_exceeds_maxpend(struct vhost_net * net)598 static bool vhost_exceeds_maxpend(struct vhost_net *net)
599 {
600 struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_TX];
601 struct vhost_virtqueue *vq = &nvq->vq;
602
603 return (nvq->upend_idx + UIO_MAXIOV - nvq->done_idx) % UIO_MAXIOV >
604 min_t(unsigned int, VHOST_MAX_PEND, vq->num >> 2);
605 }
606
init_iov_iter(struct vhost_virtqueue * vq,struct iov_iter * iter,size_t hdr_size,int out)607 static size_t init_iov_iter(struct vhost_virtqueue *vq, struct iov_iter *iter,
608 size_t hdr_size, int out)
609 {
610 /* Skip header. TODO: support TSO. */
611 size_t len = iov_length(vq->iov, out);
612
613 iov_iter_init(iter, WRITE, vq->iov, out, len);
614 iov_iter_advance(iter, hdr_size);
615
616 return iov_iter_count(iter);
617 }
618
get_tx_bufs(struct vhost_net * net,struct vhost_net_virtqueue * nvq,struct msghdr * msg,unsigned int * out,unsigned int * in,size_t * len,bool * busyloop_intr)619 static int get_tx_bufs(struct vhost_net *net,
620 struct vhost_net_virtqueue *nvq,
621 struct msghdr *msg,
622 unsigned int *out, unsigned int *in,
623 size_t *len, bool *busyloop_intr)
624 {
625 struct vhost_virtqueue *vq = &nvq->vq;
626 int ret;
627
628 ret = vhost_net_tx_get_vq_desc(net, nvq, out, in, msg, busyloop_intr);
629
630 if (ret < 0 || ret == vq->num)
631 return ret;
632
633 if (*in) {
634 vq_err(vq, "Unexpected descriptor format for TX: out %d, int %d\n",
635 *out, *in);
636 return -EFAULT;
637 }
638
639 /* Sanity check */
640 *len = init_iov_iter(vq, &msg->msg_iter, nvq->vhost_hlen, *out);
641 if (*len == 0) {
642 vq_err(vq, "Unexpected header len for TX: %zd expected %zd\n",
643 *len, nvq->vhost_hlen);
644 return -EFAULT;
645 }
646
647 return ret;
648 }
649
tx_can_batch(struct vhost_virtqueue * vq,size_t total_len)650 static bool tx_can_batch(struct vhost_virtqueue *vq, size_t total_len)
651 {
652 return total_len < VHOST_NET_WEIGHT &&
653 !vhost_vq_avail_empty(vq->dev, vq);
654 }
655
vhost_net_page_frag_refill(struct vhost_net * net,unsigned int sz,struct page_frag * pfrag,gfp_t gfp)656 static bool vhost_net_page_frag_refill(struct vhost_net *net, unsigned int sz,
657 struct page_frag *pfrag, gfp_t gfp)
658 {
659 if (pfrag->page) {
660 if (pfrag->offset + sz <= pfrag->size)
661 return true;
662 __page_frag_cache_drain(pfrag->page, net->refcnt_bias);
663 }
664
665 pfrag->offset = 0;
666 net->refcnt_bias = 0;
667 if (SKB_FRAG_PAGE_ORDER) {
668 /* Avoid direct reclaim but allow kswapd to wake */
669 pfrag->page = alloc_pages((gfp & ~__GFP_DIRECT_RECLAIM) |
670 __GFP_COMP | __GFP_NOWARN |
671 __GFP_NORETRY,
672 SKB_FRAG_PAGE_ORDER);
673 if (likely(pfrag->page)) {
674 pfrag->size = PAGE_SIZE << SKB_FRAG_PAGE_ORDER;
675 goto done;
676 }
677 }
678 pfrag->page = alloc_page(gfp);
679 if (likely(pfrag->page)) {
680 pfrag->size = PAGE_SIZE;
681 goto done;
682 }
683 return false;
684
685 done:
686 net->refcnt_bias = USHRT_MAX;
687 page_ref_add(pfrag->page, USHRT_MAX - 1);
688 return true;
689 }
690
691 #define VHOST_NET_RX_PAD (NET_IP_ALIGN + NET_SKB_PAD)
692
vhost_net_build_xdp(struct vhost_net_virtqueue * nvq,struct iov_iter * from)693 static int vhost_net_build_xdp(struct vhost_net_virtqueue *nvq,
694 struct iov_iter *from)
695 {
696 struct vhost_virtqueue *vq = &nvq->vq;
697 struct vhost_net *net = container_of(vq->dev, struct vhost_net,
698 dev);
699 struct socket *sock = vhost_vq_get_backend(vq);
700 struct page_frag *alloc_frag = &net->page_frag;
701 struct virtio_net_hdr *gso;
702 struct xdp_buff *xdp = &nvq->xdp[nvq->batched_xdp];
703 struct tun_xdp_hdr *hdr;
704 size_t len = iov_iter_count(from);
705 int headroom = vhost_sock_xdp(sock) ? XDP_PACKET_HEADROOM : 0;
706 int buflen = SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
707 int pad = SKB_DATA_ALIGN(VHOST_NET_RX_PAD + headroom + nvq->sock_hlen);
708 int sock_hlen = nvq->sock_hlen;
709 void *buf;
710 int copied;
711
712 if (unlikely(len < nvq->sock_hlen))
713 return -EFAULT;
714
715 if (SKB_DATA_ALIGN(len + pad) +
716 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) > PAGE_SIZE)
717 return -ENOSPC;
718
719 buflen += SKB_DATA_ALIGN(len + pad);
720 alloc_frag->offset = ALIGN((u64)alloc_frag->offset, SMP_CACHE_BYTES);
721 if (unlikely(!vhost_net_page_frag_refill(net, buflen,
722 alloc_frag, GFP_KERNEL)))
723 return -ENOMEM;
724
725 buf = (char *)page_address(alloc_frag->page) + alloc_frag->offset;
726 copied = copy_page_from_iter(alloc_frag->page,
727 alloc_frag->offset +
728 offsetof(struct tun_xdp_hdr, gso),
729 sock_hlen, from);
730 if (copied != sock_hlen)
731 return -EFAULT;
732
733 hdr = buf;
734 gso = &hdr->gso;
735
736 if ((gso->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) &&
737 vhost16_to_cpu(vq, gso->csum_start) +
738 vhost16_to_cpu(vq, gso->csum_offset) + 2 >
739 vhost16_to_cpu(vq, gso->hdr_len)) {
740 gso->hdr_len = cpu_to_vhost16(vq,
741 vhost16_to_cpu(vq, gso->csum_start) +
742 vhost16_to_cpu(vq, gso->csum_offset) + 2);
743
744 if (vhost16_to_cpu(vq, gso->hdr_len) > len)
745 return -EINVAL;
746 }
747
748 len -= sock_hlen;
749 copied = copy_page_from_iter(alloc_frag->page,
750 alloc_frag->offset + pad,
751 len, from);
752 if (copied != len)
753 return -EFAULT;
754
755 xdp_init_buff(xdp, buflen, NULL);
756 xdp_prepare_buff(xdp, buf, pad, len, true);
757 hdr->buflen = buflen;
758
759 --net->refcnt_bias;
760 alloc_frag->offset += buflen;
761
762 ++nvq->batched_xdp;
763
764 return 0;
765 }
766
handle_tx_copy(struct vhost_net * net,struct socket * sock)767 static void handle_tx_copy(struct vhost_net *net, struct socket *sock)
768 {
769 struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_TX];
770 struct vhost_virtqueue *vq = &nvq->vq;
771 unsigned out, in;
772 int head;
773 struct msghdr msg = {
774 .msg_name = NULL,
775 .msg_namelen = 0,
776 .msg_control = NULL,
777 .msg_controllen = 0,
778 .msg_flags = MSG_DONTWAIT,
779 };
780 size_t len, total_len = 0;
781 int err;
782 int sent_pkts = 0;
783 bool sock_can_batch = (sock->sk->sk_sndbuf == INT_MAX);
784
785 do {
786 bool busyloop_intr = false;
787
788 if (nvq->done_idx == VHOST_NET_BATCH)
789 vhost_tx_batch(net, nvq, sock, &msg);
790
791 head = get_tx_bufs(net, nvq, &msg, &out, &in, &len,
792 &busyloop_intr);
793 /* On error, stop handling until the next kick. */
794 if (unlikely(head < 0))
795 break;
796 /* Nothing new? Wait for eventfd to tell us they refilled. */
797 if (head == vq->num) {
798 if (unlikely(busyloop_intr)) {
799 vhost_poll_queue(&vq->poll);
800 } else if (unlikely(vhost_enable_notify(&net->dev,
801 vq))) {
802 vhost_disable_notify(&net->dev, vq);
803 continue;
804 }
805 break;
806 }
807
808 total_len += len;
809
810 /* For simplicity, TX batching is only enabled if
811 * sndbuf is unlimited.
812 */
813 if (sock_can_batch) {
814 err = vhost_net_build_xdp(nvq, &msg.msg_iter);
815 if (!err) {
816 goto done;
817 } else if (unlikely(err != -ENOSPC)) {
818 vhost_tx_batch(net, nvq, sock, &msg);
819 vhost_discard_vq_desc(vq, 1);
820 vhost_net_enable_vq(net, vq);
821 break;
822 }
823
824 /* We can't build XDP buff, go for single
825 * packet path but let's flush batched
826 * packets.
827 */
828 vhost_tx_batch(net, nvq, sock, &msg);
829 msg.msg_control = NULL;
830 } else {
831 if (tx_can_batch(vq, total_len))
832 msg.msg_flags |= MSG_MORE;
833 else
834 msg.msg_flags &= ~MSG_MORE;
835 }
836
837 err = sock->ops->sendmsg(sock, &msg, len);
838 if (unlikely(err < 0)) {
839 if (err == -EAGAIN || err == -ENOMEM || err == -ENOBUFS) {
840 vhost_discard_vq_desc(vq, 1);
841 vhost_net_enable_vq(net, vq);
842 break;
843 }
844 pr_debug("Fail to send packet: err %d", err);
845 } else if (unlikely(err != len))
846 pr_debug("Truncated TX packet: len %d != %zd\n",
847 err, len);
848 done:
849 vq->heads[nvq->done_idx].id = cpu_to_vhost32(vq, head);
850 vq->heads[nvq->done_idx].len = 0;
851 ++nvq->done_idx;
852 } while (likely(!vhost_exceeds_weight(vq, ++sent_pkts, total_len)));
853
854 vhost_tx_batch(net, nvq, sock, &msg);
855 }
856
handle_tx_zerocopy(struct vhost_net * net,struct socket * sock)857 static void handle_tx_zerocopy(struct vhost_net *net, struct socket *sock)
858 {
859 struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_TX];
860 struct vhost_virtqueue *vq = &nvq->vq;
861 unsigned out, in;
862 int head;
863 struct msghdr msg = {
864 .msg_name = NULL,
865 .msg_namelen = 0,
866 .msg_control = NULL,
867 .msg_controllen = 0,
868 .msg_flags = MSG_DONTWAIT,
869 };
870 struct tun_msg_ctl ctl;
871 size_t len, total_len = 0;
872 int err;
873 struct vhost_net_ubuf_ref *ubufs;
874 struct ubuf_info *ubuf;
875 bool zcopy_used;
876 int sent_pkts = 0;
877
878 do {
879 bool busyloop_intr;
880
881 /* Release DMAs done buffers first */
882 vhost_zerocopy_signal_used(net, vq);
883
884 busyloop_intr = false;
885 head = get_tx_bufs(net, nvq, &msg, &out, &in, &len,
886 &busyloop_intr);
887 /* On error, stop handling until the next kick. */
888 if (unlikely(head < 0))
889 break;
890 /* Nothing new? Wait for eventfd to tell us they refilled. */
891 if (head == vq->num) {
892 if (unlikely(busyloop_intr)) {
893 vhost_poll_queue(&vq->poll);
894 } else if (unlikely(vhost_enable_notify(&net->dev, vq))) {
895 vhost_disable_notify(&net->dev, vq);
896 continue;
897 }
898 break;
899 }
900
901 zcopy_used = len >= VHOST_GOODCOPY_LEN
902 && !vhost_exceeds_maxpend(net)
903 && vhost_net_tx_select_zcopy(net);
904
905 /* use msg_control to pass vhost zerocopy ubuf info to skb */
906 if (zcopy_used) {
907 ubuf = nvq->ubuf_info + nvq->upend_idx;
908 vq->heads[nvq->upend_idx].id = cpu_to_vhost32(vq, head);
909 vq->heads[nvq->upend_idx].len = VHOST_DMA_IN_PROGRESS;
910 ubuf->callback = vhost_zerocopy_callback;
911 ubuf->ctx = nvq->ubufs;
912 ubuf->desc = nvq->upend_idx;
913 ubuf->flags = SKBFL_ZEROCOPY_FRAG;
914 refcount_set(&ubuf->refcnt, 1);
915 msg.msg_control = &ctl;
916 ctl.type = TUN_MSG_UBUF;
917 ctl.ptr = ubuf;
918 msg.msg_controllen = sizeof(ctl);
919 ubufs = nvq->ubufs;
920 atomic_inc(&ubufs->refcount);
921 nvq->upend_idx = (nvq->upend_idx + 1) % UIO_MAXIOV;
922 } else {
923 msg.msg_control = NULL;
924 ubufs = NULL;
925 }
926 total_len += len;
927 if (tx_can_batch(vq, total_len) &&
928 likely(!vhost_exceeds_maxpend(net))) {
929 msg.msg_flags |= MSG_MORE;
930 } else {
931 msg.msg_flags &= ~MSG_MORE;
932 }
933
934 err = sock->ops->sendmsg(sock, &msg, len);
935 if (unlikely(err < 0)) {
936 bool retry = err == -EAGAIN || err == -ENOMEM || err == -ENOBUFS;
937
938 if (zcopy_used) {
939 if (vq->heads[ubuf->desc].len == VHOST_DMA_IN_PROGRESS)
940 vhost_net_ubuf_put(ubufs);
941 if (retry)
942 nvq->upend_idx = ((unsigned)nvq->upend_idx - 1)
943 % UIO_MAXIOV;
944 else
945 vq->heads[ubuf->desc].len = VHOST_DMA_DONE_LEN;
946 }
947 if (retry) {
948 vhost_discard_vq_desc(vq, 1);
949 vhost_net_enable_vq(net, vq);
950 break;
951 }
952 pr_debug("Fail to send packet: err %d", err);
953 } else if (unlikely(err != len))
954 pr_debug("Truncated TX packet: "
955 " len %d != %zd\n", err, len);
956 if (!zcopy_used)
957 vhost_add_used_and_signal(&net->dev, vq, head, 0);
958 else
959 vhost_zerocopy_signal_used(net, vq);
960 vhost_net_tx_packet(net);
961 } while (likely(!vhost_exceeds_weight(vq, ++sent_pkts, total_len)));
962 }
963
964 /* Expects to be always run from workqueue - which acts as
965 * read-size critical section for our kind of RCU. */
handle_tx(struct vhost_net * net)966 static void handle_tx(struct vhost_net *net)
967 {
968 struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_TX];
969 struct vhost_virtqueue *vq = &nvq->vq;
970 struct socket *sock;
971
972 mutex_lock_nested(&vq->mutex, VHOST_NET_VQ_TX);
973 sock = vhost_vq_get_backend(vq);
974 if (!sock)
975 goto out;
976
977 if (!vq_meta_prefetch(vq))
978 goto out;
979
980 vhost_disable_notify(&net->dev, vq);
981 vhost_net_disable_vq(net, vq);
982
983 if (vhost_sock_zcopy(sock))
984 handle_tx_zerocopy(net, sock);
985 else
986 handle_tx_copy(net, sock);
987
988 out:
989 mutex_unlock(&vq->mutex);
990 }
991
peek_head_len(struct vhost_net_virtqueue * rvq,struct sock * sk)992 static int peek_head_len(struct vhost_net_virtqueue *rvq, struct sock *sk)
993 {
994 struct sk_buff *head;
995 int len = 0;
996 unsigned long flags;
997
998 if (rvq->rx_ring)
999 return vhost_net_buf_peek(rvq);
1000
1001 spin_lock_irqsave(&sk->sk_receive_queue.lock, flags);
1002 head = skb_peek(&sk->sk_receive_queue);
1003 if (likely(head)) {
1004 len = head->len;
1005 if (skb_vlan_tag_present(head))
1006 len += VLAN_HLEN;
1007 }
1008
1009 spin_unlock_irqrestore(&sk->sk_receive_queue.lock, flags);
1010 return len;
1011 }
1012
vhost_net_rx_peek_head_len(struct vhost_net * net,struct sock * sk,bool * busyloop_intr)1013 static int vhost_net_rx_peek_head_len(struct vhost_net *net, struct sock *sk,
1014 bool *busyloop_intr)
1015 {
1016 struct vhost_net_virtqueue *rnvq = &net->vqs[VHOST_NET_VQ_RX];
1017 struct vhost_net_virtqueue *tnvq = &net->vqs[VHOST_NET_VQ_TX];
1018 struct vhost_virtqueue *rvq = &rnvq->vq;
1019 struct vhost_virtqueue *tvq = &tnvq->vq;
1020 int len = peek_head_len(rnvq, sk);
1021
1022 if (!len && rvq->busyloop_timeout) {
1023 /* Flush batched heads first */
1024 vhost_net_signal_used(rnvq);
1025 /* Both tx vq and rx socket were polled here */
1026 vhost_net_busy_poll(net, rvq, tvq, busyloop_intr, true);
1027
1028 len = peek_head_len(rnvq, sk);
1029 }
1030
1031 return len;
1032 }
1033
1034 /* This is a multi-buffer version of vhost_get_desc, that works if
1035 * vq has read descriptors only.
1036 * @vq - the relevant virtqueue
1037 * @datalen - data length we'll be reading
1038 * @iovcount - returned count of io vectors we fill
1039 * @log - vhost log
1040 * @log_num - log offset
1041 * @quota - headcount quota, 1 for big buffer
1042 * returns number of buffer heads allocated, negative on error
1043 */
get_rx_bufs(struct vhost_virtqueue * vq,struct vring_used_elem * heads,int datalen,unsigned * iovcount,struct vhost_log * log,unsigned * log_num,unsigned int quota)1044 static int get_rx_bufs(struct vhost_virtqueue *vq,
1045 struct vring_used_elem *heads,
1046 int datalen,
1047 unsigned *iovcount,
1048 struct vhost_log *log,
1049 unsigned *log_num,
1050 unsigned int quota)
1051 {
1052 unsigned int out, in;
1053 int seg = 0;
1054 int headcount = 0;
1055 unsigned d;
1056 int r, nlogs = 0;
1057 /* len is always initialized before use since we are always called with
1058 * datalen > 0.
1059 */
1060 u32 len;
1061
1062 while (datalen > 0 && headcount < quota) {
1063 if (unlikely(seg >= UIO_MAXIOV)) {
1064 r = -ENOBUFS;
1065 goto err;
1066 }
1067 r = vhost_get_vq_desc(vq, vq->iov + seg,
1068 ARRAY_SIZE(vq->iov) - seg, &out,
1069 &in, log, log_num);
1070 if (unlikely(r < 0))
1071 goto err;
1072
1073 d = r;
1074 if (d == vq->num) {
1075 r = 0;
1076 goto err;
1077 }
1078 if (unlikely(out || in <= 0)) {
1079 vq_err(vq, "unexpected descriptor format for RX: "
1080 "out %d, in %d\n", out, in);
1081 r = -EINVAL;
1082 goto err;
1083 }
1084 if (unlikely(log)) {
1085 nlogs += *log_num;
1086 log += *log_num;
1087 }
1088 heads[headcount].id = cpu_to_vhost32(vq, d);
1089 len = iov_length(vq->iov + seg, in);
1090 heads[headcount].len = cpu_to_vhost32(vq, len);
1091 datalen -= len;
1092 ++headcount;
1093 seg += in;
1094 }
1095 heads[headcount - 1].len = cpu_to_vhost32(vq, len + datalen);
1096 *iovcount = seg;
1097 if (unlikely(log))
1098 *log_num = nlogs;
1099
1100 /* Detect overrun */
1101 if (unlikely(datalen > 0)) {
1102 r = UIO_MAXIOV + 1;
1103 goto err;
1104 }
1105 return headcount;
1106 err:
1107 vhost_discard_vq_desc(vq, headcount);
1108 return r;
1109 }
1110
1111 /* Expects to be always run from workqueue - which acts as
1112 * read-size critical section for our kind of RCU. */
handle_rx(struct vhost_net * net)1113 static void handle_rx(struct vhost_net *net)
1114 {
1115 struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_RX];
1116 struct vhost_virtqueue *vq = &nvq->vq;
1117 unsigned in, log;
1118 struct vhost_log *vq_log;
1119 struct msghdr msg = {
1120 .msg_name = NULL,
1121 .msg_namelen = 0,
1122 .msg_control = NULL, /* FIXME: get and handle RX aux data. */
1123 .msg_controllen = 0,
1124 .msg_flags = MSG_DONTWAIT,
1125 };
1126 struct virtio_net_hdr hdr = {
1127 .flags = 0,
1128 .gso_type = VIRTIO_NET_HDR_GSO_NONE
1129 };
1130 size_t total_len = 0;
1131 int err, mergeable;
1132 s16 headcount;
1133 size_t vhost_hlen, sock_hlen;
1134 size_t vhost_len, sock_len;
1135 bool busyloop_intr = false;
1136 struct socket *sock;
1137 struct iov_iter fixup;
1138 __virtio16 num_buffers;
1139 int recv_pkts = 0;
1140
1141 mutex_lock_nested(&vq->mutex, VHOST_NET_VQ_RX);
1142 sock = vhost_vq_get_backend(vq);
1143 if (!sock)
1144 goto out;
1145
1146 if (!vq_meta_prefetch(vq))
1147 goto out;
1148
1149 vhost_disable_notify(&net->dev, vq);
1150 vhost_net_disable_vq(net, vq);
1151
1152 vhost_hlen = nvq->vhost_hlen;
1153 sock_hlen = nvq->sock_hlen;
1154
1155 vq_log = unlikely(vhost_has_feature(vq, VHOST_F_LOG_ALL)) ?
1156 vq->log : NULL;
1157 mergeable = vhost_has_feature(vq, VIRTIO_NET_F_MRG_RXBUF);
1158
1159 do {
1160 sock_len = vhost_net_rx_peek_head_len(net, sock->sk,
1161 &busyloop_intr);
1162 if (!sock_len)
1163 break;
1164 sock_len += sock_hlen;
1165 vhost_len = sock_len + vhost_hlen;
1166 headcount = get_rx_bufs(vq, vq->heads + nvq->done_idx,
1167 vhost_len, &in, vq_log, &log,
1168 likely(mergeable) ? UIO_MAXIOV : 1);
1169 /* On error, stop handling until the next kick. */
1170 if (unlikely(headcount < 0))
1171 goto out;
1172 /* OK, now we need to know about added descriptors. */
1173 if (!headcount) {
1174 if (unlikely(busyloop_intr)) {
1175 vhost_poll_queue(&vq->poll);
1176 } else if (unlikely(vhost_enable_notify(&net->dev, vq))) {
1177 /* They have slipped one in as we were
1178 * doing that: check again. */
1179 vhost_disable_notify(&net->dev, vq);
1180 continue;
1181 }
1182 /* Nothing new? Wait for eventfd to tell us
1183 * they refilled. */
1184 goto out;
1185 }
1186 busyloop_intr = false;
1187 if (nvq->rx_ring)
1188 msg.msg_control = vhost_net_buf_consume(&nvq->rxq);
1189 /* On overrun, truncate and discard */
1190 if (unlikely(headcount > UIO_MAXIOV)) {
1191 iov_iter_init(&msg.msg_iter, READ, vq->iov, 1, 1);
1192 err = sock->ops->recvmsg(sock, &msg,
1193 1, MSG_DONTWAIT | MSG_TRUNC);
1194 pr_debug("Discarded rx packet: len %zd\n", sock_len);
1195 continue;
1196 }
1197 /* We don't need to be notified again. */
1198 iov_iter_init(&msg.msg_iter, READ, vq->iov, in, vhost_len);
1199 fixup = msg.msg_iter;
1200 if (unlikely((vhost_hlen))) {
1201 /* We will supply the header ourselves
1202 * TODO: support TSO.
1203 */
1204 iov_iter_advance(&msg.msg_iter, vhost_hlen);
1205 }
1206 err = sock->ops->recvmsg(sock, &msg,
1207 sock_len, MSG_DONTWAIT | MSG_TRUNC);
1208 /* Userspace might have consumed the packet meanwhile:
1209 * it's not supposed to do this usually, but might be hard
1210 * to prevent. Discard data we got (if any) and keep going. */
1211 if (unlikely(err != sock_len)) {
1212 pr_debug("Discarded rx packet: "
1213 " len %d, expected %zd\n", err, sock_len);
1214 vhost_discard_vq_desc(vq, headcount);
1215 continue;
1216 }
1217 /* Supply virtio_net_hdr if VHOST_NET_F_VIRTIO_NET_HDR */
1218 if (unlikely(vhost_hlen)) {
1219 if (copy_to_iter(&hdr, sizeof(hdr),
1220 &fixup) != sizeof(hdr)) {
1221 vq_err(vq, "Unable to write vnet_hdr "
1222 "at addr %p\n", vq->iov->iov_base);
1223 goto out;
1224 }
1225 } else {
1226 /* Header came from socket; we'll need to patch
1227 * ->num_buffers over if VIRTIO_NET_F_MRG_RXBUF
1228 */
1229 iov_iter_advance(&fixup, sizeof(hdr));
1230 }
1231 /* TODO: Should check and handle checksum. */
1232
1233 num_buffers = cpu_to_vhost16(vq, headcount);
1234 if (likely(mergeable) &&
1235 copy_to_iter(&num_buffers, sizeof num_buffers,
1236 &fixup) != sizeof num_buffers) {
1237 vq_err(vq, "Failed num_buffers write");
1238 vhost_discard_vq_desc(vq, headcount);
1239 goto out;
1240 }
1241 nvq->done_idx += headcount;
1242 if (nvq->done_idx > VHOST_NET_BATCH)
1243 vhost_net_signal_used(nvq);
1244 if (unlikely(vq_log))
1245 vhost_log_write(vq, vq_log, log, vhost_len,
1246 vq->iov, in);
1247 total_len += vhost_len;
1248 } while (likely(!vhost_exceeds_weight(vq, ++recv_pkts, total_len)));
1249
1250 if (unlikely(busyloop_intr))
1251 vhost_poll_queue(&vq->poll);
1252 else if (!sock_len)
1253 vhost_net_enable_vq(net, vq);
1254 out:
1255 vhost_net_signal_used(nvq);
1256 mutex_unlock(&vq->mutex);
1257 }
1258
handle_tx_kick(struct vhost_work * work)1259 static void handle_tx_kick(struct vhost_work *work)
1260 {
1261 struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,
1262 poll.work);
1263 struct vhost_net *net = container_of(vq->dev, struct vhost_net, dev);
1264
1265 handle_tx(net);
1266 }
1267
handle_rx_kick(struct vhost_work * work)1268 static void handle_rx_kick(struct vhost_work *work)
1269 {
1270 struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,
1271 poll.work);
1272 struct vhost_net *net = container_of(vq->dev, struct vhost_net, dev);
1273
1274 handle_rx(net);
1275 }
1276
handle_tx_net(struct vhost_work * work)1277 static void handle_tx_net(struct vhost_work *work)
1278 {
1279 struct vhost_net *net = container_of(work, struct vhost_net,
1280 poll[VHOST_NET_VQ_TX].work);
1281 handle_tx(net);
1282 }
1283
handle_rx_net(struct vhost_work * work)1284 static void handle_rx_net(struct vhost_work *work)
1285 {
1286 struct vhost_net *net = container_of(work, struct vhost_net,
1287 poll[VHOST_NET_VQ_RX].work);
1288 handle_rx(net);
1289 }
1290
vhost_net_open(struct inode * inode,struct file * f)1291 static int vhost_net_open(struct inode *inode, struct file *f)
1292 {
1293 struct vhost_net *n;
1294 struct vhost_dev *dev;
1295 struct vhost_virtqueue **vqs;
1296 void **queue;
1297 struct xdp_buff *xdp;
1298 int i;
1299
1300 n = kvmalloc(sizeof *n, GFP_KERNEL | __GFP_RETRY_MAYFAIL);
1301 if (!n)
1302 return -ENOMEM;
1303 vqs = kmalloc_array(VHOST_NET_VQ_MAX, sizeof(*vqs), GFP_KERNEL);
1304 if (!vqs) {
1305 kvfree(n);
1306 return -ENOMEM;
1307 }
1308
1309 queue = kmalloc_array(VHOST_NET_BATCH, sizeof(void *),
1310 GFP_KERNEL);
1311 if (!queue) {
1312 kfree(vqs);
1313 kvfree(n);
1314 return -ENOMEM;
1315 }
1316 n->vqs[VHOST_NET_VQ_RX].rxq.queue = queue;
1317
1318 xdp = kmalloc_array(VHOST_NET_BATCH, sizeof(*xdp), GFP_KERNEL);
1319 if (!xdp) {
1320 kfree(vqs);
1321 kvfree(n);
1322 kfree(queue);
1323 return -ENOMEM;
1324 }
1325 n->vqs[VHOST_NET_VQ_TX].xdp = xdp;
1326
1327 dev = &n->dev;
1328 vqs[VHOST_NET_VQ_TX] = &n->vqs[VHOST_NET_VQ_TX].vq;
1329 vqs[VHOST_NET_VQ_RX] = &n->vqs[VHOST_NET_VQ_RX].vq;
1330 n->vqs[VHOST_NET_VQ_TX].vq.handle_kick = handle_tx_kick;
1331 n->vqs[VHOST_NET_VQ_RX].vq.handle_kick = handle_rx_kick;
1332 for (i = 0; i < VHOST_NET_VQ_MAX; i++) {
1333 n->vqs[i].ubufs = NULL;
1334 n->vqs[i].ubuf_info = NULL;
1335 n->vqs[i].upend_idx = 0;
1336 n->vqs[i].done_idx = 0;
1337 n->vqs[i].batched_xdp = 0;
1338 n->vqs[i].vhost_hlen = 0;
1339 n->vqs[i].sock_hlen = 0;
1340 n->vqs[i].rx_ring = NULL;
1341 vhost_net_buf_init(&n->vqs[i].rxq);
1342 }
1343 vhost_dev_init(dev, vqs, VHOST_NET_VQ_MAX,
1344 UIO_MAXIOV + VHOST_NET_BATCH,
1345 VHOST_NET_PKT_WEIGHT, VHOST_NET_WEIGHT, true,
1346 NULL);
1347
1348 vhost_poll_init(n->poll + VHOST_NET_VQ_TX, handle_tx_net, EPOLLOUT, dev);
1349 vhost_poll_init(n->poll + VHOST_NET_VQ_RX, handle_rx_net, EPOLLIN, dev);
1350
1351 f->private_data = n;
1352 n->page_frag.page = NULL;
1353 n->refcnt_bias = 0;
1354
1355 return 0;
1356 }
1357
vhost_net_stop_vq(struct vhost_net * n,struct vhost_virtqueue * vq)1358 static struct socket *vhost_net_stop_vq(struct vhost_net *n,
1359 struct vhost_virtqueue *vq)
1360 {
1361 struct socket *sock;
1362 struct vhost_net_virtqueue *nvq =
1363 container_of(vq, struct vhost_net_virtqueue, vq);
1364
1365 mutex_lock(&vq->mutex);
1366 sock = vhost_vq_get_backend(vq);
1367 vhost_net_disable_vq(n, vq);
1368 vhost_vq_set_backend(vq, NULL);
1369 vhost_net_buf_unproduce(nvq);
1370 nvq->rx_ring = NULL;
1371 mutex_unlock(&vq->mutex);
1372 return sock;
1373 }
1374
vhost_net_stop(struct vhost_net * n,struct socket ** tx_sock,struct socket ** rx_sock)1375 static void vhost_net_stop(struct vhost_net *n, struct socket **tx_sock,
1376 struct socket **rx_sock)
1377 {
1378 *tx_sock = vhost_net_stop_vq(n, &n->vqs[VHOST_NET_VQ_TX].vq);
1379 *rx_sock = vhost_net_stop_vq(n, &n->vqs[VHOST_NET_VQ_RX].vq);
1380 }
1381
vhost_net_flush_vq(struct vhost_net * n,int index)1382 static void vhost_net_flush_vq(struct vhost_net *n, int index)
1383 {
1384 vhost_poll_flush(n->poll + index);
1385 vhost_poll_flush(&n->vqs[index].vq.poll);
1386 }
1387
vhost_net_flush(struct vhost_net * n)1388 static void vhost_net_flush(struct vhost_net *n)
1389 {
1390 vhost_net_flush_vq(n, VHOST_NET_VQ_TX);
1391 vhost_net_flush_vq(n, VHOST_NET_VQ_RX);
1392 if (n->vqs[VHOST_NET_VQ_TX].ubufs) {
1393 mutex_lock(&n->vqs[VHOST_NET_VQ_TX].vq.mutex);
1394 n->tx_flush = true;
1395 mutex_unlock(&n->vqs[VHOST_NET_VQ_TX].vq.mutex);
1396 /* Wait for all lower device DMAs done. */
1397 vhost_net_ubuf_put_and_wait(n->vqs[VHOST_NET_VQ_TX].ubufs);
1398 mutex_lock(&n->vqs[VHOST_NET_VQ_TX].vq.mutex);
1399 n->tx_flush = false;
1400 atomic_set(&n->vqs[VHOST_NET_VQ_TX].ubufs->refcount, 1);
1401 mutex_unlock(&n->vqs[VHOST_NET_VQ_TX].vq.mutex);
1402 }
1403 }
1404
vhost_net_release(struct inode * inode,struct file * f)1405 static int vhost_net_release(struct inode *inode, struct file *f)
1406 {
1407 struct vhost_net *n = f->private_data;
1408 struct socket *tx_sock;
1409 struct socket *rx_sock;
1410
1411 vhost_net_stop(n, &tx_sock, &rx_sock);
1412 vhost_net_flush(n);
1413 vhost_dev_stop(&n->dev);
1414 vhost_dev_cleanup(&n->dev);
1415 vhost_net_vq_reset(n);
1416 if (tx_sock)
1417 sockfd_put(tx_sock);
1418 if (rx_sock)
1419 sockfd_put(rx_sock);
1420 /* Make sure no callbacks are outstanding */
1421 synchronize_rcu();
1422 /* We do an extra flush before freeing memory,
1423 * since jobs can re-queue themselves. */
1424 vhost_net_flush(n);
1425 kfree(n->vqs[VHOST_NET_VQ_RX].rxq.queue);
1426 kfree(n->vqs[VHOST_NET_VQ_TX].xdp);
1427 kfree(n->dev.vqs);
1428 if (n->page_frag.page)
1429 __page_frag_cache_drain(n->page_frag.page, n->refcnt_bias);
1430 kvfree(n);
1431 return 0;
1432 }
1433
get_raw_socket(int fd)1434 static struct socket *get_raw_socket(int fd)
1435 {
1436 int r;
1437 struct socket *sock = sockfd_lookup(fd, &r);
1438
1439 if (!sock)
1440 return ERR_PTR(-ENOTSOCK);
1441
1442 /* Parameter checking */
1443 if (sock->sk->sk_type != SOCK_RAW) {
1444 r = -ESOCKTNOSUPPORT;
1445 goto err;
1446 }
1447
1448 if (sock->sk->sk_family != AF_PACKET) {
1449 r = -EPFNOSUPPORT;
1450 goto err;
1451 }
1452 return sock;
1453 err:
1454 sockfd_put(sock);
1455 return ERR_PTR(r);
1456 }
1457
get_tap_ptr_ring(struct file * file)1458 static struct ptr_ring *get_tap_ptr_ring(struct file *file)
1459 {
1460 struct ptr_ring *ring;
1461 ring = tun_get_tx_ring(file);
1462 if (!IS_ERR(ring))
1463 goto out;
1464 ring = tap_get_ptr_ring(file);
1465 if (!IS_ERR(ring))
1466 goto out;
1467 ring = NULL;
1468 out:
1469 return ring;
1470 }
1471
get_tap_socket(int fd)1472 static struct socket *get_tap_socket(int fd)
1473 {
1474 struct file *file = fget(fd);
1475 struct socket *sock;
1476
1477 if (!file)
1478 return ERR_PTR(-EBADF);
1479 sock = tun_get_socket(file);
1480 if (!IS_ERR(sock))
1481 return sock;
1482 sock = tap_get_socket(file);
1483 if (IS_ERR(sock))
1484 fput(file);
1485 return sock;
1486 }
1487
get_socket(int fd)1488 static struct socket *get_socket(int fd)
1489 {
1490 struct socket *sock;
1491
1492 /* special case to disable backend */
1493 if (fd == -1)
1494 return NULL;
1495 sock = get_raw_socket(fd);
1496 if (!IS_ERR(sock))
1497 return sock;
1498 sock = get_tap_socket(fd);
1499 if (!IS_ERR(sock))
1500 return sock;
1501 return ERR_PTR(-ENOTSOCK);
1502 }
1503
vhost_net_set_backend(struct vhost_net * n,unsigned index,int fd)1504 static long vhost_net_set_backend(struct vhost_net *n, unsigned index, int fd)
1505 {
1506 struct socket *sock, *oldsock;
1507 struct vhost_virtqueue *vq;
1508 struct vhost_net_virtqueue *nvq;
1509 struct vhost_net_ubuf_ref *ubufs, *oldubufs = NULL;
1510 int r;
1511
1512 mutex_lock(&n->dev.mutex);
1513 r = vhost_dev_check_owner(&n->dev);
1514 if (r)
1515 goto err;
1516
1517 if (index >= VHOST_NET_VQ_MAX) {
1518 r = -ENOBUFS;
1519 goto err;
1520 }
1521 vq = &n->vqs[index].vq;
1522 nvq = &n->vqs[index];
1523 mutex_lock(&vq->mutex);
1524
1525 if (fd == -1)
1526 vhost_clear_msg(&n->dev);
1527
1528 /* Verify that ring has been setup correctly. */
1529 if (!vhost_vq_access_ok(vq)) {
1530 r = -EFAULT;
1531 goto err_vq;
1532 }
1533 sock = get_socket(fd);
1534 if (IS_ERR(sock)) {
1535 r = PTR_ERR(sock);
1536 goto err_vq;
1537 }
1538
1539 /* start polling new socket */
1540 oldsock = vhost_vq_get_backend(vq);
1541 if (sock != oldsock) {
1542 ubufs = vhost_net_ubuf_alloc(vq,
1543 sock && vhost_sock_zcopy(sock));
1544 if (IS_ERR(ubufs)) {
1545 r = PTR_ERR(ubufs);
1546 goto err_ubufs;
1547 }
1548
1549 vhost_net_disable_vq(n, vq);
1550 vhost_vq_set_backend(vq, sock);
1551 vhost_net_buf_unproduce(nvq);
1552 r = vhost_vq_init_access(vq);
1553 if (r)
1554 goto err_used;
1555 r = vhost_net_enable_vq(n, vq);
1556 if (r)
1557 goto err_used;
1558 if (index == VHOST_NET_VQ_RX) {
1559 if (sock)
1560 nvq->rx_ring = get_tap_ptr_ring(sock->file);
1561 else
1562 nvq->rx_ring = NULL;
1563 }
1564
1565 oldubufs = nvq->ubufs;
1566 nvq->ubufs = ubufs;
1567
1568 n->tx_packets = 0;
1569 n->tx_zcopy_err = 0;
1570 n->tx_flush = false;
1571 }
1572
1573 mutex_unlock(&vq->mutex);
1574
1575 if (oldubufs) {
1576 vhost_net_ubuf_put_wait_and_free(oldubufs);
1577 mutex_lock(&vq->mutex);
1578 vhost_zerocopy_signal_used(n, vq);
1579 mutex_unlock(&vq->mutex);
1580 }
1581
1582 if (oldsock) {
1583 vhost_net_flush_vq(n, index);
1584 sockfd_put(oldsock);
1585 }
1586
1587 mutex_unlock(&n->dev.mutex);
1588 return 0;
1589
1590 err_used:
1591 vhost_vq_set_backend(vq, oldsock);
1592 vhost_net_enable_vq(n, vq);
1593 if (ubufs)
1594 vhost_net_ubuf_put_wait_and_free(ubufs);
1595 err_ubufs:
1596 if (sock)
1597 sockfd_put(sock);
1598 err_vq:
1599 mutex_unlock(&vq->mutex);
1600 err:
1601 mutex_unlock(&n->dev.mutex);
1602 return r;
1603 }
1604
vhost_net_reset_owner(struct vhost_net * n)1605 static long vhost_net_reset_owner(struct vhost_net *n)
1606 {
1607 struct socket *tx_sock = NULL;
1608 struct socket *rx_sock = NULL;
1609 long err;
1610 struct vhost_iotlb *umem;
1611
1612 mutex_lock(&n->dev.mutex);
1613 err = vhost_dev_check_owner(&n->dev);
1614 if (err)
1615 goto done;
1616 umem = vhost_dev_reset_owner_prepare();
1617 if (!umem) {
1618 err = -ENOMEM;
1619 goto done;
1620 }
1621 vhost_net_stop(n, &tx_sock, &rx_sock);
1622 vhost_net_flush(n);
1623 vhost_dev_stop(&n->dev);
1624 vhost_dev_reset_owner(&n->dev, umem);
1625 vhost_net_vq_reset(n);
1626 done:
1627 mutex_unlock(&n->dev.mutex);
1628 if (tx_sock)
1629 sockfd_put(tx_sock);
1630 if (rx_sock)
1631 sockfd_put(rx_sock);
1632 return err;
1633 }
1634
vhost_net_set_features(struct vhost_net * n,u64 features)1635 static int vhost_net_set_features(struct vhost_net *n, u64 features)
1636 {
1637 size_t vhost_hlen, sock_hlen, hdr_len;
1638 int i;
1639
1640 hdr_len = (features & ((1ULL << VIRTIO_NET_F_MRG_RXBUF) |
1641 (1ULL << VIRTIO_F_VERSION_1))) ?
1642 sizeof(struct virtio_net_hdr_mrg_rxbuf) :
1643 sizeof(struct virtio_net_hdr);
1644 if (features & (1 << VHOST_NET_F_VIRTIO_NET_HDR)) {
1645 /* vhost provides vnet_hdr */
1646 vhost_hlen = hdr_len;
1647 sock_hlen = 0;
1648 } else {
1649 /* socket provides vnet_hdr */
1650 vhost_hlen = 0;
1651 sock_hlen = hdr_len;
1652 }
1653 mutex_lock(&n->dev.mutex);
1654 if ((features & (1 << VHOST_F_LOG_ALL)) &&
1655 !vhost_log_access_ok(&n->dev))
1656 goto out_unlock;
1657
1658 if ((features & (1ULL << VIRTIO_F_ACCESS_PLATFORM))) {
1659 if (vhost_init_device_iotlb(&n->dev, true))
1660 goto out_unlock;
1661 }
1662
1663 for (i = 0; i < VHOST_NET_VQ_MAX; ++i) {
1664 mutex_lock(&n->vqs[i].vq.mutex);
1665 n->vqs[i].vq.acked_features = features;
1666 n->vqs[i].vhost_hlen = vhost_hlen;
1667 n->vqs[i].sock_hlen = sock_hlen;
1668 mutex_unlock(&n->vqs[i].vq.mutex);
1669 }
1670 mutex_unlock(&n->dev.mutex);
1671 return 0;
1672
1673 out_unlock:
1674 mutex_unlock(&n->dev.mutex);
1675 return -EFAULT;
1676 }
1677
vhost_net_set_owner(struct vhost_net * n)1678 static long vhost_net_set_owner(struct vhost_net *n)
1679 {
1680 int r;
1681
1682 mutex_lock(&n->dev.mutex);
1683 if (vhost_dev_has_owner(&n->dev)) {
1684 r = -EBUSY;
1685 goto out;
1686 }
1687 r = vhost_net_set_ubuf_info(n);
1688 if (r)
1689 goto out;
1690 r = vhost_dev_set_owner(&n->dev);
1691 if (r)
1692 vhost_net_clear_ubuf_info(n);
1693 vhost_net_flush(n);
1694 out:
1695 mutex_unlock(&n->dev.mutex);
1696 return r;
1697 }
1698
vhost_net_ioctl(struct file * f,unsigned int ioctl,unsigned long arg)1699 static long vhost_net_ioctl(struct file *f, unsigned int ioctl,
1700 unsigned long arg)
1701 {
1702 struct vhost_net *n = f->private_data;
1703 void __user *argp = (void __user *)arg;
1704 u64 __user *featurep = argp;
1705 struct vhost_vring_file backend;
1706 u64 features;
1707 int r;
1708
1709 switch (ioctl) {
1710 case VHOST_NET_SET_BACKEND:
1711 if (copy_from_user(&backend, argp, sizeof backend))
1712 return -EFAULT;
1713 return vhost_net_set_backend(n, backend.index, backend.fd);
1714 case VHOST_GET_FEATURES:
1715 features = VHOST_NET_FEATURES;
1716 if (copy_to_user(featurep, &features, sizeof features))
1717 return -EFAULT;
1718 return 0;
1719 case VHOST_SET_FEATURES:
1720 if (copy_from_user(&features, featurep, sizeof features))
1721 return -EFAULT;
1722 if (features & ~VHOST_NET_FEATURES)
1723 return -EOPNOTSUPP;
1724 return vhost_net_set_features(n, features);
1725 case VHOST_GET_BACKEND_FEATURES:
1726 features = VHOST_NET_BACKEND_FEATURES;
1727 if (copy_to_user(featurep, &features, sizeof(features)))
1728 return -EFAULT;
1729 return 0;
1730 case VHOST_SET_BACKEND_FEATURES:
1731 if (copy_from_user(&features, featurep, sizeof(features)))
1732 return -EFAULT;
1733 if (features & ~VHOST_NET_BACKEND_FEATURES)
1734 return -EOPNOTSUPP;
1735 vhost_set_backend_features(&n->dev, features);
1736 return 0;
1737 case VHOST_RESET_OWNER:
1738 return vhost_net_reset_owner(n);
1739 case VHOST_SET_OWNER:
1740 return vhost_net_set_owner(n);
1741 default:
1742 mutex_lock(&n->dev.mutex);
1743 r = vhost_dev_ioctl(&n->dev, ioctl, argp);
1744 if (r == -ENOIOCTLCMD)
1745 r = vhost_vring_ioctl(&n->dev, ioctl, argp);
1746 else
1747 vhost_net_flush(n);
1748 mutex_unlock(&n->dev.mutex);
1749 return r;
1750 }
1751 }
1752
vhost_net_chr_read_iter(struct kiocb * iocb,struct iov_iter * to)1753 static ssize_t vhost_net_chr_read_iter(struct kiocb *iocb, struct iov_iter *to)
1754 {
1755 struct file *file = iocb->ki_filp;
1756 struct vhost_net *n = file->private_data;
1757 struct vhost_dev *dev = &n->dev;
1758 int noblock = file->f_flags & O_NONBLOCK;
1759
1760 return vhost_chr_read_iter(dev, to, noblock);
1761 }
1762
vhost_net_chr_write_iter(struct kiocb * iocb,struct iov_iter * from)1763 static ssize_t vhost_net_chr_write_iter(struct kiocb *iocb,
1764 struct iov_iter *from)
1765 {
1766 struct file *file = iocb->ki_filp;
1767 struct vhost_net *n = file->private_data;
1768 struct vhost_dev *dev = &n->dev;
1769
1770 return vhost_chr_write_iter(dev, from);
1771 }
1772
vhost_net_chr_poll(struct file * file,poll_table * wait)1773 static __poll_t vhost_net_chr_poll(struct file *file, poll_table *wait)
1774 {
1775 struct vhost_net *n = file->private_data;
1776 struct vhost_dev *dev = &n->dev;
1777
1778 return vhost_chr_poll(file, dev, wait);
1779 }
1780
1781 static const struct file_operations vhost_net_fops = {
1782 .owner = THIS_MODULE,
1783 .release = vhost_net_release,
1784 .read_iter = vhost_net_chr_read_iter,
1785 .write_iter = vhost_net_chr_write_iter,
1786 .poll = vhost_net_chr_poll,
1787 .unlocked_ioctl = vhost_net_ioctl,
1788 .compat_ioctl = compat_ptr_ioctl,
1789 .open = vhost_net_open,
1790 .llseek = noop_llseek,
1791 };
1792
1793 static struct miscdevice vhost_net_misc = {
1794 .minor = VHOST_NET_MINOR,
1795 .name = "vhost-net",
1796 .fops = &vhost_net_fops,
1797 };
1798
vhost_net_init(void)1799 static int vhost_net_init(void)
1800 {
1801 if (experimental_zcopytx)
1802 vhost_net_enable_zcopy(VHOST_NET_VQ_TX);
1803 return misc_register(&vhost_net_misc);
1804 }
1805 module_init(vhost_net_init);
1806
vhost_net_exit(void)1807 static void vhost_net_exit(void)
1808 {
1809 misc_deregister(&vhost_net_misc);
1810 }
1811 module_exit(vhost_net_exit);
1812
1813 MODULE_VERSION("0.0.1");
1814 MODULE_LICENSE("GPL v2");
1815 MODULE_AUTHOR("Michael S. Tsirkin");
1816 MODULE_DESCRIPTION("Host kernel accelerator for virtio net");
1817 MODULE_ALIAS_MISCDEV(VHOST_NET_MINOR);
1818 MODULE_ALIAS("devname:vhost-net");
1819