Lines Matching +full:num +full:- +full:rxq
1 // SPDX-License-Identifier: GPL-2.0-only
5 * virtio-net server in host kernel.
41 " 1 -Enable; 0 - Disable");
126 struct vhost_net_buf rxq; member
151 static void *vhost_net_buf_get_ptr(struct vhost_net_buf *rxq) in vhost_net_buf_get_ptr() argument
153 if (rxq->tail != rxq->head) in vhost_net_buf_get_ptr()
154 return rxq->queue[rxq->head]; in vhost_net_buf_get_ptr()
159 static int vhost_net_buf_get_size(struct vhost_net_buf *rxq) in vhost_net_buf_get_size() argument
161 return rxq->tail - rxq->head; in vhost_net_buf_get_size()
164 static int vhost_net_buf_is_empty(struct vhost_net_buf *rxq) in vhost_net_buf_is_empty() argument
166 return rxq->tail == rxq->head; in vhost_net_buf_is_empty()
169 static void *vhost_net_buf_consume(struct vhost_net_buf *rxq) in vhost_net_buf_consume() argument
171 void *ret = vhost_net_buf_get_ptr(rxq); in vhost_net_buf_consume()
172 ++rxq->head; in vhost_net_buf_consume()
178 struct vhost_net_buf *rxq = &nvq->rxq; in vhost_net_buf_produce() local
180 rxq->head = 0; in vhost_net_buf_produce()
181 rxq->tail = ptr_ring_consume_batched(nvq->rx_ring, rxq->queue, in vhost_net_buf_produce()
183 return rxq->tail; in vhost_net_buf_produce()
188 struct vhost_net_buf *rxq = &nvq->rxq; in vhost_net_buf_unproduce() local
190 if (nvq->rx_ring && !vhost_net_buf_is_empty(rxq)) { in vhost_net_buf_unproduce()
191 ptr_ring_unconsume(nvq->rx_ring, rxq->queue + rxq->head, in vhost_net_buf_unproduce()
192 vhost_net_buf_get_size(rxq), in vhost_net_buf_unproduce()
194 rxq->head = rxq->tail = 0; in vhost_net_buf_unproduce()
203 return xdpf->len; in vhost_net_buf_peek_len()
211 struct vhost_net_buf *rxq = &nvq->rxq; in vhost_net_buf_peek() local
213 if (!vhost_net_buf_is_empty(rxq)) in vhost_net_buf_peek()
220 return vhost_net_buf_peek_len(vhost_net_buf_get_ptr(rxq)); in vhost_net_buf_peek()
223 static void vhost_net_buf_init(struct vhost_net_buf *rxq) in vhost_net_buf_init() argument
225 rxq->head = rxq->tail = 0; in vhost_net_buf_init()
242 return ERR_PTR(-ENOMEM); in vhost_net_ubuf_alloc()
243 atomic_set(&ubufs->refcount, 1); in vhost_net_ubuf_alloc()
244 init_waitqueue_head(&ubufs->wait); in vhost_net_ubuf_alloc()
245 ubufs->vq = vq; in vhost_net_ubuf_alloc()
251 int r = atomic_sub_return(1, &ubufs->refcount); in vhost_net_ubuf_put()
253 wake_up(&ubufs->wait); in vhost_net_ubuf_put()
260 wait_event(ubufs->wait, !atomic_read(&ubufs->refcount)); in vhost_net_ubuf_put_and_wait()
274 kfree(n->vqs[i].ubuf_info); in vhost_net_clear_ubuf_info()
275 n->vqs[i].ubuf_info = NULL; in vhost_net_clear_ubuf_info()
288 n->vqs[i].ubuf_info = in vhost_net_set_ubuf_info()
290 sizeof(*n->vqs[i].ubuf_info), in vhost_net_set_ubuf_info()
292 if (!n->vqs[i].ubuf_info) in vhost_net_set_ubuf_info()
299 return -ENOMEM; in vhost_net_set_ubuf_info()
309 n->vqs[i].done_idx = 0; in vhost_net_vq_reset()
310 n->vqs[i].upend_idx = 0; in vhost_net_vq_reset()
311 n->vqs[i].ubufs = NULL; in vhost_net_vq_reset()
312 n->vqs[i].vhost_hlen = 0; in vhost_net_vq_reset()
313 n->vqs[i].sock_hlen = 0; in vhost_net_vq_reset()
314 vhost_net_buf_init(&n->vqs[i].rxq); in vhost_net_vq_reset()
321 ++net->tx_packets; in vhost_net_tx_packet()
322 if (net->tx_packets < 1024) in vhost_net_tx_packet()
324 net->tx_packets = 0; in vhost_net_tx_packet()
325 net->tx_zcopy_err = 0; in vhost_net_tx_packet()
330 ++net->tx_zcopy_err; in vhost_net_tx_err()
338 return !net->tx_flush && in vhost_net_tx_select_zcopy()
339 net->tx_packets / 64 >= net->tx_zcopy_err; in vhost_net_tx_select_zcopy()
345 sock_flag(sock->sk, SOCK_ZEROCOPY); in vhost_sock_zcopy()
350 return sock_flag(sock->sk, SOCK_XDP); in vhost_sock_xdp()
366 for (i = nvq->done_idx; i != nvq->upend_idx; i = (i + 1) % UIO_MAXIOV) { in vhost_zerocopy_signal_used()
367 if (vq->heads[i].len == VHOST_DMA_FAILED_LEN) in vhost_zerocopy_signal_used()
369 if (VHOST_DMA_IS_DONE(vq->heads[i].len)) { in vhost_zerocopy_signal_used()
370 vq->heads[i].len = VHOST_DMA_CLEAR_LEN; in vhost_zerocopy_signal_used()
376 add = min(UIO_MAXIOV - nvq->done_idx, j); in vhost_zerocopy_signal_used()
377 vhost_add_used_and_signal_n(vq->dev, vq, in vhost_zerocopy_signal_used()
378 &vq->heads[nvq->done_idx], add); in vhost_zerocopy_signal_used()
379 nvq->done_idx = (nvq->done_idx + add) % UIO_MAXIOV; in vhost_zerocopy_signal_used()
380 j -= add; in vhost_zerocopy_signal_used()
386 struct vhost_net_ubuf_ref *ubufs = ubuf->ctx; in vhost_zerocopy_callback()
387 struct vhost_virtqueue *vq = ubufs->vq; in vhost_zerocopy_callback()
393 vq->heads[ubuf->desc].len = success ? in vhost_zerocopy_callback()
405 vhost_poll_queue(&vq->poll); in vhost_zerocopy_callback()
426 struct vhost_poll *poll = n->poll + (nvq - n->vqs); in vhost_net_disable_vq()
437 struct vhost_poll *poll = n->poll + (nvq - n->vqs); in vhost_net_enable_vq()
444 return vhost_poll_start(poll, sock->file); in vhost_net_enable_vq()
449 struct vhost_virtqueue *vq = &nvq->vq; in vhost_net_signal_used()
450 struct vhost_dev *dev = vq->dev; in vhost_net_signal_used()
452 if (!nvq->done_idx) in vhost_net_signal_used()
455 vhost_add_used_and_signal_n(dev, vq, vq->heads, nvq->done_idx); in vhost_net_signal_used()
456 nvq->done_idx = 0; in vhost_net_signal_used()
466 .num = nvq->batched_xdp, in vhost_tx_batch()
467 .ptr = nvq->xdp, in vhost_tx_batch()
471 if (nvq->batched_xdp == 0) in vhost_tx_batch()
474 msghdr->msg_control = &ctl; in vhost_tx_batch()
475 msghdr->msg_controllen = sizeof(ctl); in vhost_tx_batch()
476 err = sock->ops->sendmsg(sock, msghdr, 0); in vhost_tx_batch()
478 vq_err(&nvq->vq, "Fail to batch sending packets\n"); in vhost_tx_batch()
484 for (i = 0; i < nvq->batched_xdp; ++i) in vhost_tx_batch()
485 put_page(virt_to_head_page(nvq->xdp[i].data)); in vhost_tx_batch()
486 nvq->batched_xdp = 0; in vhost_tx_batch()
487 nvq->done_idx = 0; in vhost_tx_batch()
493 nvq->batched_xdp = 0; in vhost_tx_batch()
501 if (sock->ops->peek_len) in sock_has_rx_data()
502 return sock->ops->peek_len(sock); in sock_has_rx_data()
504 return skb_queue_empty(&sock->sk->sk_receive_queue); in sock_has_rx_data()
510 if (!vhost_vq_avail_empty(&net->dev, vq)) { in vhost_net_busy_poll_try_queue()
511 vhost_poll_queue(&vq->poll); in vhost_net_busy_poll_try_queue()
512 } else if (unlikely(vhost_enable_notify(&net->dev, vq))) { in vhost_net_busy_poll_try_queue()
513 vhost_disable_notify(&net->dev, vq); in vhost_net_busy_poll_try_queue()
514 vhost_poll_queue(&vq->poll); in vhost_net_busy_poll_try_queue()
533 if (!mutex_trylock(&vq->mutex)) in vhost_net_busy_poll()
536 vhost_disable_notify(&net->dev, vq); in vhost_net_busy_poll()
539 busyloop_timeout = poll_rx ? rvq->busyloop_timeout: in vhost_net_busy_poll()
540 tvq->busyloop_timeout; in vhost_net_busy_poll()
546 if (vhost_has_work(&net->dev)) { in vhost_net_busy_poll()
552 !vhost_vq_avail_empty(&net->dev, rvq)) || in vhost_net_busy_poll()
553 !vhost_vq_avail_empty(&net->dev, tvq)) in vhost_net_busy_poll()
564 vhost_enable_notify(&net->dev, rvq); in vhost_net_busy_poll()
566 mutex_unlock(&vq->mutex); in vhost_net_busy_poll()
574 struct vhost_net_virtqueue *rnvq = &net->vqs[VHOST_NET_VQ_RX]; in vhost_net_tx_get_vq_desc()
575 struct vhost_virtqueue *rvq = &rnvq->vq; in vhost_net_tx_get_vq_desc()
576 struct vhost_virtqueue *tvq = &tnvq->vq; in vhost_net_tx_get_vq_desc()
578 int r = vhost_get_vq_desc(tvq, tvq->iov, ARRAY_SIZE(tvq->iov), in vhost_net_tx_get_vq_desc()
581 if (r == tvq->num && tvq->busyloop_timeout) { in vhost_net_tx_get_vq_desc()
590 r = vhost_get_vq_desc(tvq, tvq->iov, ARRAY_SIZE(tvq->iov), in vhost_net_tx_get_vq_desc()
599 struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_TX]; in vhost_exceeds_maxpend()
600 struct vhost_virtqueue *vq = &nvq->vq; in vhost_exceeds_maxpend()
602 return (nvq->upend_idx + UIO_MAXIOV - nvq->done_idx) % UIO_MAXIOV > in vhost_exceeds_maxpend()
603 min_t(unsigned int, VHOST_MAX_PEND, vq->num >> 2); in vhost_exceeds_maxpend()
610 size_t len = iov_length(vq->iov, out); in init_iov_iter()
612 iov_iter_init(iter, WRITE, vq->iov, out, len); in init_iov_iter()
624 struct vhost_virtqueue *vq = &nvq->vq; in get_tx_bufs()
629 if (ret < 0 || ret == vq->num) in get_tx_bufs()
635 return -EFAULT; in get_tx_bufs()
639 *len = init_iov_iter(vq, &msg->msg_iter, nvq->vhost_hlen, *out); in get_tx_bufs()
642 *len, nvq->vhost_hlen); in get_tx_bufs()
643 return -EFAULT; in get_tx_bufs()
652 !vhost_vq_avail_empty(vq->dev, vq); in tx_can_batch()
658 if (pfrag->page) { in vhost_net_page_frag_refill()
659 if (pfrag->offset + sz <= pfrag->size) in vhost_net_page_frag_refill()
661 __page_frag_cache_drain(pfrag->page, net->refcnt_bias); in vhost_net_page_frag_refill()
664 pfrag->offset = 0; in vhost_net_page_frag_refill()
665 net->refcnt_bias = 0; in vhost_net_page_frag_refill()
668 pfrag->page = alloc_pages((gfp & ~__GFP_DIRECT_RECLAIM) | in vhost_net_page_frag_refill()
672 if (likely(pfrag->page)) { in vhost_net_page_frag_refill()
673 pfrag->size = PAGE_SIZE << SKB_FRAG_PAGE_ORDER; in vhost_net_page_frag_refill()
677 pfrag->page = alloc_page(gfp); in vhost_net_page_frag_refill()
678 if (likely(pfrag->page)) { in vhost_net_page_frag_refill()
679 pfrag->size = PAGE_SIZE; in vhost_net_page_frag_refill()
685 net->refcnt_bias = USHRT_MAX; in vhost_net_page_frag_refill()
686 page_ref_add(pfrag->page, USHRT_MAX - 1); in vhost_net_page_frag_refill()
695 struct vhost_virtqueue *vq = &nvq->vq; in vhost_net_build_xdp()
696 struct vhost_net *net = container_of(vq->dev, struct vhost_net, in vhost_net_build_xdp()
699 struct page_frag *alloc_frag = &net->page_frag; in vhost_net_build_xdp()
701 struct xdp_buff *xdp = &nvq->xdp[nvq->batched_xdp]; in vhost_net_build_xdp()
706 int pad = SKB_DATA_ALIGN(VHOST_NET_RX_PAD + headroom + nvq->sock_hlen); in vhost_net_build_xdp()
707 int sock_hlen = nvq->sock_hlen; in vhost_net_build_xdp()
711 if (unlikely(len < nvq->sock_hlen)) in vhost_net_build_xdp()
712 return -EFAULT; in vhost_net_build_xdp()
716 return -ENOSPC; in vhost_net_build_xdp()
719 alloc_frag->offset = ALIGN((u64)alloc_frag->offset, SMP_CACHE_BYTES); in vhost_net_build_xdp()
722 return -ENOMEM; in vhost_net_build_xdp()
724 buf = (char *)page_address(alloc_frag->page) + alloc_frag->offset; in vhost_net_build_xdp()
725 copied = copy_page_from_iter(alloc_frag->page, in vhost_net_build_xdp()
726 alloc_frag->offset + in vhost_net_build_xdp()
730 return -EFAULT; in vhost_net_build_xdp()
733 gso = &hdr->gso; in vhost_net_build_xdp()
735 if ((gso->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) && in vhost_net_build_xdp()
736 vhost16_to_cpu(vq, gso->csum_start) + in vhost_net_build_xdp()
737 vhost16_to_cpu(vq, gso->csum_offset) + 2 > in vhost_net_build_xdp()
738 vhost16_to_cpu(vq, gso->hdr_len)) { in vhost_net_build_xdp()
739 gso->hdr_len = cpu_to_vhost16(vq, in vhost_net_build_xdp()
740 vhost16_to_cpu(vq, gso->csum_start) + in vhost_net_build_xdp()
741 vhost16_to_cpu(vq, gso->csum_offset) + 2); in vhost_net_build_xdp()
743 if (vhost16_to_cpu(vq, gso->hdr_len) > len) in vhost_net_build_xdp()
744 return -EINVAL; in vhost_net_build_xdp()
747 len -= sock_hlen; in vhost_net_build_xdp()
748 copied = copy_page_from_iter(alloc_frag->page, in vhost_net_build_xdp()
749 alloc_frag->offset + pad, in vhost_net_build_xdp()
752 return -EFAULT; in vhost_net_build_xdp()
754 xdp->data_hard_start = buf; in vhost_net_build_xdp()
755 xdp->data = buf + pad; in vhost_net_build_xdp()
756 xdp->data_end = xdp->data + len; in vhost_net_build_xdp()
757 hdr->buflen = buflen; in vhost_net_build_xdp()
758 xdp->frame_sz = buflen; in vhost_net_build_xdp()
760 --net->refcnt_bias; in vhost_net_build_xdp()
761 alloc_frag->offset += buflen; in vhost_net_build_xdp()
763 ++nvq->batched_xdp; in vhost_net_build_xdp()
770 struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_TX]; in handle_tx_copy()
771 struct vhost_virtqueue *vq = &nvq->vq; in handle_tx_copy()
784 bool sock_can_batch = (sock->sk->sk_sndbuf == INT_MAX); in handle_tx_copy()
789 if (nvq->done_idx == VHOST_NET_BATCH) in handle_tx_copy()
798 if (head == vq->num) { in handle_tx_copy()
800 vhost_poll_queue(&vq->poll); in handle_tx_copy()
801 } else if (unlikely(vhost_enable_notify(&net->dev, in handle_tx_copy()
803 vhost_disable_notify(&net->dev, vq); in handle_tx_copy()
818 } else if (unlikely(err != -ENOSPC)) { in handle_tx_copy()
839 err = sock->ops->sendmsg(sock, &msg, len); in handle_tx_copy()
849 vq->heads[nvq->done_idx].id = cpu_to_vhost32(vq, head); in handle_tx_copy()
850 vq->heads[nvq->done_idx].len = 0; in handle_tx_copy()
851 ++nvq->done_idx; in handle_tx_copy()
859 struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_TX]; in handle_tx_zerocopy()
860 struct vhost_virtqueue *vq = &nvq->vq; in handle_tx_zerocopy()
891 if (head == vq->num) { in handle_tx_zerocopy()
893 vhost_poll_queue(&vq->poll); in handle_tx_zerocopy()
894 } else if (unlikely(vhost_enable_notify(&net->dev, vq))) { in handle_tx_zerocopy()
895 vhost_disable_notify(&net->dev, vq); in handle_tx_zerocopy()
907 ubuf = nvq->ubuf_info + nvq->upend_idx; in handle_tx_zerocopy()
908 vq->heads[nvq->upend_idx].id = cpu_to_vhost32(vq, head); in handle_tx_zerocopy()
909 vq->heads[nvq->upend_idx].len = VHOST_DMA_IN_PROGRESS; in handle_tx_zerocopy()
910 ubuf->callback = vhost_zerocopy_callback; in handle_tx_zerocopy()
911 ubuf->ctx = nvq->ubufs; in handle_tx_zerocopy()
912 ubuf->desc = nvq->upend_idx; in handle_tx_zerocopy()
913 refcount_set(&ubuf->refcnt, 1); in handle_tx_zerocopy()
918 ubufs = nvq->ubufs; in handle_tx_zerocopy()
919 atomic_inc(&ubufs->refcount); in handle_tx_zerocopy()
920 nvq->upend_idx = (nvq->upend_idx + 1) % UIO_MAXIOV; in handle_tx_zerocopy()
934 err = sock->ops->sendmsg(sock, &msg, len); in handle_tx_zerocopy()
937 if (vq->heads[ubuf->desc].len == VHOST_DMA_IN_PROGRESS) in handle_tx_zerocopy()
939 nvq->upend_idx = ((unsigned)nvq->upend_idx - 1) in handle_tx_zerocopy()
950 vhost_add_used_and_signal(&net->dev, vq, head, 0); in handle_tx_zerocopy()
957 /* Expects to be always run from workqueue - which acts as
958 * read-size critical section for our kind of RCU. */
961 struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_TX]; in handle_tx()
962 struct vhost_virtqueue *vq = &nvq->vq; in handle_tx()
965 mutex_lock_nested(&vq->mutex, VHOST_NET_VQ_TX); in handle_tx()
973 vhost_disable_notify(&net->dev, vq); in handle_tx()
982 mutex_unlock(&vq->mutex); in handle_tx()
991 if (rvq->rx_ring) in peek_head_len()
994 spin_lock_irqsave(&sk->sk_receive_queue.lock, flags); in peek_head_len()
995 head = skb_peek(&sk->sk_receive_queue); in peek_head_len()
997 len = head->len; in peek_head_len()
1002 spin_unlock_irqrestore(&sk->sk_receive_queue.lock, flags); in peek_head_len()
1009 struct vhost_net_virtqueue *rnvq = &net->vqs[VHOST_NET_VQ_RX]; in vhost_net_rx_peek_head_len()
1010 struct vhost_net_virtqueue *tnvq = &net->vqs[VHOST_NET_VQ_TX]; in vhost_net_rx_peek_head_len()
1011 struct vhost_virtqueue *rvq = &rnvq->vq; in vhost_net_rx_peek_head_len()
1012 struct vhost_virtqueue *tvq = &tnvq->vq; in vhost_net_rx_peek_head_len()
1015 if (!len && rvq->busyloop_timeout) { in vhost_net_rx_peek_head_len()
1027 /* This is a multi-buffer version of vhost_get_desc, that works if
1029 * @vq - the relevant virtqueue
1030 * @datalen - data length we'll be reading
1031 * @iovcount - returned count of io vectors we fill
1032 * @log - vhost log
1033 * @log_num - log offset
1034 * @quota - headcount quota, 1 for big buffer
1057 r = -ENOBUFS; in get_rx_bufs()
1060 r = vhost_get_vq_desc(vq, vq->iov + seg, in get_rx_bufs()
1061 ARRAY_SIZE(vq->iov) - seg, &out, in get_rx_bufs()
1067 if (d == vq->num) { in get_rx_bufs()
1074 r = -EINVAL; in get_rx_bufs()
1082 len = iov_length(vq->iov + seg, in); in get_rx_bufs()
1084 datalen -= len; in get_rx_bufs()
1088 heads[headcount - 1].len = cpu_to_vhost32(vq, len + datalen); in get_rx_bufs()
1104 /* Expects to be always run from workqueue - which acts as
1105 * read-size critical section for our kind of RCU. */
1108 struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_RX]; in handle_rx()
1109 struct vhost_virtqueue *vq = &nvq->vq; in handle_rx()
1134 mutex_lock_nested(&vq->mutex, VHOST_NET_VQ_RX); in handle_rx()
1142 vhost_disable_notify(&net->dev, vq); in handle_rx()
1145 vhost_hlen = nvq->vhost_hlen; in handle_rx()
1146 sock_hlen = nvq->sock_hlen; in handle_rx()
1149 vq->log : NULL; in handle_rx()
1153 sock_len = vhost_net_rx_peek_head_len(net, sock->sk, in handle_rx()
1159 headcount = get_rx_bufs(vq, vq->heads + nvq->done_idx, in handle_rx()
1168 vhost_poll_queue(&vq->poll); in handle_rx()
1169 } else if (unlikely(vhost_enable_notify(&net->dev, vq))) { in handle_rx()
1172 vhost_disable_notify(&net->dev, vq); in handle_rx()
1180 if (nvq->rx_ring) in handle_rx()
1181 msg.msg_control = vhost_net_buf_consume(&nvq->rxq); in handle_rx()
1184 iov_iter_init(&msg.msg_iter, READ, vq->iov, 1, 1); in handle_rx()
1185 err = sock->ops->recvmsg(sock, &msg, in handle_rx()
1191 iov_iter_init(&msg.msg_iter, READ, vq->iov, in, vhost_len); in handle_rx()
1199 err = sock->ops->recvmsg(sock, &msg, in handle_rx()
1215 "at addr %p\n", vq->iov->iov_base); in handle_rx()
1220 * ->num_buffers over if VIRTIO_NET_F_MRG_RXBUF in handle_rx()
1234 nvq->done_idx += headcount; in handle_rx()
1235 if (nvq->done_idx > VHOST_NET_BATCH) in handle_rx()
1239 vq->iov, in); in handle_rx()
1244 vhost_poll_queue(&vq->poll); in handle_rx()
1249 mutex_unlock(&vq->mutex); in handle_rx()
1256 struct vhost_net *net = container_of(vq->dev, struct vhost_net, dev); in handle_tx_kick()
1265 struct vhost_net *net = container_of(vq->dev, struct vhost_net, dev); in handle_rx_kick()
1295 return -ENOMEM; in vhost_net_open()
1299 return -ENOMEM; in vhost_net_open()
1307 return -ENOMEM; in vhost_net_open()
1309 n->vqs[VHOST_NET_VQ_RX].rxq.queue = queue; in vhost_net_open()
1316 return -ENOMEM; in vhost_net_open()
1318 n->vqs[VHOST_NET_VQ_TX].xdp = xdp; in vhost_net_open()
1320 dev = &n->dev; in vhost_net_open()
1321 vqs[VHOST_NET_VQ_TX] = &n->vqs[VHOST_NET_VQ_TX].vq; in vhost_net_open()
1322 vqs[VHOST_NET_VQ_RX] = &n->vqs[VHOST_NET_VQ_RX].vq; in vhost_net_open()
1323 n->vqs[VHOST_NET_VQ_TX].vq.handle_kick = handle_tx_kick; in vhost_net_open()
1324 n->vqs[VHOST_NET_VQ_RX].vq.handle_kick = handle_rx_kick; in vhost_net_open()
1326 n->vqs[i].ubufs = NULL; in vhost_net_open()
1327 n->vqs[i].ubuf_info = NULL; in vhost_net_open()
1328 n->vqs[i].upend_idx = 0; in vhost_net_open()
1329 n->vqs[i].done_idx = 0; in vhost_net_open()
1330 n->vqs[i].batched_xdp = 0; in vhost_net_open()
1331 n->vqs[i].vhost_hlen = 0; in vhost_net_open()
1332 n->vqs[i].sock_hlen = 0; in vhost_net_open()
1333 n->vqs[i].rx_ring = NULL; in vhost_net_open()
1334 vhost_net_buf_init(&n->vqs[i].rxq); in vhost_net_open()
1341 vhost_poll_init(n->poll + VHOST_NET_VQ_TX, handle_tx_net, EPOLLOUT, dev); in vhost_net_open()
1342 vhost_poll_init(n->poll + VHOST_NET_VQ_RX, handle_rx_net, EPOLLIN, dev); in vhost_net_open()
1344 f->private_data = n; in vhost_net_open()
1345 n->page_frag.page = NULL; in vhost_net_open()
1346 n->refcnt_bias = 0; in vhost_net_open()
1358 mutex_lock(&vq->mutex); in vhost_net_stop_vq()
1363 nvq->rx_ring = NULL; in vhost_net_stop_vq()
1364 mutex_unlock(&vq->mutex); in vhost_net_stop_vq()
1371 *tx_sock = vhost_net_stop_vq(n, &n->vqs[VHOST_NET_VQ_TX].vq); in vhost_net_stop()
1372 *rx_sock = vhost_net_stop_vq(n, &n->vqs[VHOST_NET_VQ_RX].vq); in vhost_net_stop()
1377 vhost_poll_flush(n->poll + index); in vhost_net_flush_vq()
1378 vhost_poll_flush(&n->vqs[index].vq.poll); in vhost_net_flush_vq()
1385 if (n->vqs[VHOST_NET_VQ_TX].ubufs) { in vhost_net_flush()
1386 mutex_lock(&n->vqs[VHOST_NET_VQ_TX].vq.mutex); in vhost_net_flush()
1387 n->tx_flush = true; in vhost_net_flush()
1388 mutex_unlock(&n->vqs[VHOST_NET_VQ_TX].vq.mutex); in vhost_net_flush()
1390 vhost_net_ubuf_put_and_wait(n->vqs[VHOST_NET_VQ_TX].ubufs); in vhost_net_flush()
1391 mutex_lock(&n->vqs[VHOST_NET_VQ_TX].vq.mutex); in vhost_net_flush()
1392 n->tx_flush = false; in vhost_net_flush()
1393 atomic_set(&n->vqs[VHOST_NET_VQ_TX].ubufs->refcount, 1); in vhost_net_flush()
1394 mutex_unlock(&n->vqs[VHOST_NET_VQ_TX].vq.mutex); in vhost_net_flush()
1400 struct vhost_net *n = f->private_data; in vhost_net_release()
1406 vhost_dev_stop(&n->dev); in vhost_net_release()
1407 vhost_dev_cleanup(&n->dev); in vhost_net_release()
1416 * since jobs can re-queue themselves. */ in vhost_net_release()
1418 kfree(n->vqs[VHOST_NET_VQ_RX].rxq.queue); in vhost_net_release()
1419 kfree(n->vqs[VHOST_NET_VQ_TX].xdp); in vhost_net_release()
1420 kfree(n->dev.vqs); in vhost_net_release()
1421 if (n->page_frag.page) in vhost_net_release()
1422 __page_frag_cache_drain(n->page_frag.page, n->refcnt_bias); in vhost_net_release()
1433 return ERR_PTR(-ENOTSOCK); in get_raw_socket()
1436 if (sock->sk->sk_type != SOCK_RAW) { in get_raw_socket()
1437 r = -ESOCKTNOSUPPORT; in get_raw_socket()
1441 if (sock->sk->sk_family != AF_PACKET) { in get_raw_socket()
1442 r = -EPFNOSUPPORT; in get_raw_socket()
1471 return ERR_PTR(-EBADF); in get_tap_socket()
1486 if (fd == -1) in get_socket()
1494 return ERR_PTR(-ENOTSOCK); in get_socket()
1505 mutex_lock(&n->dev.mutex); in vhost_net_set_backend()
1506 r = vhost_dev_check_owner(&n->dev); in vhost_net_set_backend()
1511 r = -ENOBUFS; in vhost_net_set_backend()
1514 vq = &n->vqs[index].vq; in vhost_net_set_backend()
1515 nvq = &n->vqs[index]; in vhost_net_set_backend()
1516 mutex_lock(&vq->mutex); in vhost_net_set_backend()
1518 if (fd == -1) in vhost_net_set_backend()
1519 vhost_clear_msg(&n->dev); in vhost_net_set_backend()
1523 r = -EFAULT; in vhost_net_set_backend()
1553 nvq->rx_ring = get_tap_ptr_ring(sock->file); in vhost_net_set_backend()
1555 nvq->rx_ring = NULL; in vhost_net_set_backend()
1558 oldubufs = nvq->ubufs; in vhost_net_set_backend()
1559 nvq->ubufs = ubufs; in vhost_net_set_backend()
1561 n->tx_packets = 0; in vhost_net_set_backend()
1562 n->tx_zcopy_err = 0; in vhost_net_set_backend()
1563 n->tx_flush = false; in vhost_net_set_backend()
1566 mutex_unlock(&vq->mutex); in vhost_net_set_backend()
1570 mutex_lock(&vq->mutex); in vhost_net_set_backend()
1572 mutex_unlock(&vq->mutex); in vhost_net_set_backend()
1580 mutex_unlock(&n->dev.mutex); in vhost_net_set_backend()
1592 mutex_unlock(&vq->mutex); in vhost_net_set_backend()
1594 mutex_unlock(&n->dev.mutex); in vhost_net_set_backend()
1605 mutex_lock(&n->dev.mutex); in vhost_net_reset_owner()
1606 err = vhost_dev_check_owner(&n->dev); in vhost_net_reset_owner()
1611 err = -ENOMEM; in vhost_net_reset_owner()
1616 vhost_dev_stop(&n->dev); in vhost_net_reset_owner()
1617 vhost_dev_reset_owner(&n->dev, umem); in vhost_net_reset_owner()
1620 mutex_unlock(&n->dev.mutex); in vhost_net_reset_owner()
1646 mutex_lock(&n->dev.mutex); in vhost_net_set_features()
1648 !vhost_log_access_ok(&n->dev)) in vhost_net_set_features()
1652 if (vhost_init_device_iotlb(&n->dev, true)) in vhost_net_set_features()
1657 mutex_lock(&n->vqs[i].vq.mutex); in vhost_net_set_features()
1658 n->vqs[i].vq.acked_features = features; in vhost_net_set_features()
1659 n->vqs[i].vhost_hlen = vhost_hlen; in vhost_net_set_features()
1660 n->vqs[i].sock_hlen = sock_hlen; in vhost_net_set_features()
1661 mutex_unlock(&n->vqs[i].vq.mutex); in vhost_net_set_features()
1663 mutex_unlock(&n->dev.mutex); in vhost_net_set_features()
1667 mutex_unlock(&n->dev.mutex); in vhost_net_set_features()
1668 return -EFAULT; in vhost_net_set_features()
1675 mutex_lock(&n->dev.mutex); in vhost_net_set_owner()
1676 if (vhost_dev_has_owner(&n->dev)) { in vhost_net_set_owner()
1677 r = -EBUSY; in vhost_net_set_owner()
1683 r = vhost_dev_set_owner(&n->dev); in vhost_net_set_owner()
1688 mutex_unlock(&n->dev.mutex); in vhost_net_set_owner()
1695 struct vhost_net *n = f->private_data; in vhost_net_ioctl()
1705 return -EFAULT; in vhost_net_ioctl()
1710 return -EFAULT; in vhost_net_ioctl()
1714 return -EFAULT; in vhost_net_ioctl()
1716 return -EOPNOTSUPP; in vhost_net_ioctl()
1721 return -EFAULT; in vhost_net_ioctl()
1725 return -EFAULT; in vhost_net_ioctl()
1727 return -EOPNOTSUPP; in vhost_net_ioctl()
1728 vhost_set_backend_features(&n->dev, features); in vhost_net_ioctl()
1735 mutex_lock(&n->dev.mutex); in vhost_net_ioctl()
1736 r = vhost_dev_ioctl(&n->dev, ioctl, argp); in vhost_net_ioctl()
1737 if (r == -ENOIOCTLCMD) in vhost_net_ioctl()
1738 r = vhost_vring_ioctl(&n->dev, ioctl, argp); in vhost_net_ioctl()
1741 mutex_unlock(&n->dev.mutex); in vhost_net_ioctl()
1748 struct file *file = iocb->ki_filp; in vhost_net_chr_read_iter()
1749 struct vhost_net *n = file->private_data; in vhost_net_chr_read_iter()
1750 struct vhost_dev *dev = &n->dev; in vhost_net_chr_read_iter()
1751 int noblock = file->f_flags & O_NONBLOCK; in vhost_net_chr_read_iter()
1759 struct file *file = iocb->ki_filp; in vhost_net_chr_write_iter()
1760 struct vhost_net *n = file->private_data; in vhost_net_chr_write_iter()
1761 struct vhost_dev *dev = &n->dev; in vhost_net_chr_write_iter()
1768 struct vhost_net *n = file->private_data; in vhost_net_chr_poll()
1769 struct vhost_dev *dev = &n->dev; in vhost_net_chr_poll()
1788 .name = "vhost-net",
1811 MODULE_ALIAS("devname:vhost-net");