92 			    struct vhost_virtqueue *vq)  in vhost_transport_do_send_pkt()  argument
99 	mutex_lock(&vq->mutex);  in vhost_transport_do_send_pkt()
101 	if (!vhost_vq_get_backend(vq))  in vhost_transport_do_send_pkt()
104 	if (!vq_meta_prefetch(vq))  in vhost_transport_do_send_pkt()
108 	vhost_disable_notify(&vsock->dev, vq);  in vhost_transport_do_send_pkt()
122 			vhost_enable_notify(&vsock->dev, vq);  in vhost_transport_do_send_pkt()
131 		head = vhost_get_vq_desc(vq, vq->iov, ARRAY_SIZE(vq->iov),  in vhost_transport_do_send_pkt()
140 		if (head == vq->num) {  in vhost_transport_do_send_pkt()
148 			if (unlikely(vhost_enable_notify(&vsock->dev, vq))) {  in vhost_transport_do_send_pkt()
149 				vhost_disable_notify(&vsock->dev, vq);  in vhost_transport_do_send_pkt()
157 			vq_err(vq, "Expected 0 output buffers, got %u\n", out);  in vhost_transport_do_send_pkt()
161 		iov_len = iov_length(&vq->iov[out], in);  in vhost_transport_do_send_pkt()
164 			vq_err(vq, "Buffer len [%zu] too small\n", iov_len);  in vhost_transport_do_send_pkt()
168 		iov_iter_init(&iov_iter, READ, &vq->iov[out], in, iov_len);  in vhost_transport_do_send_pkt()
205 			vq_err(vq, "Faulted on copying pkt hdr\n");  in vhost_transport_do_send_pkt()
213 			vq_err(vq, "Faulted on copying pkt buf\n");  in vhost_transport_do_send_pkt()
222 		vhost_add_used(vq, head, sizeof(pkt->hdr) + payload_len);  in vhost_transport_do_send_pkt()
258 	} while(likely(!vhost_exceeds_weight(vq, ++pkts, total_len)));  in vhost_transport_do_send_pkt()
260 		vhost_signal(&vsock->dev, vq);  in vhost_transport_do_send_pkt()
263 	mutex_unlock(&vq->mutex);  in vhost_transport_do_send_pkt()
271 	struct vhost_virtqueue *vq;  in vhost_transport_send_pkt_work()  local
275 	vq = &vsock->vqs[VSOCK_VQ_RX];  in vhost_transport_send_pkt_work()
277 	vhost_transport_do_send_pkt(vsock, vq);  in vhost_transport_send_pkt_work()
356 vhost_vsock_alloc_pkt(struct vhost_virtqueue *vq,  in vhost_vsock_alloc_pkt()  argument
365 		vq_err(vq, "Expected 0 input buffers, got %u\n", in);  in vhost_vsock_alloc_pkt()
373 	len = iov_length(vq->iov, out);  in vhost_vsock_alloc_pkt()
374 	iov_iter_init(&iov_iter, WRITE, vq->iov, out, len);  in vhost_vsock_alloc_pkt()
378 		vq_err(vq, "Expected %zu bytes for pkt->hdr, got %zu bytes\n",  in vhost_vsock_alloc_pkt()
406 		vq_err(vq, "Expected %u byte payload, got %zu bytes\n",  in vhost_vsock_alloc_pkt()
418 	struct vhost_virtqueue *vq = &vsock->vqs[VSOCK_VQ_TX];  in vhost_vsock_more_replies()  local
424 	return val < vq->num;  in vhost_vsock_more_replies()
495 	struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,  in vhost_vsock_handle_tx_kick()  local
497 	struct vhost_vsock *vsock = container_of(vq->dev, struct vhost_vsock,  in vhost_vsock_handle_tx_kick()
504 	mutex_lock(&vq->mutex);  in vhost_vsock_handle_tx_kick()
506 	if (!vhost_vq_get_backend(vq))  in vhost_vsock_handle_tx_kick()
509 	if (!vq_meta_prefetch(vq))  in vhost_vsock_handle_tx_kick()
512 	vhost_disable_notify(&vsock->dev, vq);  in vhost_vsock_handle_tx_kick()
524 		head = vhost_get_vq_desc(vq, vq->iov, ARRAY_SIZE(vq->iov),  in vhost_vsock_handle_tx_kick()
529 		if (head == vq->num) {  in vhost_vsock_handle_tx_kick()
530 			if (unlikely(vhost_enable_notify(&vsock->dev, vq))) {  in vhost_vsock_handle_tx_kick()
531 				vhost_disable_notify(&vsock->dev, vq);  in vhost_vsock_handle_tx_kick()
537 		pkt = vhost_vsock_alloc_pkt(vq, out, in);  in vhost_vsock_handle_tx_kick()
539 			vq_err(vq, "Faulted on pkt\n");  in vhost_vsock_handle_tx_kick()
557 		vhost_add_used(vq, head, 0);  in vhost_vsock_handle_tx_kick()
560 	} while(likely(!vhost_exceeds_weight(vq, ++pkts, total_len)));  in vhost_vsock_handle_tx_kick()
564 		vhost_signal(&vsock->dev, vq);  in vhost_vsock_handle_tx_kick()
567 	mutex_unlock(&vq->mutex);  in vhost_vsock_handle_tx_kick()
572 	struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,  in vhost_vsock_handle_rx_kick()  local
574 	struct vhost_vsock *vsock = container_of(vq->dev, struct vhost_vsock,  in vhost_vsock_handle_rx_kick()
577 	vhost_transport_do_send_pkt(vsock, vq);  in vhost_vsock_handle_rx_kick()
582 	struct vhost_virtqueue *vq;  in vhost_vsock_start()  local
593 		vq = &vsock->vqs[i];  in vhost_vsock_start()
595 		mutex_lock(&vq->mutex);  in vhost_vsock_start()
597 		if (!vhost_vq_access_ok(vq)) {  in vhost_vsock_start()
602 		if (!vhost_vq_get_backend(vq)) {  in vhost_vsock_start()
603 			vhost_vq_set_backend(vq, vsock);  in vhost_vsock_start()
604 			ret = vhost_vq_init_access(vq);  in vhost_vsock_start()
609 		mutex_unlock(&vq->mutex);  in vhost_vsock_start()
621 	vhost_vq_set_backend(vq, NULL);  in vhost_vsock_start()
622 	mutex_unlock(&vq->mutex);  in vhost_vsock_start()
625 		vq = &vsock->vqs[i];  in vhost_vsock_start()
627 		mutex_lock(&vq->mutex);  in vhost_vsock_start()
628 		vhost_vq_set_backend(vq, NULL);  in vhost_vsock_start()
629 		mutex_unlock(&vq->mutex);  in vhost_vsock_start()
650 		struct vhost_virtqueue *vq = &vsock->vqs[i];  in vhost_vsock_stop()  local
652 		mutex_lock(&vq->mutex);  in vhost_vsock_stop()
653 		vhost_vq_set_backend(vq, NULL);  in vhost_vsock_stop()
654 		mutex_unlock(&vq->mutex);  in vhost_vsock_stop()
831 	struct vhost_virtqueue *vq;  in vhost_vsock_set_features()  local
852 		vq = &vsock->vqs[i];  in vhost_vsock_set_features()
853 		mutex_lock(&vq->mutex);  in vhost_vsock_set_features()
854 		vq->acked_features = features;  in vhost_vsock_set_features()
855 		mutex_unlock(&vq->mutex);  in vhost_vsock_set_features()