342 static bool vhost_sock_zcopy(struct socket *sock)  in vhost_sock_zcopy()  argument
345 		sock_flag(sock->sk, SOCK_ZEROCOPY);  in vhost_sock_zcopy()
348 static bool vhost_sock_xdp(struct socket *sock)  in vhost_sock_xdp()  argument
350 	return sock_flag(sock->sk, SOCK_XDP);  in vhost_sock_xdp()
438 	struct socket *sock;  in vhost_net_enable_vq()  local
440 	sock = vq->private_data;  in vhost_net_enable_vq()
441 	if (!sock)  in vhost_net_enable_vq()
444 	return vhost_poll_start(poll, sock->file);  in vhost_net_enable_vq()
461 			   struct socket *sock,  in vhost_tx_batch()  argument
475 	err = sock->ops->sendmsg(sock, msghdr, 0);  in vhost_tx_batch()
486 static int sock_has_rx_data(struct socket *sock)  in sock_has_rx_data()  argument
488 	if (unlikely(!sock))  in sock_has_rx_data()
491 	if (sock->ops->peek_len)  in sock_has_rx_data()
492 		return sock->ops->peek_len(sock);  in sock_has_rx_data()
494 	return skb_queue_empty(&sock->sk->sk_receive_queue);  in sock_has_rx_data()
516 	struct socket *sock;  in vhost_net_busy_poll()  local
527 	sock = rvq->private_data;  in vhost_net_busy_poll()
541 		if ((sock_has_rx_data(sock) &&  in vhost_net_busy_poll()
551 	if (poll_rx || sock_has_rx_data(sock))  in vhost_net_busy_poll()
688 	struct socket *sock = vq->private_data;  in vhost_net_build_xdp()  local
694 	int headroom = vhost_sock_xdp(sock) ? XDP_PACKET_HEADROOM : 0;  in vhost_net_build_xdp()
757 static void handle_tx_copy(struct vhost_net *net, struct socket *sock)  in handle_tx_copy()  argument
773 	bool sock_can_batch = (sock->sk->sk_sndbuf == INT_MAX);  in handle_tx_copy()
779 			vhost_tx_batch(net, nvq, sock, &msg);  in handle_tx_copy()
808 				vhost_tx_batch(net, nvq, sock, &msg);  in handle_tx_copy()
818 			vhost_tx_batch(net, nvq, sock, &msg);  in handle_tx_copy()
828 		err = sock->ops->sendmsg(sock, &msg, len);  in handle_tx_copy()
843 	vhost_tx_batch(net, nvq, sock, &msg);  in handle_tx_copy()
846 static void handle_tx_zerocopy(struct vhost_net *net, struct socket *sock)  in handle_tx_zerocopy()  argument
924 		err = sock->ops->sendmsg(sock, &msg, len);  in handle_tx_zerocopy()
952 	struct socket *sock;  in handle_tx()  local
955 	sock = vq->private_data;  in handle_tx()
956 	if (!sock)  in handle_tx()
965 	if (vhost_sock_zcopy(sock))  in handle_tx()
966 		handle_tx_zerocopy(net, sock);  in handle_tx()
968 		handle_tx_copy(net, sock);  in handle_tx()
974 static int peek_head_len(struct vhost_net_virtqueue *rvq, struct sock *sk)  in peek_head_len()
995 static int vhost_net_rx_peek_head_len(struct vhost_net *net, struct sock *sk,  in vhost_net_rx_peek_head_len()
1118 	struct socket *sock;  in handle_rx()  local
1124 	sock = vq->private_data;  in handle_rx()
1125 	if (!sock)  in handle_rx()
1142 		sock_len = vhost_net_rx_peek_head_len(net, sock->sk,  in handle_rx()
1174 			err = sock->ops->recvmsg(sock, &msg,  in handle_rx()
1188 		err = sock->ops->recvmsg(sock, &msg,  in handle_rx()
1342 	struct socket *sock;  in vhost_net_stop_vq()  local
1347 	sock = vq->private_data;  in vhost_net_stop_vq()
1353 	return sock;  in vhost_net_stop_vq()
1422 	struct socket *sock = sockfd_lookup(fd, &r);  in get_raw_socket()  local
1424 	if (!sock)  in get_raw_socket()
1428 	if (sock->sk->sk_type != SOCK_RAW) {  in get_raw_socket()
1433 	r = sock->ops->getname(sock, (struct sockaddr *)&uaddr.sa, 0);  in get_raw_socket()
1441 	return sock;  in get_raw_socket()
1443 	sockfd_put(sock);  in get_raw_socket()
1469 	struct socket *sock;  in get_tap_socket()  local
1473 	sock = tun_get_socket(file);  in get_tap_socket()
1474 	if (!IS_ERR(sock))  in get_tap_socket()
1475 		return sock;  in get_tap_socket()
1476 	sock = tap_get_socket(file);  in get_tap_socket()
1477 	if (IS_ERR(sock))  in get_tap_socket()
1479 	return sock;  in get_tap_socket()
1484 	struct socket *sock;  in get_socket()  local
1489 	sock = get_raw_socket(fd);  in get_socket()
1490 	if (!IS_ERR(sock))  in get_socket()
1491 		return sock;  in get_socket()
1492 	sock = get_tap_socket(fd);  in get_socket()
1493 	if (!IS_ERR(sock))  in get_socket()
1494 		return sock;  in get_socket()
1500 	struct socket *sock, *oldsock;  in vhost_net_set_backend()  local
1524 	sock = get_socket(fd);  in vhost_net_set_backend()
1525 	if (IS_ERR(sock)) {  in vhost_net_set_backend()
1526 		r = PTR_ERR(sock);  in vhost_net_set_backend()
1532 	if (sock != oldsock) {  in vhost_net_set_backend()
1534 					     sock && vhost_sock_zcopy(sock));  in vhost_net_set_backend()
1541 		vq->private_data = sock;  in vhost_net_set_backend()
1583 	if (sock)  in vhost_net_set_backend()
1584 		sockfd_put(sock);  in vhost_net_set_backend()