46 		      struct msghdr *msg, int len, int flags)  in __tcp_bpf_recvmsg()  argument
48 	struct iov_iter *iter = &msg->msg_iter;  in __tcp_bpf_recvmsg()
118 int tcp_bpf_recvmsg(struct sock *sk, struct msghdr *msg, size_t len,  in tcp_bpf_recvmsg()  argument
126 		return tcp_recvmsg(sk, msg, len, nonblock, flags, addr_len);  in tcp_bpf_recvmsg()
128 		return inet_recv_error(sk, msg, len, addr_len);  in tcp_bpf_recvmsg()
131 		return tcp_recvmsg(sk, msg, len, nonblock, flags, addr_len);  in tcp_bpf_recvmsg()
134 	copied = __tcp_bpf_recvmsg(sk, psock, msg, len, flags);  in tcp_bpf_recvmsg()
146 			return tcp_recvmsg(sk, msg, len, nonblock, flags, addr_len);  in tcp_bpf_recvmsg()
162 			   struct sk_msg *msg, u32 apply_bytes, int flags)  in bpf_tcp_ingress()  argument
175 	tmp->sg.start = msg->sg.start;  in bpf_tcp_ingress()
176 	i = msg->sg.start;  in bpf_tcp_ingress()
178 		sge = sk_msg_elem(msg, i);  in bpf_tcp_ingress()
188 		sk_msg_xfer(tmp, msg, i, size);  in bpf_tcp_ingress()
199 	} while (i != msg->sg.end);  in bpf_tcp_ingress()
202 		msg->sg.start = i;  in bpf_tcp_ingress()
203 		msg->sg.size -= apply_bytes;  in bpf_tcp_ingress()
215 static int tcp_bpf_push(struct sock *sk, struct sk_msg *msg, u32 apply_bytes,  in tcp_bpf_push()  argument
227 		sge = sk_msg_elem(msg, msg->sg.start);  in tcp_bpf_push()
248 		msg->sg.size -= ret;  in tcp_bpf_push()
260 			sk_msg_iter_next(msg, start);  in tcp_bpf_push()
262 			if (msg->sg.start == msg->sg.end)  in tcp_bpf_push()
272 static int tcp_bpf_push_locked(struct sock *sk, struct sk_msg *msg,  in tcp_bpf_push_locked()  argument
278 	ret = tcp_bpf_push(sk, msg, apply_bytes, flags, uncharge);  in tcp_bpf_push_locked()
283 int tcp_bpf_sendmsg_redir(struct sock *sk, struct sk_msg *msg,  in tcp_bpf_sendmsg_redir()  argument
286 	bool ingress = sk_msg_to_ingress(msg);  in tcp_bpf_sendmsg_redir()
291 		sk_msg_free(sk, msg);  in tcp_bpf_sendmsg_redir()
294 	ret = ingress ? bpf_tcp_ingress(sk, psock, msg, bytes, flags) :  in tcp_bpf_sendmsg_redir()
295 			tcp_bpf_push_locked(sk, msg, bytes, flags, false);  in tcp_bpf_sendmsg_redir()
302 				struct sk_msg *msg, int *copied, int flags)  in tcp_bpf_send_verdict()  argument
304 	bool cork = false, enospc = sk_msg_full(msg);  in tcp_bpf_send_verdict()
316 		delta = msg->sg.size;  in tcp_bpf_send_verdict()
317 		psock->eval = sk_psock_msg_verdict(sk, psock, msg);  in tcp_bpf_send_verdict()
318 		delta -= msg->sg.size;  in tcp_bpf_send_verdict()
321 	if (msg->cork_bytes &&  in tcp_bpf_send_verdict()
322 	    msg->cork_bytes > msg->sg.size && !enospc) {  in tcp_bpf_send_verdict()
323 		psock->cork_bytes = msg->cork_bytes - msg->sg.size;  in tcp_bpf_send_verdict()
330 		memcpy(psock->cork, msg, sizeof(*msg));  in tcp_bpf_send_verdict()
334 	tosend = msg->sg.size;  in tcp_bpf_send_verdict()
340 		ret = tcp_bpf_push(sk, msg, tosend, flags, true);  in tcp_bpf_send_verdict()
342 			*copied -= sk_msg_free(sk, msg);  in tcp_bpf_send_verdict()
354 		sk_msg_return(sk, msg, tosend);  in tcp_bpf_send_verdict()
356 		ret = tcp_bpf_sendmsg_redir(sk_redir, msg, tosend, flags);  in tcp_bpf_send_verdict()
359 			int free = sk_msg_free_nocharge(sk, msg);  in tcp_bpf_send_verdict()
365 			sk_msg_free(sk, msg);  in tcp_bpf_send_verdict()
366 			kfree(msg);  in tcp_bpf_send_verdict()
367 			msg = NULL;  in tcp_bpf_send_verdict()
373 		sk_msg_free_partial(sk, msg, tosend);  in tcp_bpf_send_verdict()
387 		if (msg &&  in tcp_bpf_send_verdict()
388 		    msg->sg.data[msg->sg.start].page_link &&  in tcp_bpf_send_verdict()
389 		    msg->sg.data[msg->sg.start].length)  in tcp_bpf_send_verdict()
395 static int tcp_bpf_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)  in tcp_bpf_sendmsg()  argument
404 	flags = (msg->msg_flags & ~MSG_SENDPAGE_DECRYPTED);  in tcp_bpf_sendmsg()
409 		return tcp_sendmsg(sk, msg, size);  in tcp_bpf_sendmsg()
412 	timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT);  in tcp_bpf_sendmsg()
413 	while (msg_data_left(msg)) {  in tcp_bpf_sendmsg()
422 		copy = msg_data_left(msg);  in tcp_bpf_sendmsg()
441 		err = sk_msg_memcopy_from_iter(sk, &msg->msg_iter, msg_tx,  in tcp_bpf_sendmsg()
477 		err = sk_stream_error(sk, msg->msg_flags, err);  in tcp_bpf_sendmsg()
486 	struct sk_msg tmp, *msg = NULL;  in tcp_bpf_sendpage()  local
497 		msg = psock->cork;  in tcp_bpf_sendpage()
499 		msg = &tmp;  in tcp_bpf_sendpage()
500 		sk_msg_init(msg);  in tcp_bpf_sendpage()
504 	if (unlikely(sk_msg_full(msg)))  in tcp_bpf_sendpage()
507 	sk_msg_page_add(msg, page, size, offset);  in tcp_bpf_sendpage()
510 	if (sk_msg_full(msg))  in tcp_bpf_sendpage()
524 	err = tcp_bpf_send_verdict(sk, psock, msg, &copied, flags);  in tcp_bpf_sendpage()