15 		      struct msghdr *msg, int len, int flags)  in __tcp_bpf_recvmsg()  argument
17 	struct iov_iter *iter = &msg->msg_iter;  in __tcp_bpf_recvmsg()
95 			   struct sk_msg *msg, u32 apply_bytes, int flags)  in bpf_tcp_ingress()  argument
108 	tmp->sg.start = msg->sg.start;  in bpf_tcp_ingress()
109 	i = msg->sg.start;  in bpf_tcp_ingress()
111 		sge = sk_msg_elem(msg, i);  in bpf_tcp_ingress()
121 		sk_msg_xfer(tmp, msg, i, size);  in bpf_tcp_ingress()
135 	} while (i != msg->sg.end);  in bpf_tcp_ingress()
138 		msg->sg.start = i;  in bpf_tcp_ingress()
150 static int tcp_bpf_push(struct sock *sk, struct sk_msg *msg, u32 apply_bytes,  in tcp_bpf_push()  argument
162 		sge = sk_msg_elem(msg, msg->sg.start);  in tcp_bpf_push()
183 		msg->sg.size -= ret;  in tcp_bpf_push()
195 			sk_msg_iter_next(msg, start);  in tcp_bpf_push()
197 			if (msg->sg.start == msg->sg.end)  in tcp_bpf_push()
207 static int tcp_bpf_push_locked(struct sock *sk, struct sk_msg *msg,  in tcp_bpf_push_locked()  argument
213 	ret = tcp_bpf_push(sk, msg, apply_bytes, flags, uncharge);  in tcp_bpf_push_locked()
218 int tcp_bpf_sendmsg_redir(struct sock *sk, struct sk_msg *msg,  in tcp_bpf_sendmsg_redir()  argument
221 	bool ingress = sk_msg_to_ingress(msg);  in tcp_bpf_sendmsg_redir()
228 	ret = ingress ? bpf_tcp_ingress(sk, psock, msg, bytes, flags) :  in tcp_bpf_sendmsg_redir()
229 			tcp_bpf_push_locked(sk, msg, bytes, flags, false);  in tcp_bpf_sendmsg_redir()
271 static int tcp_bpf_recvmsg(struct sock *sk, struct msghdr *msg, size_t len,  in tcp_bpf_recvmsg()  argument
278 		return inet_recv_error(sk, msg, len, addr_len);  in tcp_bpf_recvmsg()
282 		return tcp_recvmsg(sk, msg, len, nonblock, flags, addr_len);  in tcp_bpf_recvmsg()
286 		return tcp_recvmsg(sk, msg, len, nonblock, flags, addr_len);  in tcp_bpf_recvmsg()
290 	copied = __tcp_bpf_recvmsg(sk, psock, msg, len, flags);  in tcp_bpf_recvmsg()
302 			return tcp_recvmsg(sk, msg, len, nonblock, flags, addr_len);  in tcp_bpf_recvmsg()
318 				struct sk_msg *msg, int *copied, int flags)  in tcp_bpf_send_verdict()  argument
320 	bool cork = false, enospc = sk_msg_full(msg);  in tcp_bpf_send_verdict()
333 		delta = msg->sg.size;  in tcp_bpf_send_verdict()
334 		psock->eval = sk_psock_msg_verdict(sk, psock, msg);  in tcp_bpf_send_verdict()
335 		delta -= msg->sg.size;  in tcp_bpf_send_verdict()
338 	if (msg->cork_bytes &&  in tcp_bpf_send_verdict()
339 	    msg->cork_bytes > msg->sg.size && !enospc) {  in tcp_bpf_send_verdict()
340 		psock->cork_bytes = msg->cork_bytes - msg->sg.size;  in tcp_bpf_send_verdict()
347 		memcpy(psock->cork, msg, sizeof(*msg));  in tcp_bpf_send_verdict()
351 	tosend = msg->sg.size;  in tcp_bpf_send_verdict()
358 		ret = tcp_bpf_push(sk, msg, tosend, flags, true);  in tcp_bpf_send_verdict()
360 			*copied -= sk_msg_free(sk, msg);  in tcp_bpf_send_verdict()
378 		sk_msg_return(sk, msg, tosend);  in tcp_bpf_send_verdict()
381 		origsize = msg->sg.size;  in tcp_bpf_send_verdict()
382 		ret = tcp_bpf_sendmsg_redir(sk_redir, msg, tosend, flags);  in tcp_bpf_send_verdict()
383 		sent = origsize - msg->sg.size;  in tcp_bpf_send_verdict()
390 			int free = sk_msg_free_nocharge(sk, msg);  in tcp_bpf_send_verdict()
396 			sk_msg_free(sk, msg);  in tcp_bpf_send_verdict()
397 			kfree(msg);  in tcp_bpf_send_verdict()
398 			msg = NULL;  in tcp_bpf_send_verdict()
404 		sk_msg_free_partial(sk, msg, tosend);  in tcp_bpf_send_verdict()
418 		if (msg &&  in tcp_bpf_send_verdict()
419 		    msg->sg.data[msg->sg.start].page_link &&  in tcp_bpf_send_verdict()
420 		    msg->sg.data[msg->sg.start].length) {  in tcp_bpf_send_verdict()
429 static int tcp_bpf_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)  in tcp_bpf_sendmsg()  argument
438 	flags = (msg->msg_flags & ~MSG_SENDPAGE_DECRYPTED);  in tcp_bpf_sendmsg()
443 		return tcp_sendmsg(sk, msg, size);  in tcp_bpf_sendmsg()
446 	timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT);  in tcp_bpf_sendmsg()
447 	while (msg_data_left(msg)) {  in tcp_bpf_sendmsg()
456 		copy = msg_data_left(msg);  in tcp_bpf_sendmsg()
475 		err = sk_msg_memcopy_from_iter(sk, &msg->msg_iter, msg_tx,  in tcp_bpf_sendmsg()
511 		err = sk_stream_error(sk, msg->msg_flags, err);  in tcp_bpf_sendmsg()
520 	struct sk_msg tmp, *msg = NULL;  in tcp_bpf_sendpage()  local
531 		msg = psock->cork;  in tcp_bpf_sendpage()
533 		msg = &tmp;  in tcp_bpf_sendpage()
534 		sk_msg_init(msg);  in tcp_bpf_sendpage()
538 	if (unlikely(sk_msg_full(msg)))  in tcp_bpf_sendpage()
541 	sk_msg_page_add(msg, page, size, offset);  in tcp_bpf_sendpage()
544 	if (sk_msg_full(msg))  in tcp_bpf_sendpage()
558 	err = tcp_bpf_send_verdict(sk, psock, msg, &copied, flags);  in tcp_bpf_sendpage()