• Home
  • Raw
  • Download

Lines Matching refs:psock

13 int __tcp_bpf_recvmsg(struct sock *sk, struct sk_psock *psock,  in __tcp_bpf_recvmsg()  argument
21 msg_rx = list_first_entry_or_null(&psock->ingress_msg, in __tcp_bpf_recvmsg()
71 if (msg_rx == list_last_entry(&psock->ingress_msg, in __tcp_bpf_recvmsg()
85 msg_rx = list_first_entry_or_null(&psock->ingress_msg, in __tcp_bpf_recvmsg()
93 static int bpf_tcp_ingress(struct sock *sk, struct sk_psock *psock, in bpf_tcp_ingress() argument
135 sk_psock_queue_msg(psock, tmp); in bpf_tcp_ingress()
136 sk_psock_data_ready(sk, psock); in bpf_tcp_ingress()
218 struct sk_psock *psock = sk_psock_get(sk); in tcp_bpf_sendmsg_redir() local
221 if (unlikely(!psock)) { in tcp_bpf_sendmsg_redir()
225 ret = ingress ? bpf_tcp_ingress(sk, psock, msg, bytes, flags) : in tcp_bpf_sendmsg_redir()
227 sk_psock_put(sk, psock); in tcp_bpf_sendmsg_redir()
235 struct sk_psock *psock; in tcp_bpf_stream_read() local
239 psock = sk_psock(sk); in tcp_bpf_stream_read()
240 if (likely(psock)) in tcp_bpf_stream_read()
241 empty = list_empty(&psock->ingress_msg); in tcp_bpf_stream_read()
246 static int tcp_bpf_wait_data(struct sock *sk, struct sk_psock *psock, in tcp_bpf_wait_data() argument
261 !list_empty(&psock->ingress_msg) || in tcp_bpf_wait_data()
271 struct sk_psock *psock; in tcp_bpf_recvmsg() local
277 psock = sk_psock_get(sk); in tcp_bpf_recvmsg()
278 if (unlikely(!psock)) in tcp_bpf_recvmsg()
281 sk_psock_queue_empty(psock)) { in tcp_bpf_recvmsg()
282 sk_psock_put(sk, psock); in tcp_bpf_recvmsg()
287 copied = __tcp_bpf_recvmsg(sk, psock, msg, len, flags); in tcp_bpf_recvmsg()
293 data = tcp_bpf_wait_data(sk, psock, flags, timeo, &err); in tcp_bpf_recvmsg()
295 if (!sk_psock_queue_empty(psock)) in tcp_bpf_recvmsg()
298 sk_psock_put(sk, psock); in tcp_bpf_recvmsg()
310 sk_psock_put(sk, psock); in tcp_bpf_recvmsg()
314 static int tcp_bpf_send_verdict(struct sock *sk, struct sk_psock *psock, in tcp_bpf_send_verdict() argument
324 if (psock->eval == __SK_NONE) { in tcp_bpf_send_verdict()
331 psock->eval = sk_psock_msg_verdict(sk, psock, msg); in tcp_bpf_send_verdict()
337 psock->cork_bytes = msg->cork_bytes - msg->sg.size; in tcp_bpf_send_verdict()
338 if (!psock->cork) { in tcp_bpf_send_verdict()
339 psock->cork = kzalloc(sizeof(*psock->cork), in tcp_bpf_send_verdict()
341 if (!psock->cork) in tcp_bpf_send_verdict()
344 memcpy(psock->cork, msg, sizeof(*msg)); in tcp_bpf_send_verdict()
349 if (psock->apply_bytes && psock->apply_bytes < tosend) in tcp_bpf_send_verdict()
350 tosend = psock->apply_bytes; in tcp_bpf_send_verdict()
352 switch (psock->eval) { in tcp_bpf_send_verdict()
359 sk_msg_apply_bytes(psock, tosend); in tcp_bpf_send_verdict()
362 sk_redir = psock->sk_redir; in tcp_bpf_send_verdict()
363 sk_msg_apply_bytes(psock, tosend); in tcp_bpf_send_verdict()
364 if (!psock->apply_bytes) { in tcp_bpf_send_verdict()
366 eval = psock->eval; in tcp_bpf_send_verdict()
367 psock->eval = __SK_NONE; in tcp_bpf_send_verdict()
368 psock->sk_redir = NULL; in tcp_bpf_send_verdict()
370 if (psock->cork) { in tcp_bpf_send_verdict()
372 psock->cork = NULL; in tcp_bpf_send_verdict()
399 sk_msg_apply_bytes(psock, tosend); in tcp_bpf_send_verdict()
405 if (!psock->apply_bytes) { in tcp_bpf_send_verdict()
406 psock->eval = __SK_NONE; in tcp_bpf_send_verdict()
407 if (psock->sk_redir) { in tcp_bpf_send_verdict()
408 sock_put(psock->sk_redir); in tcp_bpf_send_verdict()
409 psock->sk_redir = NULL; in tcp_bpf_send_verdict()
424 struct sk_psock *psock; in tcp_bpf_sendmsg() local
432 psock = sk_psock_get(sk); in tcp_bpf_sendmsg()
433 if (unlikely(!psock)) in tcp_bpf_sendmsg()
450 if (psock->cork) { in tcp_bpf_sendmsg()
451 msg_tx = psock->cork; in tcp_bpf_sendmsg()
474 if (psock->cork_bytes) { in tcp_bpf_sendmsg()
475 if (size > psock->cork_bytes) in tcp_bpf_sendmsg()
476 psock->cork_bytes = 0; in tcp_bpf_sendmsg()
478 psock->cork_bytes -= size; in tcp_bpf_sendmsg()
479 if (psock->cork_bytes && !enospc) in tcp_bpf_sendmsg()
482 psock->eval = __SK_NONE; in tcp_bpf_sendmsg()
483 psock->cork_bytes = 0; in tcp_bpf_sendmsg()
486 err = tcp_bpf_send_verdict(sk, psock, msg_tx, &copied, flags); in tcp_bpf_sendmsg()
495 if (msg_tx && msg_tx != psock->cork) in tcp_bpf_sendmsg()
504 sk_psock_put(sk, psock); in tcp_bpf_sendmsg()
513 struct sk_psock *psock; in tcp_bpf_sendpage() local
516 psock = sk_psock_get(sk); in tcp_bpf_sendpage()
517 if (unlikely(!psock)) in tcp_bpf_sendpage()
521 if (psock->cork) { in tcp_bpf_sendpage()
522 msg = psock->cork; in tcp_bpf_sendpage()
537 if (psock->cork_bytes) { in tcp_bpf_sendpage()
538 if (size > psock->cork_bytes) in tcp_bpf_sendpage()
539 psock->cork_bytes = 0; in tcp_bpf_sendpage()
541 psock->cork_bytes -= size; in tcp_bpf_sendpage()
542 if (psock->cork_bytes && !enospc) in tcp_bpf_sendpage()
545 psock->eval = __SK_NONE; in tcp_bpf_sendpage()
546 psock->cork_bytes = 0; in tcp_bpf_sendpage()
549 err = tcp_bpf_send_verdict(sk, psock, msg, &copied, flags); in tcp_bpf_sendpage()
552 sk_psock_put(sk, psock); in tcp_bpf_sendpage()
615 struct proto *tcp_bpf_get_proto(struct sock *sk, struct sk_psock *psock) in tcp_bpf_get_proto() argument
618 int config = psock->progs.msg_parser ? TCP_BPF_TX : TCP_BPF_BASE; in tcp_bpf_get_proto()
621 if (tcp_bpf_assert_proto_ops(psock->sk_proto)) in tcp_bpf_get_proto()
624 tcp_bpf_check_v6_needs_rebuild(psock->sk_proto); in tcp_bpf_get_proto()