Lines Matching refs:psock
422 struct sk_psock *psock, in sk_psock_skb_ingress_enqueue() argument
445 sk_psock_queue_msg(psock, msg); in sk_psock_skb_ingress_enqueue()
446 sk_psock_data_ready(sk, psock); in sk_psock_skb_ingress_enqueue()
450 static int sk_psock_skb_ingress_self(struct sk_psock *psock, struct sk_buff *skb);
452 static int sk_psock_skb_ingress(struct sk_psock *psock, struct sk_buff *skb) in sk_psock_skb_ingress() argument
454 struct sock *sk = psock->sk; in sk_psock_skb_ingress()
463 return sk_psock_skb_ingress_self(psock, skb); in sk_psock_skb_ingress()
475 err = sk_psock_skb_ingress_enqueue(skb, psock, sk, msg); in sk_psock_skb_ingress()
485 static int sk_psock_skb_ingress_self(struct sk_psock *psock, struct sk_buff *skb) in sk_psock_skb_ingress_self() argument
488 struct sock *sk = psock->sk; in sk_psock_skb_ingress_self()
495 err = sk_psock_skb_ingress_enqueue(skb, psock, sk, msg); in sk_psock_skb_ingress_self()
501 static int sk_psock_handle_skb(struct sk_psock *psock, struct sk_buff *skb, in sk_psock_handle_skb() argument
505 if (!sock_writeable(psock->sk)) in sk_psock_handle_skb()
507 return skb_send_sock_locked(psock->sk, skb, off, len); in sk_psock_handle_skb()
509 return sk_psock_skb_ingress(psock, skb); in sk_psock_handle_skb()
514 struct sk_psock *psock = container_of(work, struct sk_psock, work); in sk_psock_backlog() local
515 struct sk_psock_work_state *state = &psock->work_state; in sk_psock_backlog()
522 lock_sock(psock->sk); in sk_psock_backlog()
531 while ((skb = skb_dequeue(&psock->ingress_skb))) { in sk_psock_backlog()
538 if (likely(psock->sk->sk_socket)) in sk_psock_backlog()
539 ret = sk_psock_handle_skb(psock, skb, off, in sk_psock_backlog()
549 sk_psock_report_error(psock, ret ? -ret : EPIPE); in sk_psock_backlog()
550 sk_psock_clear_state(psock, SK_PSOCK_TX_ENABLED); in sk_psock_backlog()
562 release_sock(psock->sk); in sk_psock_backlog()
567 struct sk_psock *psock; in sk_psock_init() local
573 psock = ERR_PTR(-EINVAL); in sk_psock_init()
578 psock = ERR_PTR(-EBUSY); in sk_psock_init()
582 psock = kzalloc_node(sizeof(*psock), GFP_ATOMIC | __GFP_NOWARN, node); in sk_psock_init()
583 if (!psock) { in sk_psock_init()
584 psock = ERR_PTR(-ENOMEM); in sk_psock_init()
589 psock->sk = sk; in sk_psock_init()
590 psock->eval = __SK_NONE; in sk_psock_init()
591 psock->sk_proto = prot; in sk_psock_init()
592 psock->saved_unhash = prot->unhash; in sk_psock_init()
593 psock->saved_close = prot->close; in sk_psock_init()
594 psock->saved_write_space = sk->sk_write_space; in sk_psock_init()
596 INIT_LIST_HEAD(&psock->link); in sk_psock_init()
597 spin_lock_init(&psock->link_lock); in sk_psock_init()
599 INIT_WORK(&psock->work, sk_psock_backlog); in sk_psock_init()
600 INIT_LIST_HEAD(&psock->ingress_msg); in sk_psock_init()
601 skb_queue_head_init(&psock->ingress_skb); in sk_psock_init()
603 sk_psock_set_state(psock, SK_PSOCK_TX_ENABLED); in sk_psock_init()
604 refcount_set(&psock->refcnt, 1); in sk_psock_init()
606 rcu_assign_sk_user_data_nocopy(sk, psock); in sk_psock_init()
611 return psock; in sk_psock_init()
615 struct sk_psock_link *sk_psock_link_pop(struct sk_psock *psock) in sk_psock_link_pop() argument
619 spin_lock_bh(&psock->link_lock); in sk_psock_link_pop()
620 link = list_first_entry_or_null(&psock->link, struct sk_psock_link, in sk_psock_link_pop()
624 spin_unlock_bh(&psock->link_lock); in sk_psock_link_pop()
628 void __sk_psock_purge_ingress_msg(struct sk_psock *psock) in __sk_psock_purge_ingress_msg() argument
632 list_for_each_entry_safe(msg, tmp, &psock->ingress_msg, list) { in __sk_psock_purge_ingress_msg()
634 sk_msg_free(psock->sk, msg); in __sk_psock_purge_ingress_msg()
639 static void sk_psock_zap_ingress(struct sk_psock *psock) in sk_psock_zap_ingress() argument
641 __skb_queue_purge(&psock->ingress_skb); in sk_psock_zap_ingress()
642 __sk_psock_purge_ingress_msg(psock); in sk_psock_zap_ingress()
645 static void sk_psock_link_destroy(struct sk_psock *psock) in sk_psock_link_destroy() argument
649 list_for_each_entry_safe(link, tmp, &psock->link, list) { in sk_psock_link_destroy()
657 struct sk_psock *psock = container_of(gc, struct sk_psock, gc); in sk_psock_destroy_deferred() local
662 if (psock->progs.skb_parser) in sk_psock_destroy_deferred()
663 strp_done(&psock->parser.strp); in sk_psock_destroy_deferred()
665 cancel_work_sync(&psock->work); in sk_psock_destroy_deferred()
667 psock_progs_drop(&psock->progs); in sk_psock_destroy_deferred()
669 sk_psock_link_destroy(psock); in sk_psock_destroy_deferred()
670 sk_psock_cork_free(psock); in sk_psock_destroy_deferred()
671 sk_psock_zap_ingress(psock); in sk_psock_destroy_deferred()
673 if (psock->sk_redir) in sk_psock_destroy_deferred()
674 sock_put(psock->sk_redir); in sk_psock_destroy_deferred()
675 sock_put(psock->sk); in sk_psock_destroy_deferred()
676 kfree(psock); in sk_psock_destroy_deferred()
681 struct sk_psock *psock = container_of(rcu, struct sk_psock, rcu); in sk_psock_destroy() local
683 INIT_WORK(&psock->gc, sk_psock_destroy_deferred); in sk_psock_destroy()
684 schedule_work(&psock->gc); in sk_psock_destroy()
687 void sk_psock_drop(struct sock *sk, struct sk_psock *psock) in sk_psock_drop() argument
689 sk_psock_cork_free(psock); in sk_psock_drop()
690 sk_psock_zap_ingress(psock); in sk_psock_drop()
693 sk_psock_restore_proto(sk, psock); in sk_psock_drop()
695 if (psock->progs.skb_parser) in sk_psock_drop()
696 sk_psock_stop_strp(sk, psock); in sk_psock_drop()
697 else if (psock->progs.skb_verdict) in sk_psock_drop()
698 sk_psock_stop_verdict(sk, psock); in sk_psock_drop()
700 sk_psock_clear_state(psock, SK_PSOCK_TX_ENABLED); in sk_psock_drop()
702 call_rcu(&psock->rcu, sk_psock_destroy); in sk_psock_drop()
719 int sk_psock_msg_verdict(struct sock *sk, struct sk_psock *psock, in sk_psock_msg_verdict() argument
726 prog = READ_ONCE(psock->progs.msg_parser); in sk_psock_msg_verdict()
736 psock->apply_bytes = msg->apply_bytes; in sk_psock_msg_verdict()
738 if (psock->sk_redir) in sk_psock_msg_verdict()
739 sock_put(psock->sk_redir); in sk_psock_msg_verdict()
740 psock->sk_redir = msg->sk_redir; in sk_psock_msg_verdict()
741 if (!psock->sk_redir) { in sk_psock_msg_verdict()
745 sock_hold(psock->sk_redir); in sk_psock_msg_verdict()
753 static int sk_psock_bpf_run(struct sk_psock *psock, struct bpf_prog *prog, in sk_psock_bpf_run() argument
809 int sk_psock_tls_strp_read(struct sk_psock *psock, struct sk_buff *skb) in sk_psock_tls_strp_read() argument
815 prog = READ_ONCE(psock->progs.skb_verdict); in sk_psock_tls_strp_read()
817 skb->sk = psock->sk; in sk_psock_tls_strp_read()
819 ret = sk_psock_bpf_run(psock, prog, skb); in sk_psock_tls_strp_read()
823 sk_psock_tls_verdict_apply(skb, psock->sk, ret); in sk_psock_tls_strp_read()
829 static void sk_psock_verdict_apply(struct sk_psock *psock, in sk_psock_verdict_apply() argument
838 sk_other = psock->sk; in sk_psock_verdict_apply()
840 !sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED)) { in sk_psock_verdict_apply()
853 if (skb_queue_empty(&psock->ingress_skb)) { in sk_psock_verdict_apply()
854 err = sk_psock_skb_ingress_self(psock, skb); in sk_psock_verdict_apply()
857 skb_queue_tail(&psock->ingress_skb, skb); in sk_psock_verdict_apply()
858 schedule_work(&psock->work); in sk_psock_verdict_apply()
873 struct sk_psock *psock; in sk_psock_strp_read() local
880 psock = sk_psock(sk); in sk_psock_strp_read()
881 if (unlikely(!psock)) { in sk_psock_strp_read()
885 prog = READ_ONCE(psock->progs.skb_verdict); in sk_psock_strp_read()
889 ret = sk_psock_bpf_run(psock, prog, skb); in sk_psock_strp_read()
893 sk_psock_verdict_apply(psock, skb, ret); in sk_psock_strp_read()
905 struct sk_psock *psock = sk_psock_from_strp(strp); in sk_psock_strp_parse() local
910 prog = READ_ONCE(psock->progs.skb_parser); in sk_psock_strp_parse()
912 skb->sk = psock->sk; in sk_psock_strp_parse()
913 ret = sk_psock_bpf_run(psock, prog, skb); in sk_psock_strp_parse()
923 struct sk_psock *psock; in sk_psock_strp_data_ready() local
926 psock = sk_psock(sk); in sk_psock_strp_data_ready()
927 if (likely(psock)) { in sk_psock_strp_data_ready()
929 psock->parser.saved_data_ready(sk); in sk_psock_strp_data_ready()
932 strp_data_ready(&psock->parser.strp); in sk_psock_strp_data_ready()
943 struct sk_psock *psock; in sk_psock_verdict_recv() local
956 psock = sk_psock(sk); in sk_psock_verdict_recv()
957 if (unlikely(!psock)) { in sk_psock_verdict_recv()
962 prog = READ_ONCE(psock->progs.skb_verdict); in sk_psock_verdict_recv()
966 ret = sk_psock_bpf_run(psock, prog, skb); in sk_psock_verdict_recv()
970 sk_psock_verdict_apply(psock, skb, ret); in sk_psock_verdict_recv()
993 struct sk_psock *psock; in sk_psock_write_space() local
997 psock = sk_psock(sk); in sk_psock_write_space()
998 if (likely(psock)) { in sk_psock_write_space()
999 if (sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED)) in sk_psock_write_space()
1000 schedule_work(&psock->work); in sk_psock_write_space()
1001 write_space = psock->saved_write_space; in sk_psock_write_space()
1008 int sk_psock_init_strp(struct sock *sk, struct sk_psock *psock) in sk_psock_init_strp() argument
1016 psock->parser.enabled = false; in sk_psock_init_strp()
1017 return strp_init(&psock->parser.strp, sk, &cb); in sk_psock_init_strp()
1020 void sk_psock_start_verdict(struct sock *sk, struct sk_psock *psock) in sk_psock_start_verdict() argument
1022 struct sk_psock_parser *parser = &psock->parser; in sk_psock_start_verdict()
1033 void sk_psock_start_strp(struct sock *sk, struct sk_psock *psock) in sk_psock_start_strp() argument
1035 struct sk_psock_parser *parser = &psock->parser; in sk_psock_start_strp()
1046 void sk_psock_stop_strp(struct sock *sk, struct sk_psock *psock) in sk_psock_stop_strp() argument
1048 struct sk_psock_parser *parser = &psock->parser; in sk_psock_stop_strp()
1059 void sk_psock_stop_verdict(struct sock *sk, struct sk_psock *psock) in sk_psock_stop_verdict() argument
1061 struct sk_psock_parser *parser = &psock->parser; in sk_psock_stop_verdict()