Lines Matching refs:sk
49 static u32 sk_ehashfn(const struct sock *sk) in sk_ehashfn() argument
52 if (sk->sk_family == AF_INET6 && in sk_ehashfn()
53 !ipv6_addr_v4mapped(&sk->sk_v6_daddr)) in sk_ehashfn()
54 return inet6_ehashfn(sock_net(sk), in sk_ehashfn()
55 &sk->sk_v6_rcv_saddr, sk->sk_num, in sk_ehashfn()
56 &sk->sk_v6_daddr, sk->sk_dport); in sk_ehashfn()
61 if (sk->sk_family == AF_NINET) in sk_ehashfn()
62 return ninet_ehashfn(sock_net(sk), in sk_ehashfn()
63 &sk->sk_nip_rcv_saddr, sk->sk_num, in sk_ehashfn()
64 &sk->sk_nip_daddr, sk->sk_dport); in sk_ehashfn()
67 return inet_ehashfn(sock_net(sk), in sk_ehashfn()
68 sk->sk_rcv_saddr, sk->sk_num, in sk_ehashfn()
69 sk->sk_daddr, sk->sk_dport); in sk_ehashfn()
107 void inet_bind_hash(struct sock *sk, struct inet_bind_bucket *tb, in inet_bind_hash() argument
110 inet_sk(sk)->inet_num = snum; in inet_bind_hash()
111 sk_add_bind_node(sk, &tb->owners); in inet_bind_hash()
112 inet_csk(sk)->icsk_bind_hash = tb; in inet_bind_hash()
118 static void __inet_put_port(struct sock *sk) in __inet_put_port() argument
120 struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo; in __inet_put_port()
121 const int bhash = inet_bhashfn(sock_net(sk), inet_sk(sk)->inet_num, in __inet_put_port()
127 tb = inet_csk(sk)->icsk_bind_hash; in __inet_put_port()
128 __sk_del_bind_node(sk); in __inet_put_port()
129 inet_csk(sk)->icsk_bind_hash = NULL; in __inet_put_port()
130 inet_sk(sk)->inet_num = 0; in __inet_put_port()
135 void inet_put_port(struct sock *sk) in inet_put_port() argument
138 __inet_put_port(sk); in inet_put_port()
143 int __inet_inherit_port(const struct sock *sk, struct sock *child) in __inet_inherit_port() argument
145 struct inet_hashinfo *table = sk->sk_prot->h.hashinfo; in __inet_inherit_port()
147 const int bhash = inet_bhashfn(sock_net(sk), port, in __inet_inherit_port()
154 tb = inet_csk(sk)->icsk_bind_hash; in __inet_inherit_port()
160 l3mdev = inet_sk_bound_l3mdev(sk); in __inet_inherit_port()
168 if (net_eq(ib_net(tb), sock_net(sk)) && in __inet_inherit_port()
174 sock_net(sk), head, port, in __inet_inherit_port()
191 inet_lhash2_bucket_sk(struct inet_hashinfo *h, struct sock *sk) in inet_lhash2_bucket_sk() argument
196 if (sk->sk_family == AF_INET6) in inet_lhash2_bucket_sk()
197 hash = ipv6_portaddr_hash(sock_net(sk), in inet_lhash2_bucket_sk()
198 &sk->sk_v6_rcv_saddr, in inet_lhash2_bucket_sk()
199 inet_sk(sk)->inet_num); in inet_lhash2_bucket_sk()
202 hash = ipv4_portaddr_hash(sock_net(sk), in inet_lhash2_bucket_sk()
203 inet_sk(sk)->inet_rcv_saddr, in inet_lhash2_bucket_sk()
204 inet_sk(sk)->inet_num); in inet_lhash2_bucket_sk()
208 static void inet_hash2(struct inet_hashinfo *h, struct sock *sk) in inet_hash2() argument
215 ilb2 = inet_lhash2_bucket_sk(h, sk); in inet_hash2()
218 if (sk->sk_reuseport && sk->sk_family == AF_INET6) in inet_hash2()
219 hlist_add_tail_rcu(&inet_csk(sk)->icsk_listen_portaddr_node, in inet_hash2()
222 hlist_add_head_rcu(&inet_csk(sk)->icsk_listen_portaddr_node, in inet_hash2()
228 static void inet_unhash2(struct inet_hashinfo *h, struct sock *sk) in inet_unhash2() argument
233 WARN_ON_ONCE(hlist_unhashed(&inet_csk(sk)->icsk_listen_portaddr_node))) in inet_unhash2()
236 ilb2 = inet_lhash2_bucket_sk(h, sk); in inet_unhash2()
239 hlist_del_init_rcu(&inet_csk(sk)->icsk_listen_portaddr_node); in inet_unhash2()
244 static inline int compute_score(struct sock *sk, struct net *net, in compute_score() argument
250 if (net_eq(sock_net(sk), net) && sk->sk_num == hnum && in compute_score()
251 !ipv6_only_sock(sk)) { in compute_score()
252 if (sk->sk_rcv_saddr != daddr) in compute_score()
255 if (!inet_sk_bound_dev_eq(net, sk->sk_bound_dev_if, dif, sdif)) in compute_score()
257 score = sk->sk_bound_dev_if ? 2 : 1; in compute_score()
259 if (sk->sk_family == PF_INET) in compute_score()
261 if (READ_ONCE(sk->sk_incoming_cpu) == raw_smp_processor_id()) in compute_score()
267 static inline struct sock *lookup_reuseport(struct net *net, struct sock *sk, in lookup_reuseport() argument
275 if (sk->sk_reuseport) { in lookup_reuseport()
277 reuse_sk = reuseport_select_sock(sk, phash, skb, doff); in lookup_reuseport()
298 struct sock *sk, *result = NULL; in inet_lhash2_lookup() local
302 sk = (struct sock *)icsk; in inet_lhash2_lookup()
303 score = compute_score(sk, net, hnum, daddr, dif, sdif); in inet_lhash2_lookup()
305 result = lookup_reuseport(net, sk, skb, doff, in inet_lhash2_lookup()
310 result = sk; in inet_lhash2_lookup()
324 struct sock *sk, *reuse_sk; in inet_lookup_run_bpf() local
331 saddr, sport, daddr, hnum, &sk); in inet_lookup_run_bpf()
332 if (no_reuseport || IS_ERR_OR_NULL(sk)) in inet_lookup_run_bpf()
333 return sk; in inet_lookup_run_bpf()
335 reuse_sk = lookup_reuseport(net, sk, skb, doff, saddr, sport, daddr, hnum); in inet_lookup_run_bpf()
337 sk = reuse_sk; in inet_lookup_run_bpf()
338 return sk; in inet_lookup_run_bpf()
384 void sock_gen_put(struct sock *sk) in sock_gen_put() argument
386 if (!refcount_dec_and_test(&sk->sk_refcnt)) in sock_gen_put()
389 if (sk->sk_state == TCP_TIME_WAIT) in sock_gen_put()
390 inet_twsk_free(inet_twsk(sk)); in sock_gen_put()
391 else if (sk->sk_state == TCP_NEW_SYN_RECV) in sock_gen_put()
392 reqsk_free(inet_reqsk(sk)); in sock_gen_put()
394 sk_free(sk); in sock_gen_put()
400 sock_gen_put(skb->sk); in sock_edemux()
412 struct sock *sk; in __inet_lookup_established() local
422 sk_nulls_for_each_rcu(sk, node, &head->chain) { in __inet_lookup_established()
423 if (sk->sk_hash != hash) in __inet_lookup_established()
425 if (likely(INET_MATCH(sk, net, acookie, in __inet_lookup_established()
427 if (unlikely(!refcount_inc_not_zero(&sk->sk_refcnt))) in __inet_lookup_established()
429 if (unlikely(!INET_MATCH(sk, net, acookie, in __inet_lookup_established()
432 sock_gen_put(sk); in __inet_lookup_established()
446 sk = NULL; in __inet_lookup_established()
448 return sk; in __inet_lookup_established()
454 struct sock *sk, __u16 lport, in __inet_check_established() argument
458 struct inet_sock *inet = inet_sk(sk); in __inet_check_established()
461 int dif = sk->sk_bound_dev_if; in __inet_check_established()
462 struct net *net = sock_net(sk); in __inet_check_established()
484 if (twsk_unique(sk, sk2, twp)) in __inet_check_established()
496 sk->sk_hash = hash; in __inet_check_established()
497 WARN_ON(!sk_unhashed(sk)); in __inet_check_established()
498 __sk_nulls_add_node_rcu(sk, &head->chain); in __inet_check_established()
504 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1); in __inet_check_established()
519 static u64 inet_sk_port_offset(const struct sock *sk) in inet_sk_port_offset() argument
521 const struct inet_sock *inet = inet_sk(sk); in inet_sk_port_offset()
531 static bool inet_ehash_lookup_by_sk(struct sock *sk, in inet_ehash_lookup_by_sk() argument
534 const __portpair ports = INET_COMBINED_PORTS(sk->sk_dport, sk->sk_num); in inet_ehash_lookup_by_sk()
535 const int sdif = sk->sk_bound_dev_if; in inet_ehash_lookup_by_sk()
536 const int dif = sk->sk_bound_dev_if; in inet_ehash_lookup_by_sk()
538 struct net *net = sock_net(sk); in inet_ehash_lookup_by_sk()
541 INET_ADDR_COOKIE(acookie, sk->sk_daddr, sk->sk_rcv_saddr); in inet_ehash_lookup_by_sk()
544 if (esk->sk_hash != sk->sk_hash) in inet_ehash_lookup_by_sk()
546 if (sk->sk_family == AF_INET) { in inet_ehash_lookup_by_sk()
548 sk->sk_daddr, in inet_ehash_lookup_by_sk()
549 sk->sk_rcv_saddr, in inet_ehash_lookup_by_sk()
555 else if (sk->sk_family == AF_INET6) { in inet_ehash_lookup_by_sk()
557 &sk->sk_v6_daddr, in inet_ehash_lookup_by_sk()
558 &sk->sk_v6_rcv_saddr, in inet_ehash_lookup_by_sk()
573 bool inet_ehash_insert(struct sock *sk, struct sock *osk, bool *found_dup_sk) in inet_ehash_insert() argument
575 struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo; in inet_ehash_insert()
581 WARN_ON_ONCE(!sk_unhashed(sk)); in inet_ehash_insert()
583 sk->sk_hash = sk_ehashfn(sk); in inet_ehash_insert()
584 head = inet_ehash_bucket(hashinfo, sk->sk_hash); in inet_ehash_insert()
586 lock = inet_ehash_lockp(hashinfo, sk->sk_hash); in inet_ehash_insert()
590 WARN_ON_ONCE(sk->sk_hash != osk->sk_hash); in inet_ehash_insert()
593 *found_dup_sk = inet_ehash_lookup_by_sk(sk, list); in inet_ehash_insert()
599 __sk_nulls_add_node_rcu(sk, list); in inet_ehash_insert()
606 bool inet_ehash_nolisten(struct sock *sk, struct sock *osk, bool *found_dup_sk) in inet_ehash_nolisten() argument
608 bool ok = inet_ehash_insert(sk, osk, found_dup_sk); in inet_ehash_nolisten()
611 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1); in inet_ehash_nolisten()
613 this_cpu_inc(*sk->sk_prot->orphan_count); in inet_ehash_nolisten()
614 inet_sk_set_state(sk, TCP_CLOSE); in inet_ehash_nolisten()
615 sock_set_flag(sk, SOCK_DEAD); in inet_ehash_nolisten()
616 inet_csk_destroy_sock(sk); in inet_ehash_nolisten()
622 static int inet_reuseport_add_sock(struct sock *sk, in inet_reuseport_add_sock() argument
625 struct inet_bind_bucket *tb = inet_csk(sk)->icsk_bind_hash; in inet_reuseport_add_sock()
628 kuid_t uid = sock_i_uid(sk); in inet_reuseport_add_sock()
631 if (sk2 != sk && in inet_reuseport_add_sock()
632 sk2->sk_family == sk->sk_family && in inet_reuseport_add_sock()
633 ipv6_only_sock(sk2) == ipv6_only_sock(sk) && in inet_reuseport_add_sock()
634 sk2->sk_bound_dev_if == sk->sk_bound_dev_if && in inet_reuseport_add_sock()
637 inet_rcv_saddr_equal(sk, sk2, false)) in inet_reuseport_add_sock()
638 return reuseport_add_sock(sk, sk2, in inet_reuseport_add_sock()
639 inet_rcv_saddr_any(sk)); in inet_reuseport_add_sock()
642 return reuseport_alloc(sk, inet_rcv_saddr_any(sk)); in inet_reuseport_add_sock()
645 int __inet_hash(struct sock *sk, struct sock *osk) in __inet_hash() argument
647 struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo; in __inet_hash()
651 if (sk->sk_state != TCP_LISTEN) { in __inet_hash()
652 inet_ehash_nolisten(sk, osk, NULL); in __inet_hash()
655 WARN_ON(!sk_unhashed(sk)); in __inet_hash()
656 ilb = &hashinfo->listening_hash[inet_sk_listen_hashfn(sk)]; in __inet_hash()
659 if (sk->sk_reuseport) { in __inet_hash()
660 err = inet_reuseport_add_sock(sk, ilb); in __inet_hash()
664 if (IS_ENABLED(CONFIG_IPV6) && sk->sk_reuseport && in __inet_hash()
665 sk->sk_family == AF_INET6) in __inet_hash()
666 __sk_nulls_add_node_tail_rcu(sk, &ilb->nulls_head); in __inet_hash()
668 __sk_nulls_add_node_rcu(sk, &ilb->nulls_head); in __inet_hash()
669 inet_hash2(hashinfo, sk); in __inet_hash()
671 sock_set_flag(sk, SOCK_RCU_FREE); in __inet_hash()
672 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1); in __inet_hash()
680 int inet_hash(struct sock *sk) in inet_hash() argument
684 if (sk->sk_state != TCP_CLOSE) { in inet_hash()
686 err = __inet_hash(sk, NULL); in inet_hash()
694 void inet_unhash(struct sock *sk) in inet_unhash() argument
696 struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo; in inet_unhash()
700 if (sk_unhashed(sk)) in inet_unhash()
703 if (sk->sk_state == TCP_LISTEN) { in inet_unhash()
704 ilb = &hashinfo->listening_hash[inet_sk_listen_hashfn(sk)]; in inet_unhash()
707 lock = inet_ehash_lockp(hashinfo, sk->sk_hash); in inet_unhash()
710 if (sk_unhashed(sk)) in inet_unhash()
713 if (rcu_access_pointer(sk->sk_reuseport_cb)) in inet_unhash()
714 reuseport_detach_sock(sk); in inet_unhash()
716 inet_unhash2(hashinfo, sk); in inet_unhash()
719 __sk_nulls_del_node_init_rcu(sk); in inet_unhash()
720 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1); in inet_unhash()
739 struct sock *sk, u64 port_offset, in __inet_hash_connect() argument
746 int port = inet_sk(sk)->inet_num; in __inet_hash_connect()
747 struct net *net = sock_net(sk); in __inet_hash_connect()
757 tb = inet_csk(sk)->icsk_bind_hash; in __inet_hash_connect()
759 if (sk_head(&tb->owners) == sk && !sk->sk_bind_node.next) { in __inet_hash_connect()
760 inet_ehash_nolisten(sk, NULL, NULL); in __inet_hash_connect()
766 ret = check_established(death_row, sk, port, NULL); in __inet_hash_connect()
771 l3mdev = inet_sk_bound_l3mdev(sk); in __inet_hash_connect()
809 if (!check_established(death_row, sk, in __inet_hash_connect()
840 inet_bind_hash(sk, tb, port); in __inet_hash_connect()
841 if (sk_unhashed(sk)) { in __inet_hash_connect()
842 inet_sk(sk)->inet_sport = htons(port); in __inet_hash_connect()
843 inet_ehash_nolisten(sk, (struct sock *)tw, NULL); in __inet_hash_connect()
858 struct sock *sk) in inet_hash_connect() argument
862 if (!inet_sk(sk)->inet_num) in inet_hash_connect()
863 port_offset = inet_sk_port_offset(sk); in inet_hash_connect()
864 return __inet_hash_connect(death_row, sk, port_offset, in inet_hash_connect()