23 	struct sock_reuseport *reuse;  in reuseport_has_conns_set()  local
29 	reuse = rcu_dereference_protected(sk->sk_reuseport_cb,  in reuseport_has_conns_set()
31 	if (likely(reuse))  in reuseport_has_conns_set()
32 		reuse->has_conns = 1;  in reuseport_has_conns_set()
41 	struct sock_reuseport *reuse = kzalloc(size, GFP_ATOMIC);  in __reuseport_alloc()  local
43 	if (!reuse)  in __reuseport_alloc()
46 	reuse->max_socks = max_socks;  in __reuseport_alloc()
48 	RCU_INIT_POINTER(reuse->prog, NULL);  in __reuseport_alloc()
49 	return reuse;  in __reuseport_alloc()
54 	struct sock_reuseport *reuse;  in reuseport_alloc()  local
65 	reuse = rcu_dereference_protected(sk->sk_reuseport_cb,  in reuseport_alloc()
67 	if (reuse) {  in reuseport_alloc()
73 			reuse->bind_inany = bind_inany;  in reuseport_alloc()
77 	reuse = __reuseport_alloc(INIT_SOCKS);  in reuseport_alloc()
78 	if (!reuse) {  in reuseport_alloc()
85 		kfree(reuse);  in reuseport_alloc()
90 	reuse->reuseport_id = id;  in reuseport_alloc()
91 	reuse->socks[0] = sk;  in reuseport_alloc()
92 	reuse->num_socks = 1;  in reuseport_alloc()
93 	reuse->bind_inany = bind_inany;  in reuseport_alloc()
94 	rcu_assign_pointer(sk->sk_reuseport_cb, reuse);  in reuseport_alloc()
103 static struct sock_reuseport *reuseport_grow(struct sock_reuseport *reuse)  in reuseport_grow()  argument
108 	more_socks_size = reuse->max_socks * 2U;  in reuseport_grow()
116 	more_reuse->num_socks = reuse->num_socks;  in reuseport_grow()
117 	more_reuse->prog = reuse->prog;  in reuseport_grow()
118 	more_reuse->reuseport_id = reuse->reuseport_id;  in reuseport_grow()
119 	more_reuse->bind_inany = reuse->bind_inany;  in reuseport_grow()
120 	more_reuse->has_conns = reuse->has_conns;  in reuseport_grow()
122 	memcpy(more_reuse->socks, reuse->socks,  in reuseport_grow()
123 	       reuse->num_socks * sizeof(struct sock *));  in reuseport_grow()
124 	more_reuse->synq_overflow_ts = READ_ONCE(reuse->synq_overflow_ts);  in reuseport_grow()
126 	for (i = 0; i < reuse->num_socks; ++i)  in reuseport_grow()
127 		rcu_assign_pointer(reuse->socks[i]->sk_reuseport_cb,  in reuseport_grow()
134 	kfree_rcu(reuse, rcu);  in reuseport_grow()
140 	struct sock_reuseport *reuse;  in reuseport_free_rcu()  local
142 	reuse = container_of(head, struct sock_reuseport, rcu);  in reuseport_free_rcu()
143 	sk_reuseport_prog_free(rcu_dereference_protected(reuse->prog, 1));  in reuseport_free_rcu()
144 	ida_free(&reuseport_ida, reuse->reuseport_id);  in reuseport_free_rcu()
145 	kfree(reuse);  in reuseport_free_rcu()
158 	struct sock_reuseport *old_reuse, *reuse;  in reuseport_add_sock()  local
168 	reuse = rcu_dereference_protected(sk2->sk_reuseport_cb,  in reuseport_add_sock()
177 	if (reuse->num_socks == reuse->max_socks) {  in reuseport_add_sock()
178 		reuse = reuseport_grow(reuse);  in reuseport_add_sock()
179 		if (!reuse) {  in reuseport_add_sock()
185 	reuse->socks[reuse->num_socks] = sk;  in reuseport_add_sock()
188 	reuse->num_socks++;  in reuseport_add_sock()
189 	rcu_assign_pointer(sk->sk_reuseport_cb, reuse);  in reuseport_add_sock()
201 	struct sock_reuseport *reuse;  in reuseport_detach_sock()  local
205 	reuse = rcu_dereference_protected(sk->sk_reuseport_cb,  in reuseport_detach_sock()
220 	for (i = 0; i < reuse->num_socks; i++) {  in reuseport_detach_sock()
221 		if (reuse->socks[i] == sk) {  in reuseport_detach_sock()
222 			reuse->socks[i] = reuse->socks[reuse->num_socks - 1];  in reuseport_detach_sock()
223 			reuse->num_socks--;  in reuseport_detach_sock()
224 			if (reuse->num_socks == 0)  in reuseport_detach_sock()
225 				call_rcu(&reuse->rcu, reuseport_free_rcu);  in reuseport_detach_sock()
233 static struct sock *run_bpf_filter(struct sock_reuseport *reuse, u16 socks,  in run_bpf_filter()  argument
260 	return reuse->socks[index];  in run_bpf_filter()
278 	struct sock_reuseport *reuse;  in reuseport_select_sock()  local
284 	reuse = rcu_dereference(sk->sk_reuseport_cb);  in reuseport_select_sock()
287 	if (!reuse)  in reuseport_select_sock()
290 	prog = rcu_dereference(reuse->prog);  in reuseport_select_sock()
291 	socks = READ_ONCE(reuse->num_socks);  in reuseport_select_sock()
300 			sk2 = bpf_run_sk_reuseport(reuse, sk, prog, skb, hash);  in reuseport_select_sock()
302 			sk2 = run_bpf_filter(reuse, socks, prog, skb, hdr_len);  in reuseport_select_sock()
310 			while (reuse->socks[i]->sk_state == TCP_ESTABLISHED) {  in reuseport_select_sock()
317 			sk2 = reuse->socks[i];  in reuseport_select_sock()
329 	struct sock_reuseport *reuse;  in reuseport_attach_prog()  local
343 	reuse = rcu_dereference_protected(sk->sk_reuseport_cb,  in reuseport_attach_prog()
345 	old_prog = rcu_dereference_protected(reuse->prog,  in reuseport_attach_prog()
347 	rcu_assign_pointer(reuse->prog, prog);  in reuseport_attach_prog()
357 	struct sock_reuseport *reuse;  in reuseport_detach_prog()  local
365 	reuse = rcu_dereference_protected(sk->sk_reuseport_cb,  in reuseport_detach_prog()
367 	old_prog = rcu_replace_pointer(reuse->prog, old_prog,  in reuseport_detach_prog()