1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * INET An implementation of the TCP/IP protocol suite for the LINUX
4 * operating system. INET is implemented using the BSD Socket
5 * interface as the means of communication with the user level.
6 *
7 * Generic socket support routines. Memory allocators, socket lock/release
8 * handler for protocols to use and generic option handler.
9 *
10 * Authors: Ross Biro
11 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
12 * Florian La Roche, <flla@stud.uni-sb.de>
13 * Alan Cox, <A.Cox@swansea.ac.uk>
14 *
15 * Fixes:
16 * Alan Cox : Numerous verify_area() problems
17 * Alan Cox : Connecting on a connecting socket
18 * now returns an error for tcp.
19 * Alan Cox : sock->protocol is set correctly.
20 * and is not sometimes left as 0.
21 * Alan Cox : connect handles icmp errors on a
22 * connect properly. Unfortunately there
23 * is a restart syscall nasty there. I
24 * can't match BSD without hacking the C
25 * library. Ideas urgently sought!
26 * Alan Cox : Disallow bind() to addresses that are
27 * not ours - especially broadcast ones!!
28 * Alan Cox : Socket 1024 _IS_ ok for users. (fencepost)
29 * Alan Cox : sock_wfree/sock_rfree don't destroy sockets,
30 * instead they leave that for the DESTROY timer.
31 * Alan Cox : Clean up error flag in accept
32 * Alan Cox : TCP ack handling is buggy, the DESTROY timer
33 * was buggy. Put a remove_sock() in the handler
34 * for memory when we hit 0. Also altered the timer
35 * code. The ACK stuff can wait and needs major
36 * TCP layer surgery.
37 * Alan Cox : Fixed TCP ack bug, removed remove sock
38 * and fixed timer/inet_bh race.
39 * Alan Cox : Added zapped flag for TCP
40 * Alan Cox : Move kfree_skb into skbuff.c and tidied up surplus code
41 * Alan Cox : for new sk_buff allocations wmalloc/rmalloc now call alloc_skb
42 * Alan Cox : kfree_s calls now are kfree_skbmem so we can track skb resources
43 * Alan Cox : Supports socket option broadcast now as does udp. Packet and raw need fixing.
44 * Alan Cox : Added RCVBUF,SNDBUF size setting. It suddenly occurred to me how easy it was so...
45 * Rick Sladkey : Relaxed UDP rules for matching packets.
46 * C.E.Hawkins : IFF_PROMISC/SIOCGHWADDR support
47 * Pauline Middelink : identd support
48 * Alan Cox : Fixed connect() taking signals I think.
49 * Alan Cox : SO_LINGER supported
50 * Alan Cox : Error reporting fixes
51 * Anonymous : inet_create tidied up (sk->reuse setting)
52 * Alan Cox : inet sockets don't set sk->type!
53 * Alan Cox : Split socket option code
54 * Alan Cox : Callbacks
55 * Alan Cox : Nagle flag for Charles & Johannes stuff
56 * Alex : Removed restriction on inet fioctl
57 * Alan Cox : Splitting INET from NET core
58 * Alan Cox : Fixed bogus SO_TYPE handling in getsockopt()
59 * Adam Caldwell : Missing return in SO_DONTROUTE/SO_DEBUG code
60 * Alan Cox : Split IP from generic code
61 * Alan Cox : New kfree_skbmem()
62 * Alan Cox : Make SO_DEBUG superuser only.
63 * Alan Cox : Allow anyone to clear SO_DEBUG
64 * (compatibility fix)
65 * Alan Cox : Added optimistic memory grabbing for AF_UNIX throughput.
66 * Alan Cox : Allocator for a socket is settable.
67 * Alan Cox : SO_ERROR includes soft errors.
68 * Alan Cox : Allow NULL arguments on some SO_ opts
69 * Alan Cox : Generic socket allocation to make hooks
70 * easier (suggested by Craig Metz).
71 * Michael Pall : SO_ERROR returns positive errno again
72 * Steve Whitehouse: Added default destructor to free
73 * protocol private data.
74 * Steve Whitehouse: Added various other default routines
75 * common to several socket families.
76 * Chris Evans : Call suser() check last on F_SETOWN
77 * Jay Schulist : Added SO_ATTACH_FILTER and SO_DETACH_FILTER.
78 * Andi Kleen : Add sock_kmalloc()/sock_kfree_s()
79 * Andi Kleen : Fix write_space callback
80 * Chris Evans : Security fixes - signedness again
81 * Arnaldo C. Melo : cleanups, use skb_queue_purge
82 *
83 * To Fix:
84 */
85
86 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
87
88 #include <asm/unaligned.h>
89 #include <linux/capability.h>
90 #include <linux/errno.h>
91 #include <linux/errqueue.h>
92 #include <linux/types.h>
93 #include <linux/socket.h>
94 #include <linux/in.h>
95 #include <linux/kernel.h>
96 #include <linux/module.h>
97 #include <linux/proc_fs.h>
98 #include <linux/seq_file.h>
99 #include <linux/sched.h>
100 #include <linux/sched/mm.h>
101 #include <linux/timer.h>
102 #include <linux/string.h>
103 #include <linux/sockios.h>
104 #include <linux/net.h>
105 #include <linux/mm.h>
106 #include <linux/slab.h>
107 #include <linux/interrupt.h>
108 #include <linux/poll.h>
109 #include <linux/tcp.h>
110 #include <linux/init.h>
111 #include <linux/highmem.h>
112 #include <linux/user_namespace.h>
113 #include <linux/static_key.h>
114 #include <linux/memcontrol.h>
115 #include <linux/prefetch.h>
116 #include <linux/compat.h>
117
118 #include <linux/uaccess.h>
119
120 #include <linux/netdevice.h>
121 #include <net/protocol.h>
122 #include <linux/skbuff.h>
123 #include <net/net_namespace.h>
124 #include <net/request_sock.h>
125 #include <net/sock.h>
126 #include <linux/net_tstamp.h>
127 #include <net/xfrm.h>
128 #include <linux/ipsec.h>
129 #include <net/cls_cgroup.h>
130 #include <net/netprio_cgroup.h>
131 #include <linux/sock_diag.h>
132
133 #include <linux/filter.h>
134 #include <net/sock_reuseport.h>
135 #include <net/bpf_sk_storage.h>
136
137 #include <trace/events/sock.h>
138
139 #include <net/tcp.h>
140 #include <net/busy_poll.h>
141
142 static DEFINE_MUTEX(proto_list_mutex);
143 static LIST_HEAD(proto_list);
144
145 static void sock_inuse_add(struct net *net, int val);
146
147 /**
148 * sk_ns_capable - General socket capability test
149 * @sk: Socket to use a capability on or through
150 * @user_ns: The user namespace of the capability to use
151 * @cap: The capability to use
152 *
153 * Test to see if the opener of the socket had when the socket was
154 * created and the current process has the capability @cap in the user
155 * namespace @user_ns.
156 */
sk_ns_capable(const struct sock * sk,struct user_namespace * user_ns,int cap)157 bool sk_ns_capable(const struct sock *sk,
158 struct user_namespace *user_ns, int cap)
159 {
160 return file_ns_capable(sk->sk_socket->file, user_ns, cap) &&
161 ns_capable(user_ns, cap);
162 }
163 EXPORT_SYMBOL(sk_ns_capable);
164
165 /**
166 * sk_capable - Socket global capability test
167 * @sk: Socket to use a capability on or through
168 * @cap: The global capability to use
169 *
170 * Test to see if the opener of the socket had when the socket was
171 * created and the current process has the capability @cap in all user
172 * namespaces.
173 */
sk_capable(const struct sock * sk,int cap)174 bool sk_capable(const struct sock *sk, int cap)
175 {
176 return sk_ns_capable(sk, &init_user_ns, cap);
177 }
178 EXPORT_SYMBOL(sk_capable);
179
180 /**
181 * sk_net_capable - Network namespace socket capability test
182 * @sk: Socket to use a capability on or through
183 * @cap: The capability to use
184 *
185 * Test to see if the opener of the socket had when the socket was created
186 * and the current process has the capability @cap over the network namespace
187 * the socket is a member of.
188 */
sk_net_capable(const struct sock * sk,int cap)189 bool sk_net_capable(const struct sock *sk, int cap)
190 {
191 return sk_ns_capable(sk, sock_net(sk)->user_ns, cap);
192 }
193 EXPORT_SYMBOL(sk_net_capable);
194
195 /*
196 * Each address family might have different locking rules, so we have
197 * one slock key per address family and separate keys for internal and
198 * userspace sockets.
199 */
200 static struct lock_class_key af_family_keys[AF_MAX];
201 static struct lock_class_key af_family_kern_keys[AF_MAX];
202 static struct lock_class_key af_family_slock_keys[AF_MAX];
203 static struct lock_class_key af_family_kern_slock_keys[AF_MAX];
204
205 /*
206 * Make lock validator output more readable. (we pre-construct these
207 * strings build-time, so that runtime initialization of socket
208 * locks is fast):
209 */
210
211 #define _sock_locks(x) \
212 x "AF_UNSPEC", x "AF_UNIX" , x "AF_INET" , \
213 x "AF_AX25" , x "AF_IPX" , x "AF_APPLETALK", \
214 x "AF_NETROM", x "AF_BRIDGE" , x "AF_ATMPVC" , \
215 x "AF_X25" , x "AF_INET6" , x "AF_ROSE" , \
216 x "AF_DECnet", x "AF_NETBEUI" , x "AF_SECURITY" , \
217 x "AF_KEY" , x "AF_NETLINK" , x "AF_PACKET" , \
218 x "AF_ASH" , x "AF_ECONET" , x "AF_ATMSVC" , \
219 x "AF_RDS" , x "AF_SNA" , x "AF_IRDA" , \
220 x "AF_PPPOX" , x "AF_WANPIPE" , x "AF_LLC" , \
221 x "27" , x "28" , x "AF_CAN" , \
222 x "AF_TIPC" , x "AF_BLUETOOTH", x "IUCV" , \
223 x "AF_RXRPC" , x "AF_ISDN" , x "AF_PHONET" , \
224 x "AF_IEEE802154", x "AF_CAIF" , x "AF_ALG" , \
225 x "AF_NFC" , x "AF_VSOCK" , x "AF_KCM" , \
226 x "AF_QIPCRTR", x "AF_SMC" , x "AF_XDP" , \
227 x "AF_MAX"
228
229 static const char *const af_family_key_strings[AF_MAX+1] = {
230 _sock_locks("sk_lock-")
231 };
232 static const char *const af_family_slock_key_strings[AF_MAX+1] = {
233 _sock_locks("slock-")
234 };
235 static const char *const af_family_clock_key_strings[AF_MAX+1] = {
236 _sock_locks("clock-")
237 };
238
239 static const char *const af_family_kern_key_strings[AF_MAX+1] = {
240 _sock_locks("k-sk_lock-")
241 };
242 static const char *const af_family_kern_slock_key_strings[AF_MAX+1] = {
243 _sock_locks("k-slock-")
244 };
245 static const char *const af_family_kern_clock_key_strings[AF_MAX+1] = {
246 _sock_locks("k-clock-")
247 };
248 static const char *const af_family_rlock_key_strings[AF_MAX+1] = {
249 _sock_locks("rlock-")
250 };
251 static const char *const af_family_wlock_key_strings[AF_MAX+1] = {
252 _sock_locks("wlock-")
253 };
254 static const char *const af_family_elock_key_strings[AF_MAX+1] = {
255 _sock_locks("elock-")
256 };
257
258 /*
259 * sk_callback_lock and sk queues locking rules are per-address-family,
260 * so split the lock classes by using a per-AF key:
261 */
262 static struct lock_class_key af_callback_keys[AF_MAX];
263 static struct lock_class_key af_rlock_keys[AF_MAX];
264 static struct lock_class_key af_wlock_keys[AF_MAX];
265 static struct lock_class_key af_elock_keys[AF_MAX];
266 static struct lock_class_key af_kern_callback_keys[AF_MAX];
267
268 /* Run time adjustable parameters. */
269 __u32 sysctl_wmem_max __read_mostly = SK_WMEM_MAX;
270 EXPORT_SYMBOL(sysctl_wmem_max);
271 __u32 sysctl_rmem_max __read_mostly = SK_RMEM_MAX;
272 EXPORT_SYMBOL(sysctl_rmem_max);
273 __u32 sysctl_wmem_default __read_mostly = SK_WMEM_MAX;
274 __u32 sysctl_rmem_default __read_mostly = SK_RMEM_MAX;
275
276 /* Maximal space eaten by iovec or ancillary data plus some space */
277 int sysctl_optmem_max __read_mostly = sizeof(unsigned long)*(2*UIO_MAXIOV+512);
278 EXPORT_SYMBOL(sysctl_optmem_max);
279
280 int sysctl_tstamp_allow_data __read_mostly = 1;
281
282 DEFINE_STATIC_KEY_FALSE(memalloc_socks_key);
283 EXPORT_SYMBOL_GPL(memalloc_socks_key);
284
285 /**
286 * sk_set_memalloc - sets %SOCK_MEMALLOC
287 * @sk: socket to set it on
288 *
289 * Set %SOCK_MEMALLOC on a socket for access to emergency reserves.
290 * It's the responsibility of the admin to adjust min_free_kbytes
291 * to meet the requirements
292 */
sk_set_memalloc(struct sock * sk)293 void sk_set_memalloc(struct sock *sk)
294 {
295 sock_set_flag(sk, SOCK_MEMALLOC);
296 sk->sk_allocation |= __GFP_MEMALLOC;
297 static_branch_inc(&memalloc_socks_key);
298 }
299 EXPORT_SYMBOL_GPL(sk_set_memalloc);
300
sk_clear_memalloc(struct sock * sk)301 void sk_clear_memalloc(struct sock *sk)
302 {
303 sock_reset_flag(sk, SOCK_MEMALLOC);
304 sk->sk_allocation &= ~__GFP_MEMALLOC;
305 static_branch_dec(&memalloc_socks_key);
306
307 /*
308 * SOCK_MEMALLOC is allowed to ignore rmem limits to ensure forward
309 * progress of swapping. SOCK_MEMALLOC may be cleared while
310 * it has rmem allocations due to the last swapfile being deactivated
311 * but there is a risk that the socket is unusable due to exceeding
312 * the rmem limits. Reclaim the reserves and obey rmem limits again.
313 */
314 sk_mem_reclaim(sk);
315 }
316 EXPORT_SYMBOL_GPL(sk_clear_memalloc);
317
__sk_backlog_rcv(struct sock * sk,struct sk_buff * skb)318 int __sk_backlog_rcv(struct sock *sk, struct sk_buff *skb)
319 {
320 int ret;
321 unsigned int noreclaim_flag;
322
323 /* these should have been dropped before queueing */
324 BUG_ON(!sock_flag(sk, SOCK_MEMALLOC));
325
326 noreclaim_flag = memalloc_noreclaim_save();
327 ret = sk->sk_backlog_rcv(sk, skb);
328 memalloc_noreclaim_restore(noreclaim_flag);
329
330 return ret;
331 }
332 EXPORT_SYMBOL(__sk_backlog_rcv);
333
sock_get_timeout(long timeo,void * optval,bool old_timeval)334 static int sock_get_timeout(long timeo, void *optval, bool old_timeval)
335 {
336 struct __kernel_sock_timeval tv;
337
338 if (timeo == MAX_SCHEDULE_TIMEOUT) {
339 tv.tv_sec = 0;
340 tv.tv_usec = 0;
341 } else {
342 tv.tv_sec = timeo / HZ;
343 tv.tv_usec = ((timeo % HZ) * USEC_PER_SEC) / HZ;
344 }
345
346 if (old_timeval && in_compat_syscall() && !COMPAT_USE_64BIT_TIME) {
347 struct old_timeval32 tv32 = { tv.tv_sec, tv.tv_usec };
348 *(struct old_timeval32 *)optval = tv32;
349 return sizeof(tv32);
350 }
351
352 if (old_timeval) {
353 struct __kernel_old_timeval old_tv;
354 old_tv.tv_sec = tv.tv_sec;
355 old_tv.tv_usec = tv.tv_usec;
356 *(struct __kernel_old_timeval *)optval = old_tv;
357 return sizeof(old_tv);
358 }
359
360 *(struct __kernel_sock_timeval *)optval = tv;
361 return sizeof(tv);
362 }
363
sock_set_timeout(long * timeo_p,sockptr_t optval,int optlen,bool old_timeval)364 static int sock_set_timeout(long *timeo_p, sockptr_t optval, int optlen,
365 bool old_timeval)
366 {
367 struct __kernel_sock_timeval tv;
368
369 if (old_timeval && in_compat_syscall() && !COMPAT_USE_64BIT_TIME) {
370 struct old_timeval32 tv32;
371
372 if (optlen < sizeof(tv32))
373 return -EINVAL;
374
375 if (copy_from_sockptr(&tv32, optval, sizeof(tv32)))
376 return -EFAULT;
377 tv.tv_sec = tv32.tv_sec;
378 tv.tv_usec = tv32.tv_usec;
379 } else if (old_timeval) {
380 struct __kernel_old_timeval old_tv;
381
382 if (optlen < sizeof(old_tv))
383 return -EINVAL;
384 if (copy_from_sockptr(&old_tv, optval, sizeof(old_tv)))
385 return -EFAULT;
386 tv.tv_sec = old_tv.tv_sec;
387 tv.tv_usec = old_tv.tv_usec;
388 } else {
389 if (optlen < sizeof(tv))
390 return -EINVAL;
391 if (copy_from_sockptr(&tv, optval, sizeof(tv)))
392 return -EFAULT;
393 }
394 if (tv.tv_usec < 0 || tv.tv_usec >= USEC_PER_SEC)
395 return -EDOM;
396
397 if (tv.tv_sec < 0) {
398 static int warned __read_mostly;
399
400 *timeo_p = 0;
401 if (warned < 10 && net_ratelimit()) {
402 warned++;
403 pr_info("%s: `%s' (pid %d) tries to set negative timeout\n",
404 __func__, current->comm, task_pid_nr(current));
405 }
406 return 0;
407 }
408 *timeo_p = MAX_SCHEDULE_TIMEOUT;
409 if (tv.tv_sec == 0 && tv.tv_usec == 0)
410 return 0;
411 if (tv.tv_sec < (MAX_SCHEDULE_TIMEOUT / HZ - 1))
412 *timeo_p = tv.tv_sec * HZ + DIV_ROUND_UP((unsigned long)tv.tv_usec, USEC_PER_SEC / HZ);
413 return 0;
414 }
415
sock_needs_netstamp(const struct sock * sk)416 static bool sock_needs_netstamp(const struct sock *sk)
417 {
418 switch (sk->sk_family) {
419 case AF_UNSPEC:
420 case AF_UNIX:
421 return false;
422 default:
423 return true;
424 }
425 }
426
sock_disable_timestamp(struct sock * sk,unsigned long flags)427 static void sock_disable_timestamp(struct sock *sk, unsigned long flags)
428 {
429 if (sk->sk_flags & flags) {
430 sk->sk_flags &= ~flags;
431 if (sock_needs_netstamp(sk) &&
432 !(sk->sk_flags & SK_FLAGS_TIMESTAMP))
433 net_disable_timestamp();
434 }
435 }
436
437
__sock_queue_rcv_skb(struct sock * sk,struct sk_buff * skb)438 int __sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
439 {
440 unsigned long flags;
441 struct sk_buff_head *list = &sk->sk_receive_queue;
442
443 if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf) {
444 atomic_inc(&sk->sk_drops);
445 trace_sock_rcvqueue_full(sk, skb);
446 return -ENOMEM;
447 }
448
449 if (!sk_rmem_schedule(sk, skb, skb->truesize)) {
450 atomic_inc(&sk->sk_drops);
451 return -ENOBUFS;
452 }
453
454 skb->dev = NULL;
455 skb_set_owner_r(skb, sk);
456
457 /* we escape from rcu protected region, make sure we dont leak
458 * a norefcounted dst
459 */
460 skb_dst_force(skb);
461
462 spin_lock_irqsave(&list->lock, flags);
463 sock_skb_set_dropcount(sk, skb);
464 __skb_queue_tail(list, skb);
465 spin_unlock_irqrestore(&list->lock, flags);
466
467 if (!sock_flag(sk, SOCK_DEAD))
468 sk->sk_data_ready(sk);
469 return 0;
470 }
471 EXPORT_SYMBOL(__sock_queue_rcv_skb);
472
sock_queue_rcv_skb(struct sock * sk,struct sk_buff * skb)473 int sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
474 {
475 int err;
476
477 err = sk_filter(sk, skb);
478 if (err)
479 return err;
480
481 return __sock_queue_rcv_skb(sk, skb);
482 }
483 EXPORT_SYMBOL(sock_queue_rcv_skb);
484
__sk_receive_skb(struct sock * sk,struct sk_buff * skb,const int nested,unsigned int trim_cap,bool refcounted)485 int __sk_receive_skb(struct sock *sk, struct sk_buff *skb,
486 const int nested, unsigned int trim_cap, bool refcounted)
487 {
488 int rc = NET_RX_SUCCESS;
489
490 if (sk_filter_trim_cap(sk, skb, trim_cap))
491 goto discard_and_relse;
492
493 skb->dev = NULL;
494
495 if (sk_rcvqueues_full(sk, sk->sk_rcvbuf)) {
496 atomic_inc(&sk->sk_drops);
497 goto discard_and_relse;
498 }
499 if (nested)
500 bh_lock_sock_nested(sk);
501 else
502 bh_lock_sock(sk);
503 if (!sock_owned_by_user(sk)) {
504 /*
505 * trylock + unlock semantics:
506 */
507 mutex_acquire(&sk->sk_lock.dep_map, 0, 1, _RET_IP_);
508
509 rc = sk_backlog_rcv(sk, skb);
510
511 mutex_release(&sk->sk_lock.dep_map, _RET_IP_);
512 } else if (sk_add_backlog(sk, skb, READ_ONCE(sk->sk_rcvbuf))) {
513 bh_unlock_sock(sk);
514 atomic_inc(&sk->sk_drops);
515 goto discard_and_relse;
516 }
517
518 bh_unlock_sock(sk);
519 out:
520 if (refcounted)
521 sock_put(sk);
522 return rc;
523 discard_and_relse:
524 kfree_skb(skb);
525 goto out;
526 }
527 EXPORT_SYMBOL(__sk_receive_skb);
528
__sk_dst_check(struct sock * sk,u32 cookie)529 struct dst_entry *__sk_dst_check(struct sock *sk, u32 cookie)
530 {
531 struct dst_entry *dst = __sk_dst_get(sk);
532
533 if (dst && dst->obsolete && dst->ops->check(dst, cookie) == NULL) {
534 sk_tx_queue_clear(sk);
535 sk->sk_dst_pending_confirm = 0;
536 RCU_INIT_POINTER(sk->sk_dst_cache, NULL);
537 dst_release(dst);
538 return NULL;
539 }
540
541 return dst;
542 }
543 EXPORT_SYMBOL(__sk_dst_check);
544
sk_dst_check(struct sock * sk,u32 cookie)545 struct dst_entry *sk_dst_check(struct sock *sk, u32 cookie)
546 {
547 struct dst_entry *dst = sk_dst_get(sk);
548
549 if (dst && dst->obsolete && dst->ops->check(dst, cookie) == NULL) {
550 sk_dst_reset(sk);
551 dst_release(dst);
552 return NULL;
553 }
554
555 return dst;
556 }
557 EXPORT_SYMBOL(sk_dst_check);
558
sock_bindtoindex_locked(struct sock * sk,int ifindex)559 static int sock_bindtoindex_locked(struct sock *sk, int ifindex)
560 {
561 int ret = -ENOPROTOOPT;
562 #ifdef CONFIG_NETDEVICES
563 struct net *net = sock_net(sk);
564
565 /* Sorry... */
566 ret = -EPERM;
567 if (sk->sk_bound_dev_if && !ns_capable(net->user_ns, CAP_NET_RAW))
568 goto out;
569
570 ret = -EINVAL;
571 if (ifindex < 0)
572 goto out;
573
574 sk->sk_bound_dev_if = ifindex;
575 if (sk->sk_prot->rehash)
576 sk->sk_prot->rehash(sk);
577 sk_dst_reset(sk);
578
579 ret = 0;
580
581 out:
582 #endif
583
584 return ret;
585 }
586
sock_bindtoindex(struct sock * sk,int ifindex,bool lock_sk)587 int sock_bindtoindex(struct sock *sk, int ifindex, bool lock_sk)
588 {
589 int ret;
590
591 if (lock_sk)
592 lock_sock(sk);
593 ret = sock_bindtoindex_locked(sk, ifindex);
594 if (lock_sk)
595 release_sock(sk);
596
597 return ret;
598 }
599 EXPORT_SYMBOL(sock_bindtoindex);
600
sock_setbindtodevice(struct sock * sk,sockptr_t optval,int optlen)601 static int sock_setbindtodevice(struct sock *sk, sockptr_t optval, int optlen)
602 {
603 int ret = -ENOPROTOOPT;
604 #ifdef CONFIG_NETDEVICES
605 struct net *net = sock_net(sk);
606 char devname[IFNAMSIZ];
607 int index;
608
609 ret = -EINVAL;
610 if (optlen < 0)
611 goto out;
612
613 /* Bind this socket to a particular device like "eth0",
614 * as specified in the passed interface name. If the
615 * name is "" or the option length is zero the socket
616 * is not bound.
617 */
618 if (optlen > IFNAMSIZ - 1)
619 optlen = IFNAMSIZ - 1;
620 memset(devname, 0, sizeof(devname));
621
622 ret = -EFAULT;
623 if (copy_from_sockptr(devname, optval, optlen))
624 goto out;
625
626 index = 0;
627 if (devname[0] != '\0') {
628 struct net_device *dev;
629
630 rcu_read_lock();
631 dev = dev_get_by_name_rcu(net, devname);
632 if (dev)
633 index = dev->ifindex;
634 rcu_read_unlock();
635 ret = -ENODEV;
636 if (!dev)
637 goto out;
638 }
639
640 return sock_bindtoindex(sk, index, true);
641 out:
642 #endif
643
644 return ret;
645 }
646
sock_getbindtodevice(struct sock * sk,char __user * optval,int __user * optlen,int len)647 static int sock_getbindtodevice(struct sock *sk, char __user *optval,
648 int __user *optlen, int len)
649 {
650 int ret = -ENOPROTOOPT;
651 #ifdef CONFIG_NETDEVICES
652 struct net *net = sock_net(sk);
653 char devname[IFNAMSIZ];
654
655 if (sk->sk_bound_dev_if == 0) {
656 len = 0;
657 goto zero;
658 }
659
660 ret = -EINVAL;
661 if (len < IFNAMSIZ)
662 goto out;
663
664 ret = netdev_get_name(net, devname, sk->sk_bound_dev_if);
665 if (ret)
666 goto out;
667
668 len = strlen(devname) + 1;
669
670 ret = -EFAULT;
671 if (copy_to_user(optval, devname, len))
672 goto out;
673
674 zero:
675 ret = -EFAULT;
676 if (put_user(len, optlen))
677 goto out;
678
679 ret = 0;
680
681 out:
682 #endif
683
684 return ret;
685 }
686
sk_mc_loop(struct sock * sk)687 bool sk_mc_loop(struct sock *sk)
688 {
689 if (dev_recursion_level())
690 return false;
691 if (!sk)
692 return true;
693 switch (sk->sk_family) {
694 case AF_INET:
695 return inet_sk(sk)->mc_loop;
696 #if IS_ENABLED(CONFIG_IPV6)
697 case AF_INET6:
698 return inet6_sk(sk)->mc_loop;
699 #endif
700 }
701 WARN_ON_ONCE(1);
702 return true;
703 }
704 EXPORT_SYMBOL(sk_mc_loop);
705
sock_set_reuseaddr(struct sock * sk)706 void sock_set_reuseaddr(struct sock *sk)
707 {
708 lock_sock(sk);
709 sk->sk_reuse = SK_CAN_REUSE;
710 release_sock(sk);
711 }
712 EXPORT_SYMBOL(sock_set_reuseaddr);
713
sock_set_reuseport(struct sock * sk)714 void sock_set_reuseport(struct sock *sk)
715 {
716 lock_sock(sk);
717 sk->sk_reuseport = true;
718 release_sock(sk);
719 }
720 EXPORT_SYMBOL(sock_set_reuseport);
721
sock_no_linger(struct sock * sk)722 void sock_no_linger(struct sock *sk)
723 {
724 lock_sock(sk);
725 sk->sk_lingertime = 0;
726 sock_set_flag(sk, SOCK_LINGER);
727 release_sock(sk);
728 }
729 EXPORT_SYMBOL(sock_no_linger);
730
sock_set_priority(struct sock * sk,u32 priority)731 void sock_set_priority(struct sock *sk, u32 priority)
732 {
733 lock_sock(sk);
734 sk->sk_priority = priority;
735 release_sock(sk);
736 }
737 EXPORT_SYMBOL(sock_set_priority);
738
sock_set_sndtimeo(struct sock * sk,s64 secs)739 void sock_set_sndtimeo(struct sock *sk, s64 secs)
740 {
741 lock_sock(sk);
742 if (secs && secs < MAX_SCHEDULE_TIMEOUT / HZ - 1)
743 sk->sk_sndtimeo = secs * HZ;
744 else
745 sk->sk_sndtimeo = MAX_SCHEDULE_TIMEOUT;
746 release_sock(sk);
747 }
748 EXPORT_SYMBOL(sock_set_sndtimeo);
749
__sock_set_timestamps(struct sock * sk,bool val,bool new,bool ns)750 static void __sock_set_timestamps(struct sock *sk, bool val, bool new, bool ns)
751 {
752 if (val) {
753 sock_valbool_flag(sk, SOCK_TSTAMP_NEW, new);
754 sock_valbool_flag(sk, SOCK_RCVTSTAMPNS, ns);
755 sock_set_flag(sk, SOCK_RCVTSTAMP);
756 sock_enable_timestamp(sk, SOCK_TIMESTAMP);
757 } else {
758 sock_reset_flag(sk, SOCK_RCVTSTAMP);
759 sock_reset_flag(sk, SOCK_RCVTSTAMPNS);
760 }
761 }
762
sock_enable_timestamps(struct sock * sk)763 void sock_enable_timestamps(struct sock *sk)
764 {
765 lock_sock(sk);
766 __sock_set_timestamps(sk, true, false, true);
767 release_sock(sk);
768 }
769 EXPORT_SYMBOL(sock_enable_timestamps);
770
sock_set_keepalive(struct sock * sk)771 void sock_set_keepalive(struct sock *sk)
772 {
773 lock_sock(sk);
774 if (sk->sk_prot->keepalive)
775 sk->sk_prot->keepalive(sk, true);
776 sock_valbool_flag(sk, SOCK_KEEPOPEN, true);
777 release_sock(sk);
778 }
779 EXPORT_SYMBOL(sock_set_keepalive);
780
__sock_set_rcvbuf(struct sock * sk,int val)781 static void __sock_set_rcvbuf(struct sock *sk, int val)
782 {
783 /* Ensure val * 2 fits into an int, to prevent max_t() from treating it
784 * as a negative value.
785 */
786 val = min_t(int, val, INT_MAX / 2);
787 sk->sk_userlocks |= SOCK_RCVBUF_LOCK;
788
789 /* We double it on the way in to account for "struct sk_buff" etc.
790 * overhead. Applications assume that the SO_RCVBUF setting they make
791 * will allow that much actual data to be received on that socket.
792 *
793 * Applications are unaware that "struct sk_buff" and other overheads
794 * allocate from the receive buffer during socket buffer allocation.
795 *
796 * And after considering the possible alternatives, returning the value
797 * we actually used in getsockopt is the most desirable behavior.
798 */
799 WRITE_ONCE(sk->sk_rcvbuf, max_t(int, val * 2, SOCK_MIN_RCVBUF));
800 }
801
sock_set_rcvbuf(struct sock * sk,int val)802 void sock_set_rcvbuf(struct sock *sk, int val)
803 {
804 lock_sock(sk);
805 __sock_set_rcvbuf(sk, val);
806 release_sock(sk);
807 }
808 EXPORT_SYMBOL(sock_set_rcvbuf);
809
__sock_set_mark(struct sock * sk,u32 val)810 static void __sock_set_mark(struct sock *sk, u32 val)
811 {
812 if (val != sk->sk_mark) {
813 sk->sk_mark = val;
814 sk_dst_reset(sk);
815 }
816 }
817
sock_set_mark(struct sock * sk,u32 val)818 void sock_set_mark(struct sock *sk, u32 val)
819 {
820 lock_sock(sk);
821 __sock_set_mark(sk, val);
822 release_sock(sk);
823 }
824 EXPORT_SYMBOL(sock_set_mark);
825
826 /*
827 * This is meant for all protocols to use and covers goings on
828 * at the socket level. Everything here is generic.
829 */
830
sock_setsockopt(struct socket * sock,int level,int optname,sockptr_t optval,unsigned int optlen)831 int sock_setsockopt(struct socket *sock, int level, int optname,
832 sockptr_t optval, unsigned int optlen)
833 {
834 struct sock_txtime sk_txtime;
835 struct sock *sk = sock->sk;
836 int val;
837 int valbool;
838 struct linger ling;
839 int ret = 0;
840
841 /*
842 * Options without arguments
843 */
844
845 if (optname == SO_BINDTODEVICE)
846 return sock_setbindtodevice(sk, optval, optlen);
847
848 if (optlen < sizeof(int))
849 return -EINVAL;
850
851 if (copy_from_sockptr(&val, optval, sizeof(val)))
852 return -EFAULT;
853
854 valbool = val ? 1 : 0;
855
856 lock_sock(sk);
857
858 switch (optname) {
859 case SO_DEBUG:
860 if (val && !capable(CAP_NET_ADMIN))
861 ret = -EACCES;
862 else
863 sock_valbool_flag(sk, SOCK_DBG, valbool);
864 break;
865 case SO_REUSEADDR:
866 sk->sk_reuse = (valbool ? SK_CAN_REUSE : SK_NO_REUSE);
867 break;
868 case SO_REUSEPORT:
869 sk->sk_reuseport = valbool;
870 break;
871 case SO_TYPE:
872 case SO_PROTOCOL:
873 case SO_DOMAIN:
874 case SO_ERROR:
875 ret = -ENOPROTOOPT;
876 break;
877 case SO_DONTROUTE:
878 sock_valbool_flag(sk, SOCK_LOCALROUTE, valbool);
879 sk_dst_reset(sk);
880 break;
881 case SO_BROADCAST:
882 sock_valbool_flag(sk, SOCK_BROADCAST, valbool);
883 break;
884 case SO_SNDBUF:
885 /* Don't error on this BSD doesn't and if you think
886 * about it this is right. Otherwise apps have to
887 * play 'guess the biggest size' games. RCVBUF/SNDBUF
888 * are treated in BSD as hints
889 */
890 val = min_t(u32, val, READ_ONCE(sysctl_wmem_max));
891 set_sndbuf:
892 /* Ensure val * 2 fits into an int, to prevent max_t()
893 * from treating it as a negative value.
894 */
895 val = min_t(int, val, INT_MAX / 2);
896 sk->sk_userlocks |= SOCK_SNDBUF_LOCK;
897 WRITE_ONCE(sk->sk_sndbuf,
898 max_t(int, val * 2, SOCK_MIN_SNDBUF));
899 /* Wake up sending tasks if we upped the value. */
900 sk->sk_write_space(sk);
901 break;
902
903 case SO_SNDBUFFORCE:
904 if (!capable(CAP_NET_ADMIN)) {
905 ret = -EPERM;
906 break;
907 }
908
909 /* No negative values (to prevent underflow, as val will be
910 * multiplied by 2).
911 */
912 if (val < 0)
913 val = 0;
914 goto set_sndbuf;
915
916 case SO_RCVBUF:
917 /* Don't error on this BSD doesn't and if you think
918 * about it this is right. Otherwise apps have to
919 * play 'guess the biggest size' games. RCVBUF/SNDBUF
920 * are treated in BSD as hints
921 */
922 __sock_set_rcvbuf(sk, min_t(u32, val, READ_ONCE(sysctl_rmem_max)));
923 break;
924
925 case SO_RCVBUFFORCE:
926 if (!capable(CAP_NET_ADMIN)) {
927 ret = -EPERM;
928 break;
929 }
930
931 /* No negative values (to prevent underflow, as val will be
932 * multiplied by 2).
933 */
934 __sock_set_rcvbuf(sk, max(val, 0));
935 break;
936
937 case SO_KEEPALIVE:
938 if (sk->sk_prot->keepalive)
939 sk->sk_prot->keepalive(sk, valbool);
940 sock_valbool_flag(sk, SOCK_KEEPOPEN, valbool);
941 break;
942
943 case SO_OOBINLINE:
944 sock_valbool_flag(sk, SOCK_URGINLINE, valbool);
945 break;
946
947 case SO_NO_CHECK:
948 sk->sk_no_check_tx = valbool;
949 break;
950
951 case SO_PRIORITY:
952 if ((val >= 0 && val <= 6) ||
953 ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN))
954 sk->sk_priority = val;
955 else
956 ret = -EPERM;
957 break;
958
959 case SO_LINGER:
960 if (optlen < sizeof(ling)) {
961 ret = -EINVAL; /* 1003.1g */
962 break;
963 }
964 if (copy_from_sockptr(&ling, optval, sizeof(ling))) {
965 ret = -EFAULT;
966 break;
967 }
968 if (!ling.l_onoff)
969 sock_reset_flag(sk, SOCK_LINGER);
970 else {
971 #if (BITS_PER_LONG == 32)
972 if ((unsigned int)ling.l_linger >= MAX_SCHEDULE_TIMEOUT/HZ)
973 sk->sk_lingertime = MAX_SCHEDULE_TIMEOUT;
974 else
975 #endif
976 sk->sk_lingertime = (unsigned int)ling.l_linger * HZ;
977 sock_set_flag(sk, SOCK_LINGER);
978 }
979 break;
980
981 case SO_BSDCOMPAT:
982 break;
983
984 case SO_PASSCRED:
985 if (valbool)
986 set_bit(SOCK_PASSCRED, &sock->flags);
987 else
988 clear_bit(SOCK_PASSCRED, &sock->flags);
989 break;
990
991 case SO_TIMESTAMP_OLD:
992 __sock_set_timestamps(sk, valbool, false, false);
993 break;
994 case SO_TIMESTAMP_NEW:
995 __sock_set_timestamps(sk, valbool, true, false);
996 break;
997 case SO_TIMESTAMPNS_OLD:
998 __sock_set_timestamps(sk, valbool, false, true);
999 break;
1000 case SO_TIMESTAMPNS_NEW:
1001 __sock_set_timestamps(sk, valbool, true, true);
1002 break;
1003 case SO_TIMESTAMPING_NEW:
1004 case SO_TIMESTAMPING_OLD:
1005 if (val & ~SOF_TIMESTAMPING_MASK) {
1006 ret = -EINVAL;
1007 break;
1008 }
1009
1010 if (val & SOF_TIMESTAMPING_OPT_ID &&
1011 !(sk->sk_tsflags & SOF_TIMESTAMPING_OPT_ID)) {
1012 if (sk->sk_protocol == IPPROTO_TCP &&
1013 sk->sk_type == SOCK_STREAM) {
1014 if ((1 << sk->sk_state) &
1015 (TCPF_CLOSE | TCPF_LISTEN)) {
1016 ret = -EINVAL;
1017 break;
1018 }
1019 sk->sk_tskey = tcp_sk(sk)->snd_una;
1020 } else {
1021 sk->sk_tskey = 0;
1022 }
1023 }
1024
1025 if (val & SOF_TIMESTAMPING_OPT_STATS &&
1026 !(val & SOF_TIMESTAMPING_OPT_TSONLY)) {
1027 ret = -EINVAL;
1028 break;
1029 }
1030
1031 sk->sk_tsflags = val;
1032 sock_valbool_flag(sk, SOCK_TSTAMP_NEW, optname == SO_TIMESTAMPING_NEW);
1033
1034 if (val & SOF_TIMESTAMPING_RX_SOFTWARE)
1035 sock_enable_timestamp(sk,
1036 SOCK_TIMESTAMPING_RX_SOFTWARE);
1037 else
1038 sock_disable_timestamp(sk,
1039 (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE));
1040 break;
1041
1042 case SO_RCVLOWAT:
1043 if (val < 0)
1044 val = INT_MAX;
1045 if (sock->ops->set_rcvlowat)
1046 ret = sock->ops->set_rcvlowat(sk, val);
1047 else
1048 WRITE_ONCE(sk->sk_rcvlowat, val ? : 1);
1049 break;
1050
1051 case SO_RCVTIMEO_OLD:
1052 case SO_RCVTIMEO_NEW:
1053 ret = sock_set_timeout(&sk->sk_rcvtimeo, optval,
1054 optlen, optname == SO_RCVTIMEO_OLD);
1055 break;
1056
1057 case SO_SNDTIMEO_OLD:
1058 case SO_SNDTIMEO_NEW:
1059 ret = sock_set_timeout(&sk->sk_sndtimeo, optval,
1060 optlen, optname == SO_SNDTIMEO_OLD);
1061 break;
1062
1063 case SO_ATTACH_FILTER: {
1064 struct sock_fprog fprog;
1065
1066 ret = copy_bpf_fprog_from_user(&fprog, optval, optlen);
1067 if (!ret)
1068 ret = sk_attach_filter(&fprog, sk);
1069 break;
1070 }
1071 case SO_ATTACH_BPF:
1072 ret = -EINVAL;
1073 if (optlen == sizeof(u32)) {
1074 u32 ufd;
1075
1076 ret = -EFAULT;
1077 if (copy_from_sockptr(&ufd, optval, sizeof(ufd)))
1078 break;
1079
1080 ret = sk_attach_bpf(ufd, sk);
1081 }
1082 break;
1083
1084 case SO_ATTACH_REUSEPORT_CBPF: {
1085 struct sock_fprog fprog;
1086
1087 ret = copy_bpf_fprog_from_user(&fprog, optval, optlen);
1088 if (!ret)
1089 ret = sk_reuseport_attach_filter(&fprog, sk);
1090 break;
1091 }
1092 case SO_ATTACH_REUSEPORT_EBPF:
1093 ret = -EINVAL;
1094 if (optlen == sizeof(u32)) {
1095 u32 ufd;
1096
1097 ret = -EFAULT;
1098 if (copy_from_sockptr(&ufd, optval, sizeof(ufd)))
1099 break;
1100
1101 ret = sk_reuseport_attach_bpf(ufd, sk);
1102 }
1103 break;
1104
1105 case SO_DETACH_REUSEPORT_BPF:
1106 ret = reuseport_detach_prog(sk);
1107 break;
1108
1109 case SO_DETACH_FILTER:
1110 ret = sk_detach_filter(sk);
1111 break;
1112
1113 case SO_LOCK_FILTER:
1114 if (sock_flag(sk, SOCK_FILTER_LOCKED) && !valbool)
1115 ret = -EPERM;
1116 else
1117 sock_valbool_flag(sk, SOCK_FILTER_LOCKED, valbool);
1118 break;
1119
1120 case SO_PASSSEC:
1121 if (valbool)
1122 set_bit(SOCK_PASSSEC, &sock->flags);
1123 else
1124 clear_bit(SOCK_PASSSEC, &sock->flags);
1125 break;
1126 case SO_MARK:
1127 if (!ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) {
1128 ret = -EPERM;
1129 break;
1130 }
1131
1132 __sock_set_mark(sk, val);
1133 break;
1134
1135 case SO_RXQ_OVFL:
1136 sock_valbool_flag(sk, SOCK_RXQ_OVFL, valbool);
1137 break;
1138
1139 case SO_WIFI_STATUS:
1140 sock_valbool_flag(sk, SOCK_WIFI_STATUS, valbool);
1141 break;
1142
1143 case SO_PEEK_OFF:
1144 if (sock->ops->set_peek_off)
1145 ret = sock->ops->set_peek_off(sk, val);
1146 else
1147 ret = -EOPNOTSUPP;
1148 break;
1149
1150 case SO_NOFCS:
1151 sock_valbool_flag(sk, SOCK_NOFCS, valbool);
1152 break;
1153
1154 case SO_SELECT_ERR_QUEUE:
1155 sock_valbool_flag(sk, SOCK_SELECT_ERR_QUEUE, valbool);
1156 break;
1157
1158 #ifdef CONFIG_NET_RX_BUSY_POLL
1159 case SO_BUSY_POLL:
1160 /* allow unprivileged users to decrease the value */
1161 if ((val > sk->sk_ll_usec) && !capable(CAP_NET_ADMIN))
1162 ret = -EPERM;
1163 else {
1164 if (val < 0)
1165 ret = -EINVAL;
1166 else
1167 WRITE_ONCE(sk->sk_ll_usec, val);
1168 }
1169 break;
1170 #endif
1171
1172 case SO_MAX_PACING_RATE:
1173 {
1174 unsigned long ulval = (val == ~0U) ? ~0UL : (unsigned int)val;
1175
1176 if (sizeof(ulval) != sizeof(val) &&
1177 optlen >= sizeof(ulval) &&
1178 copy_from_sockptr(&ulval, optval, sizeof(ulval))) {
1179 ret = -EFAULT;
1180 break;
1181 }
1182 if (ulval != ~0UL)
1183 cmpxchg(&sk->sk_pacing_status,
1184 SK_PACING_NONE,
1185 SK_PACING_NEEDED);
1186 /* Pairs with READ_ONCE() from sk_getsockopt() */
1187 WRITE_ONCE(sk->sk_max_pacing_rate, ulval);
1188 sk->sk_pacing_rate = min(sk->sk_pacing_rate, ulval);
1189 break;
1190 }
1191 case SO_INCOMING_CPU:
1192 WRITE_ONCE(sk->sk_incoming_cpu, val);
1193 break;
1194
1195 case SO_CNX_ADVICE:
1196 if (val == 1)
1197 dst_negative_advice(sk);
1198 break;
1199
1200 case SO_ZEROCOPY:
1201 if (sk->sk_family == PF_INET || sk->sk_family == PF_INET6) {
1202 if (!((sk->sk_type == SOCK_STREAM &&
1203 sk->sk_protocol == IPPROTO_TCP) ||
1204 (sk->sk_type == SOCK_DGRAM &&
1205 sk->sk_protocol == IPPROTO_UDP)))
1206 ret = -ENOTSUPP;
1207 } else if (sk->sk_family != PF_RDS) {
1208 ret = -ENOTSUPP;
1209 }
1210 if (!ret) {
1211 if (val < 0 || val > 1)
1212 ret = -EINVAL;
1213 else
1214 sock_valbool_flag(sk, SOCK_ZEROCOPY, valbool);
1215 }
1216 break;
1217
1218 case SO_TXTIME:
1219 if (optlen != sizeof(struct sock_txtime)) {
1220 ret = -EINVAL;
1221 break;
1222 } else if (copy_from_sockptr(&sk_txtime, optval,
1223 sizeof(struct sock_txtime))) {
1224 ret = -EFAULT;
1225 break;
1226 } else if (sk_txtime.flags & ~SOF_TXTIME_FLAGS_MASK) {
1227 ret = -EINVAL;
1228 break;
1229 }
1230 /* CLOCK_MONOTONIC is only used by sch_fq, and this packet
1231 * scheduler has enough safe guards.
1232 */
1233 if (sk_txtime.clockid != CLOCK_MONOTONIC &&
1234 !ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) {
1235 ret = -EPERM;
1236 break;
1237 }
1238 sock_valbool_flag(sk, SOCK_TXTIME, true);
1239 sk->sk_clockid = sk_txtime.clockid;
1240 sk->sk_txtime_deadline_mode =
1241 !!(sk_txtime.flags & SOF_TXTIME_DEADLINE_MODE);
1242 sk->sk_txtime_report_errors =
1243 !!(sk_txtime.flags & SOF_TXTIME_REPORT_ERRORS);
1244 break;
1245
1246 case SO_BINDTOIFINDEX:
1247 ret = sock_bindtoindex_locked(sk, val);
1248 break;
1249
1250 default:
1251 ret = -ENOPROTOOPT;
1252 break;
1253 }
1254 release_sock(sk);
1255 return ret;
1256 }
1257 EXPORT_SYMBOL(sock_setsockopt);
1258
sk_get_peer_cred(struct sock * sk)1259 static const struct cred *sk_get_peer_cred(struct sock *sk)
1260 {
1261 const struct cred *cred;
1262
1263 spin_lock(&sk->sk_peer_lock);
1264 cred = get_cred(sk->sk_peer_cred);
1265 spin_unlock(&sk->sk_peer_lock);
1266
1267 return cred;
1268 }
1269
cred_to_ucred(struct pid * pid,const struct cred * cred,struct ucred * ucred)1270 static void cred_to_ucred(struct pid *pid, const struct cred *cred,
1271 struct ucred *ucred)
1272 {
1273 ucred->pid = pid_vnr(pid);
1274 ucred->uid = ucred->gid = -1;
1275 if (cred) {
1276 struct user_namespace *current_ns = current_user_ns();
1277
1278 ucred->uid = from_kuid_munged(current_ns, cred->euid);
1279 ucred->gid = from_kgid_munged(current_ns, cred->egid);
1280 }
1281 }
1282
groups_to_user(gid_t __user * dst,const struct group_info * src)1283 static int groups_to_user(gid_t __user *dst, const struct group_info *src)
1284 {
1285 struct user_namespace *user_ns = current_user_ns();
1286 int i;
1287
1288 for (i = 0; i < src->ngroups; i++)
1289 if (put_user(from_kgid_munged(user_ns, src->gid[i]), dst + i))
1290 return -EFAULT;
1291
1292 return 0;
1293 }
1294
sock_getsockopt(struct socket * sock,int level,int optname,char __user * optval,int __user * optlen)1295 int sock_getsockopt(struct socket *sock, int level, int optname,
1296 char __user *optval, int __user *optlen)
1297 {
1298 struct sock *sk = sock->sk;
1299
1300 union {
1301 int val;
1302 u64 val64;
1303 unsigned long ulval;
1304 struct linger ling;
1305 struct old_timeval32 tm32;
1306 struct __kernel_old_timeval tm;
1307 struct __kernel_sock_timeval stm;
1308 struct sock_txtime txtime;
1309 } v;
1310
1311 int lv = sizeof(int);
1312 int len;
1313
1314 if (get_user(len, optlen))
1315 return -EFAULT;
1316 if (len < 0)
1317 return -EINVAL;
1318
1319 memset(&v, 0, sizeof(v));
1320
1321 switch (optname) {
1322 case SO_DEBUG:
1323 v.val = sock_flag(sk, SOCK_DBG);
1324 break;
1325
1326 case SO_DONTROUTE:
1327 v.val = sock_flag(sk, SOCK_LOCALROUTE);
1328 break;
1329
1330 case SO_BROADCAST:
1331 v.val = sock_flag(sk, SOCK_BROADCAST);
1332 break;
1333
1334 case SO_SNDBUF:
1335 v.val = READ_ONCE(sk->sk_sndbuf);
1336 break;
1337
1338 case SO_RCVBUF:
1339 v.val = READ_ONCE(sk->sk_rcvbuf);
1340 break;
1341
1342 case SO_REUSEADDR:
1343 v.val = sk->sk_reuse;
1344 break;
1345
1346 case SO_REUSEPORT:
1347 v.val = sk->sk_reuseport;
1348 break;
1349
1350 case SO_KEEPALIVE:
1351 v.val = sock_flag(sk, SOCK_KEEPOPEN);
1352 break;
1353
1354 case SO_TYPE:
1355 v.val = sk->sk_type;
1356 break;
1357
1358 case SO_PROTOCOL:
1359 v.val = sk->sk_protocol;
1360 break;
1361
1362 case SO_DOMAIN:
1363 v.val = sk->sk_family;
1364 break;
1365
1366 case SO_ERROR:
1367 v.val = -sock_error(sk);
1368 if (v.val == 0)
1369 v.val = xchg(&sk->sk_err_soft, 0);
1370 break;
1371
1372 case SO_OOBINLINE:
1373 v.val = sock_flag(sk, SOCK_URGINLINE);
1374 break;
1375
1376 case SO_NO_CHECK:
1377 v.val = sk->sk_no_check_tx;
1378 break;
1379
1380 case SO_PRIORITY:
1381 v.val = sk->sk_priority;
1382 break;
1383
1384 case SO_LINGER:
1385 lv = sizeof(v.ling);
1386 v.ling.l_onoff = sock_flag(sk, SOCK_LINGER);
1387 v.ling.l_linger = sk->sk_lingertime / HZ;
1388 break;
1389
1390 case SO_BSDCOMPAT:
1391 break;
1392
1393 case SO_TIMESTAMP_OLD:
1394 v.val = sock_flag(sk, SOCK_RCVTSTAMP) &&
1395 !sock_flag(sk, SOCK_TSTAMP_NEW) &&
1396 !sock_flag(sk, SOCK_RCVTSTAMPNS);
1397 break;
1398
1399 case SO_TIMESTAMPNS_OLD:
1400 v.val = sock_flag(sk, SOCK_RCVTSTAMPNS) && !sock_flag(sk, SOCK_TSTAMP_NEW);
1401 break;
1402
1403 case SO_TIMESTAMP_NEW:
1404 v.val = sock_flag(sk, SOCK_RCVTSTAMP) && sock_flag(sk, SOCK_TSTAMP_NEW);
1405 break;
1406
1407 case SO_TIMESTAMPNS_NEW:
1408 v.val = sock_flag(sk, SOCK_RCVTSTAMPNS) && sock_flag(sk, SOCK_TSTAMP_NEW);
1409 break;
1410
1411 case SO_TIMESTAMPING_OLD:
1412 v.val = sk->sk_tsflags;
1413 break;
1414
1415 case SO_RCVTIMEO_OLD:
1416 case SO_RCVTIMEO_NEW:
1417 lv = sock_get_timeout(sk->sk_rcvtimeo, &v, SO_RCVTIMEO_OLD == optname);
1418 break;
1419
1420 case SO_SNDTIMEO_OLD:
1421 case SO_SNDTIMEO_NEW:
1422 lv = sock_get_timeout(sk->sk_sndtimeo, &v, SO_SNDTIMEO_OLD == optname);
1423 break;
1424
1425 case SO_RCVLOWAT:
1426 v.val = READ_ONCE(sk->sk_rcvlowat);
1427 break;
1428
1429 case SO_SNDLOWAT:
1430 v.val = 1;
1431 break;
1432
1433 case SO_PASSCRED:
1434 v.val = !!test_bit(SOCK_PASSCRED, &sock->flags);
1435 break;
1436
1437 case SO_PEERCRED:
1438 {
1439 struct ucred peercred;
1440 if (len > sizeof(peercred))
1441 len = sizeof(peercred);
1442
1443 spin_lock(&sk->sk_peer_lock);
1444 cred_to_ucred(sk->sk_peer_pid, sk->sk_peer_cred, &peercred);
1445 spin_unlock(&sk->sk_peer_lock);
1446
1447 if (copy_to_user(optval, &peercred, len))
1448 return -EFAULT;
1449 goto lenout;
1450 }
1451
1452 case SO_PEERGROUPS:
1453 {
1454 const struct cred *cred;
1455 int ret, n;
1456
1457 cred = sk_get_peer_cred(sk);
1458 if (!cred)
1459 return -ENODATA;
1460
1461 n = cred->group_info->ngroups;
1462 if (len < n * sizeof(gid_t)) {
1463 len = n * sizeof(gid_t);
1464 put_cred(cred);
1465 return put_user(len, optlen) ? -EFAULT : -ERANGE;
1466 }
1467 len = n * sizeof(gid_t);
1468
1469 ret = groups_to_user((gid_t __user *)optval, cred->group_info);
1470 put_cred(cred);
1471 if (ret)
1472 return ret;
1473 goto lenout;
1474 }
1475
1476 case SO_PEERNAME:
1477 {
1478 char address[128];
1479
1480 lv = sock->ops->getname(sock, (struct sockaddr *)address, 2);
1481 if (lv < 0)
1482 return -ENOTCONN;
1483 if (lv < len)
1484 return -EINVAL;
1485 if (copy_to_user(optval, address, len))
1486 return -EFAULT;
1487 goto lenout;
1488 }
1489
1490 /* Dubious BSD thing... Probably nobody even uses it, but
1491 * the UNIX standard wants it for whatever reason... -DaveM
1492 */
1493 case SO_ACCEPTCONN:
1494 v.val = sk->sk_state == TCP_LISTEN;
1495 break;
1496
1497 case SO_PASSSEC:
1498 v.val = !!test_bit(SOCK_PASSSEC, &sock->flags);
1499 break;
1500
1501 case SO_PEERSEC:
1502 return security_socket_getpeersec_stream(sock, optval, optlen, len);
1503
1504 case SO_MARK:
1505 v.val = sk->sk_mark;
1506 break;
1507
1508 case SO_RXQ_OVFL:
1509 v.val = sock_flag(sk, SOCK_RXQ_OVFL);
1510 break;
1511
1512 case SO_WIFI_STATUS:
1513 v.val = sock_flag(sk, SOCK_WIFI_STATUS);
1514 break;
1515
1516 case SO_PEEK_OFF:
1517 if (!sock->ops->set_peek_off)
1518 return -EOPNOTSUPP;
1519
1520 v.val = READ_ONCE(sk->sk_peek_off);
1521 break;
1522 case SO_NOFCS:
1523 v.val = sock_flag(sk, SOCK_NOFCS);
1524 break;
1525
1526 case SO_BINDTODEVICE:
1527 return sock_getbindtodevice(sk, optval, optlen, len);
1528
1529 case SO_GET_FILTER:
1530 len = sk_get_filter(sk, (struct sock_filter __user *)optval, len);
1531 if (len < 0)
1532 return len;
1533
1534 goto lenout;
1535
1536 case SO_LOCK_FILTER:
1537 v.val = sock_flag(sk, SOCK_FILTER_LOCKED);
1538 break;
1539
1540 case SO_BPF_EXTENSIONS:
1541 v.val = bpf_tell_extensions();
1542 break;
1543
1544 case SO_SELECT_ERR_QUEUE:
1545 v.val = sock_flag(sk, SOCK_SELECT_ERR_QUEUE);
1546 break;
1547
1548 #ifdef CONFIG_NET_RX_BUSY_POLL
1549 case SO_BUSY_POLL:
1550 v.val = READ_ONCE(sk->sk_ll_usec);
1551 break;
1552 #endif
1553
1554 case SO_MAX_PACING_RATE:
1555 /* The READ_ONCE() pair with the WRITE_ONCE() in sk_setsockopt() */
1556 if (sizeof(v.ulval) != sizeof(v.val) && len >= sizeof(v.ulval)) {
1557 lv = sizeof(v.ulval);
1558 v.ulval = READ_ONCE(sk->sk_max_pacing_rate);
1559 } else {
1560 /* 32bit version */
1561 v.val = min_t(unsigned long, ~0U,
1562 READ_ONCE(sk->sk_max_pacing_rate));
1563 }
1564 break;
1565
1566 case SO_INCOMING_CPU:
1567 v.val = READ_ONCE(sk->sk_incoming_cpu);
1568 break;
1569
1570 case SO_MEMINFO:
1571 {
1572 u32 meminfo[SK_MEMINFO_VARS];
1573
1574 sk_get_meminfo(sk, meminfo);
1575
1576 len = min_t(unsigned int, len, sizeof(meminfo));
1577 if (copy_to_user(optval, &meminfo, len))
1578 return -EFAULT;
1579
1580 goto lenout;
1581 }
1582
1583 #ifdef CONFIG_NET_RX_BUSY_POLL
1584 case SO_INCOMING_NAPI_ID:
1585 v.val = READ_ONCE(sk->sk_napi_id);
1586
1587 /* aggregate non-NAPI IDs down to 0 */
1588 if (v.val < MIN_NAPI_ID)
1589 v.val = 0;
1590
1591 break;
1592 #endif
1593
1594 case SO_COOKIE:
1595 lv = sizeof(u64);
1596 if (len < lv)
1597 return -EINVAL;
1598 v.val64 = sock_gen_cookie(sk);
1599 break;
1600
1601 case SO_ZEROCOPY:
1602 v.val = sock_flag(sk, SOCK_ZEROCOPY);
1603 break;
1604
1605 case SO_TXTIME:
1606 lv = sizeof(v.txtime);
1607 v.txtime.clockid = sk->sk_clockid;
1608 v.txtime.flags |= sk->sk_txtime_deadline_mode ?
1609 SOF_TXTIME_DEADLINE_MODE : 0;
1610 v.txtime.flags |= sk->sk_txtime_report_errors ?
1611 SOF_TXTIME_REPORT_ERRORS : 0;
1612 break;
1613
1614 case SO_BINDTOIFINDEX:
1615 v.val = sk->sk_bound_dev_if;
1616 break;
1617
1618 default:
1619 /* We implement the SO_SNDLOWAT etc to not be settable
1620 * (1003.1g 7).
1621 */
1622 return -ENOPROTOOPT;
1623 }
1624
1625 if (len > lv)
1626 len = lv;
1627 if (copy_to_user(optval, &v, len))
1628 return -EFAULT;
1629 lenout:
1630 if (put_user(len, optlen))
1631 return -EFAULT;
1632 return 0;
1633 }
1634
1635 /*
1636 * Initialize an sk_lock.
1637 *
1638 * (We also register the sk_lock with the lock validator.)
1639 */
sock_lock_init(struct sock * sk)1640 static inline void sock_lock_init(struct sock *sk)
1641 {
1642 if (sk->sk_kern_sock)
1643 sock_lock_init_class_and_name(
1644 sk,
1645 af_family_kern_slock_key_strings[sk->sk_family],
1646 af_family_kern_slock_keys + sk->sk_family,
1647 af_family_kern_key_strings[sk->sk_family],
1648 af_family_kern_keys + sk->sk_family);
1649 else
1650 sock_lock_init_class_and_name(
1651 sk,
1652 af_family_slock_key_strings[sk->sk_family],
1653 af_family_slock_keys + sk->sk_family,
1654 af_family_key_strings[sk->sk_family],
1655 af_family_keys + sk->sk_family);
1656 }
1657
1658 /*
1659 * Copy all fields from osk to nsk but nsk->sk_refcnt must not change yet,
1660 * even temporarly, because of RCU lookups. sk_node should also be left as is.
1661 * We must not copy fields between sk_dontcopy_begin and sk_dontcopy_end
1662 */
sock_copy(struct sock * nsk,const struct sock * osk)1663 static void sock_copy(struct sock *nsk, const struct sock *osk)
1664 {
1665 const struct proto *prot = READ_ONCE(osk->sk_prot);
1666 #ifdef CONFIG_SECURITY_NETWORK
1667 void *sptr = nsk->sk_security;
1668 #endif
1669 memcpy(nsk, osk, offsetof(struct sock, sk_dontcopy_begin));
1670
1671 memcpy(&nsk->sk_dontcopy_end, &osk->sk_dontcopy_end,
1672 prot->obj_size - offsetof(struct sock, sk_dontcopy_end));
1673
1674 #ifdef CONFIG_SECURITY_NETWORK
1675 nsk->sk_security = sptr;
1676 security_sk_clone(osk, nsk);
1677 #endif
1678 }
1679
sk_prot_alloc(struct proto * prot,gfp_t priority,int family)1680 static struct sock *sk_prot_alloc(struct proto *prot, gfp_t priority,
1681 int family)
1682 {
1683 struct sock *sk;
1684 struct kmem_cache *slab;
1685
1686 slab = prot->slab;
1687 if (slab != NULL) {
1688 sk = kmem_cache_alloc(slab, priority & ~__GFP_ZERO);
1689 if (!sk)
1690 return sk;
1691 if (want_init_on_alloc(priority))
1692 sk_prot_clear_nulls(sk, prot->obj_size);
1693 } else
1694 sk = kmalloc(prot->obj_size, priority);
1695
1696 if (sk != NULL) {
1697 if (security_sk_alloc(sk, family, priority))
1698 goto out_free;
1699
1700 if (!try_module_get(prot->owner))
1701 goto out_free_sec;
1702 sk_tx_queue_clear(sk);
1703 }
1704
1705 return sk;
1706
1707 out_free_sec:
1708 security_sk_free(sk);
1709 out_free:
1710 if (slab != NULL)
1711 kmem_cache_free(slab, sk);
1712 else
1713 kfree(sk);
1714 return NULL;
1715 }
1716
sk_prot_free(struct proto * prot,struct sock * sk)1717 static void sk_prot_free(struct proto *prot, struct sock *sk)
1718 {
1719 struct kmem_cache *slab;
1720 struct module *owner;
1721
1722 owner = prot->owner;
1723 slab = prot->slab;
1724
1725 cgroup_sk_free(&sk->sk_cgrp_data);
1726 mem_cgroup_sk_free(sk);
1727 security_sk_free(sk);
1728 if (slab != NULL)
1729 kmem_cache_free(slab, sk);
1730 else
1731 kfree(sk);
1732 module_put(owner);
1733 }
1734
1735 /**
1736 * sk_alloc - All socket objects are allocated here
1737 * @net: the applicable net namespace
1738 * @family: protocol family
1739 * @priority: for allocation (%GFP_KERNEL, %GFP_ATOMIC, etc)
1740 * @prot: struct proto associated with this new sock instance
1741 * @kern: is this to be a kernel socket?
1742 */
sk_alloc(struct net * net,int family,gfp_t priority,struct proto * prot,int kern)1743 struct sock *sk_alloc(struct net *net, int family, gfp_t priority,
1744 struct proto *prot, int kern)
1745 {
1746 struct sock *sk;
1747
1748 sk = sk_prot_alloc(prot, priority | __GFP_ZERO, family);
1749 if (sk) {
1750 sk->sk_family = family;
1751 /*
1752 * See comment in struct sock definition to understand
1753 * why we need sk_prot_creator -acme
1754 */
1755 sk->sk_prot = sk->sk_prot_creator = prot;
1756 sk->sk_kern_sock = kern;
1757 sock_lock_init(sk);
1758 sk->sk_net_refcnt = kern ? 0 : 1;
1759 if (likely(sk->sk_net_refcnt)) {
1760 get_net(net);
1761 sock_inuse_add(net, 1);
1762 }
1763
1764 sock_net_set(sk, net);
1765 refcount_set(&sk->sk_wmem_alloc, 1);
1766
1767 mem_cgroup_sk_alloc(sk);
1768 cgroup_sk_alloc(&sk->sk_cgrp_data);
1769 sock_update_classid(&sk->sk_cgrp_data);
1770 sock_update_netprioidx(&sk->sk_cgrp_data);
1771 sk_tx_queue_clear(sk);
1772 }
1773
1774 return sk;
1775 }
1776 EXPORT_SYMBOL(sk_alloc);
1777
1778 /* Sockets having SOCK_RCU_FREE will call this function after one RCU
1779 * grace period. This is the case for UDP sockets and TCP listeners.
1780 */
__sk_destruct(struct rcu_head * head)1781 static void __sk_destruct(struct rcu_head *head)
1782 {
1783 struct sock *sk = container_of(head, struct sock, sk_rcu);
1784 struct sk_filter *filter;
1785
1786 if (sk->sk_destruct)
1787 sk->sk_destruct(sk);
1788
1789 filter = rcu_dereference_check(sk->sk_filter,
1790 refcount_read(&sk->sk_wmem_alloc) == 0);
1791 if (filter) {
1792 sk_filter_uncharge(sk, filter);
1793 RCU_INIT_POINTER(sk->sk_filter, NULL);
1794 }
1795
1796 sock_disable_timestamp(sk, SK_FLAGS_TIMESTAMP);
1797
1798 #ifdef CONFIG_BPF_SYSCALL
1799 bpf_sk_storage_free(sk);
1800 #endif
1801
1802 if (atomic_read(&sk->sk_omem_alloc))
1803 pr_debug("%s: optmem leakage (%d bytes) detected\n",
1804 __func__, atomic_read(&sk->sk_omem_alloc));
1805
1806 if (sk->sk_frag.page) {
1807 put_page(sk->sk_frag.page);
1808 sk->sk_frag.page = NULL;
1809 }
1810
1811 /* We do not need to acquire sk->sk_peer_lock, we are the last user. */
1812 put_cred(sk->sk_peer_cred);
1813 put_pid(sk->sk_peer_pid);
1814
1815 if (likely(sk->sk_net_refcnt))
1816 put_net(sock_net(sk));
1817 sk_prot_free(sk->sk_prot_creator, sk);
1818 }
1819
sk_destruct(struct sock * sk)1820 void sk_destruct(struct sock *sk)
1821 {
1822 bool use_call_rcu = sock_flag(sk, SOCK_RCU_FREE);
1823
1824 if (rcu_access_pointer(sk->sk_reuseport_cb)) {
1825 reuseport_detach_sock(sk);
1826 use_call_rcu = true;
1827 }
1828
1829 if (use_call_rcu)
1830 call_rcu(&sk->sk_rcu, __sk_destruct);
1831 else
1832 __sk_destruct(&sk->sk_rcu);
1833 }
1834
__sk_free(struct sock * sk)1835 static void __sk_free(struct sock *sk)
1836 {
1837 if (likely(sk->sk_net_refcnt))
1838 sock_inuse_add(sock_net(sk), -1);
1839
1840 if (unlikely(sk->sk_net_refcnt && sock_diag_has_destroy_listeners(sk)))
1841 sock_diag_broadcast_destroy(sk);
1842 else
1843 sk_destruct(sk);
1844 }
1845
sk_free(struct sock * sk)1846 void sk_free(struct sock *sk)
1847 {
1848 /*
1849 * We subtract one from sk_wmem_alloc and can know if
1850 * some packets are still in some tx queue.
1851 * If not null, sock_wfree() will call __sk_free(sk) later
1852 */
1853 if (refcount_dec_and_test(&sk->sk_wmem_alloc))
1854 __sk_free(sk);
1855 }
1856 EXPORT_SYMBOL(sk_free);
1857
sk_init_common(struct sock * sk)1858 static void sk_init_common(struct sock *sk)
1859 {
1860 skb_queue_head_init(&sk->sk_receive_queue);
1861 skb_queue_head_init(&sk->sk_write_queue);
1862 skb_queue_head_init(&sk->sk_error_queue);
1863
1864 rwlock_init(&sk->sk_callback_lock);
1865 lockdep_set_class_and_name(&sk->sk_receive_queue.lock,
1866 af_rlock_keys + sk->sk_family,
1867 af_family_rlock_key_strings[sk->sk_family]);
1868 lockdep_set_class_and_name(&sk->sk_write_queue.lock,
1869 af_wlock_keys + sk->sk_family,
1870 af_family_wlock_key_strings[sk->sk_family]);
1871 lockdep_set_class_and_name(&sk->sk_error_queue.lock,
1872 af_elock_keys + sk->sk_family,
1873 af_family_elock_key_strings[sk->sk_family]);
1874 lockdep_set_class_and_name(&sk->sk_callback_lock,
1875 af_callback_keys + sk->sk_family,
1876 af_family_clock_key_strings[sk->sk_family]);
1877 }
1878
1879 /**
1880 * sk_clone_lock - clone a socket, and lock its clone
1881 * @sk: the socket to clone
1882 * @priority: for allocation (%GFP_KERNEL, %GFP_ATOMIC, etc)
1883 *
1884 * Caller must unlock socket even in error path (bh_unlock_sock(newsk))
1885 */
sk_clone_lock(const struct sock * sk,const gfp_t priority)1886 struct sock *sk_clone_lock(const struct sock *sk, const gfp_t priority)
1887 {
1888 struct proto *prot = READ_ONCE(sk->sk_prot);
1889 struct sk_filter *filter;
1890 bool is_charged = true;
1891 struct sock *newsk;
1892
1893 newsk = sk_prot_alloc(prot, priority, sk->sk_family);
1894 if (!newsk)
1895 goto out;
1896
1897 sock_copy(newsk, sk);
1898
1899 newsk->sk_prot_creator = prot;
1900
1901 /* SANITY */
1902 if (likely(newsk->sk_net_refcnt)) {
1903 get_net(sock_net(newsk));
1904 sock_inuse_add(sock_net(newsk), 1);
1905 }
1906 sk_node_init(&newsk->sk_node);
1907 sock_lock_init(newsk);
1908 bh_lock_sock(newsk);
1909 newsk->sk_backlog.head = newsk->sk_backlog.tail = NULL;
1910 newsk->sk_backlog.len = 0;
1911
1912 atomic_set(&newsk->sk_rmem_alloc, 0);
1913
1914 /* sk_wmem_alloc set to one (see sk_free() and sock_wfree()) */
1915 refcount_set(&newsk->sk_wmem_alloc, 1);
1916
1917 atomic_set(&newsk->sk_omem_alloc, 0);
1918 sk_init_common(newsk);
1919
1920 newsk->sk_dst_cache = NULL;
1921 newsk->sk_dst_pending_confirm = 0;
1922 newsk->sk_wmem_queued = 0;
1923 newsk->sk_forward_alloc = 0;
1924 atomic_set(&newsk->sk_drops, 0);
1925 newsk->sk_send_head = NULL;
1926 newsk->sk_userlocks = sk->sk_userlocks & ~SOCK_BINDPORT_LOCK;
1927 atomic_set(&newsk->sk_zckey, 0);
1928
1929 sock_reset_flag(newsk, SOCK_DONE);
1930
1931 /* sk->sk_memcg will be populated at accept() time */
1932 newsk->sk_memcg = NULL;
1933
1934 cgroup_sk_clone(&newsk->sk_cgrp_data);
1935
1936 rcu_read_lock();
1937 filter = rcu_dereference(sk->sk_filter);
1938 if (filter != NULL)
1939 /* though it's an empty new sock, the charging may fail
1940 * if sysctl_optmem_max was changed between creation of
1941 * original socket and cloning
1942 */
1943 is_charged = sk_filter_charge(newsk, filter);
1944 RCU_INIT_POINTER(newsk->sk_filter, filter);
1945 rcu_read_unlock();
1946
1947 if (unlikely(!is_charged || xfrm_sk_clone_policy(newsk, sk))) {
1948 /* We need to make sure that we don't uncharge the new
1949 * socket if we couldn't charge it in the first place
1950 * as otherwise we uncharge the parent's filter.
1951 */
1952 if (!is_charged)
1953 RCU_INIT_POINTER(newsk->sk_filter, NULL);
1954 sk_free_unlock_clone(newsk);
1955 newsk = NULL;
1956 goto out;
1957 }
1958 RCU_INIT_POINTER(newsk->sk_reuseport_cb, NULL);
1959
1960 if (bpf_sk_storage_clone(sk, newsk)) {
1961 sk_free_unlock_clone(newsk);
1962 newsk = NULL;
1963 goto out;
1964 }
1965
1966 /* Clear sk_user_data if parent had the pointer tagged
1967 * as not suitable for copying when cloning.
1968 */
1969 if (sk_user_data_is_nocopy(newsk))
1970 newsk->sk_user_data = NULL;
1971
1972 newsk->sk_err = 0;
1973 newsk->sk_err_soft = 0;
1974 newsk->sk_priority = 0;
1975 newsk->sk_incoming_cpu = raw_smp_processor_id();
1976
1977 /* Before updating sk_refcnt, we must commit prior changes to memory
1978 * (Documentation/RCU/rculist_nulls.rst for details)
1979 */
1980 smp_wmb();
1981 refcount_set(&newsk->sk_refcnt, 2);
1982
1983 /* Increment the counter in the same struct proto as the master
1984 * sock (sk_refcnt_debug_inc uses newsk->sk_prot->socks, that
1985 * is the same as sk->sk_prot->socks, as this field was copied
1986 * with memcpy).
1987 *
1988 * This _changes_ the previous behaviour, where
1989 * tcp_create_openreq_child always was incrementing the
1990 * equivalent to tcp_prot->socks (inet_sock_nr), so this have
1991 * to be taken into account in all callers. -acme
1992 */
1993 sk_refcnt_debug_inc(newsk);
1994 sk_set_socket(newsk, NULL);
1995 sk_tx_queue_clear(newsk);
1996 RCU_INIT_POINTER(newsk->sk_wq, NULL);
1997
1998 if (newsk->sk_prot->sockets_allocated)
1999 sk_sockets_allocated_inc(newsk);
2000
2001 if (sock_needs_netstamp(sk) && newsk->sk_flags & SK_FLAGS_TIMESTAMP)
2002 net_enable_timestamp();
2003 out:
2004 return newsk;
2005 }
2006 EXPORT_SYMBOL_GPL(sk_clone_lock);
2007
sk_free_unlock_clone(struct sock * sk)2008 void sk_free_unlock_clone(struct sock *sk)
2009 {
2010 /* It is still raw copy of parent, so invalidate
2011 * destructor and make plain sk_free() */
2012 sk->sk_destruct = NULL;
2013 bh_unlock_sock(sk);
2014 sk_free(sk);
2015 }
2016 EXPORT_SYMBOL_GPL(sk_free_unlock_clone);
2017
sk_setup_caps(struct sock * sk,struct dst_entry * dst)2018 void sk_setup_caps(struct sock *sk, struct dst_entry *dst)
2019 {
2020 u32 max_segs = 1;
2021
2022 sk->sk_route_caps = dst->dev->features | sk->sk_route_forced_caps;
2023 if (sk->sk_route_caps & NETIF_F_GSO)
2024 sk->sk_route_caps |= NETIF_F_GSO_SOFTWARE;
2025 sk->sk_route_caps &= ~sk->sk_route_nocaps;
2026 if (sk_can_gso(sk)) {
2027 if (dst->header_len && !xfrm_dst_offload_ok(dst)) {
2028 sk->sk_route_caps &= ~NETIF_F_GSO_MASK;
2029 } else {
2030 sk->sk_route_caps |= NETIF_F_SG | NETIF_F_HW_CSUM;
2031 sk->sk_gso_max_size = dst->dev->gso_max_size;
2032 max_segs = max_t(u32, dst->dev->gso_max_segs, 1);
2033 }
2034 }
2035 sk->sk_gso_max_segs = max_segs;
2036 sk_dst_set(sk, dst);
2037 }
2038 EXPORT_SYMBOL_GPL(sk_setup_caps);
2039
2040 /*
2041 * Simple resource managers for sockets.
2042 */
2043
2044
2045 /*
2046 * Write buffer destructor automatically called from kfree_skb.
2047 */
sock_wfree(struct sk_buff * skb)2048 void sock_wfree(struct sk_buff *skb)
2049 {
2050 struct sock *sk = skb->sk;
2051 unsigned int len = skb->truesize;
2052
2053 if (!sock_flag(sk, SOCK_USE_WRITE_QUEUE)) {
2054 /*
2055 * Keep a reference on sk_wmem_alloc, this will be released
2056 * after sk_write_space() call
2057 */
2058 WARN_ON(refcount_sub_and_test(len - 1, &sk->sk_wmem_alloc));
2059 sk->sk_write_space(sk);
2060 len = 1;
2061 }
2062 /*
2063 * if sk_wmem_alloc reaches 0, we must finish what sk_free()
2064 * could not do because of in-flight packets
2065 */
2066 if (refcount_sub_and_test(len, &sk->sk_wmem_alloc))
2067 __sk_free(sk);
2068 }
2069 EXPORT_SYMBOL(sock_wfree);
2070
2071 /* This variant of sock_wfree() is used by TCP,
2072 * since it sets SOCK_USE_WRITE_QUEUE.
2073 */
__sock_wfree(struct sk_buff * skb)2074 void __sock_wfree(struct sk_buff *skb)
2075 {
2076 struct sock *sk = skb->sk;
2077
2078 if (refcount_sub_and_test(skb->truesize, &sk->sk_wmem_alloc))
2079 __sk_free(sk);
2080 }
2081
skb_set_owner_w(struct sk_buff * skb,struct sock * sk)2082 void skb_set_owner_w(struct sk_buff *skb, struct sock *sk)
2083 {
2084 skb_orphan(skb);
2085 skb->sk = sk;
2086 #ifdef CONFIG_INET
2087 if (unlikely(!sk_fullsock(sk))) {
2088 skb->destructor = sock_edemux;
2089 sock_hold(sk);
2090 return;
2091 }
2092 #endif
2093 skb->destructor = sock_wfree;
2094 skb_set_hash_from_sk(skb, sk);
2095 /*
2096 * We used to take a refcount on sk, but following operation
2097 * is enough to guarantee sk_free() wont free this sock until
2098 * all in-flight packets are completed
2099 */
2100 refcount_add(skb->truesize, &sk->sk_wmem_alloc);
2101 }
2102 EXPORT_SYMBOL(skb_set_owner_w);
2103
can_skb_orphan_partial(const struct sk_buff * skb)2104 static bool can_skb_orphan_partial(const struct sk_buff *skb)
2105 {
2106 #ifdef CONFIG_TLS_DEVICE
2107 /* Drivers depend on in-order delivery for crypto offload,
2108 * partial orphan breaks out-of-order-OK logic.
2109 */
2110 if (skb->decrypted)
2111 return false;
2112 #endif
2113 return (skb->destructor == sock_wfree ||
2114 (IS_ENABLED(CONFIG_INET) && skb->destructor == tcp_wfree));
2115 }
2116
2117 /* This helper is used by netem, as it can hold packets in its
2118 * delay queue. We want to allow the owner socket to send more
2119 * packets, as if they were already TX completed by a typical driver.
2120 * But we also want to keep skb->sk set because some packet schedulers
2121 * rely on it (sch_fq for example).
2122 */
skb_orphan_partial(struct sk_buff * skb)2123 void skb_orphan_partial(struct sk_buff *skb)
2124 {
2125 if (skb_is_tcp_pure_ack(skb))
2126 return;
2127
2128 if (can_skb_orphan_partial(skb) && skb_set_owner_sk_safe(skb, skb->sk))
2129 return;
2130
2131 skb_orphan(skb);
2132 }
2133 EXPORT_SYMBOL(skb_orphan_partial);
2134
2135 /*
2136 * Read buffer destructor automatically called from kfree_skb.
2137 */
sock_rfree(struct sk_buff * skb)2138 void sock_rfree(struct sk_buff *skb)
2139 {
2140 struct sock *sk = skb->sk;
2141 unsigned int len = skb->truesize;
2142
2143 atomic_sub(len, &sk->sk_rmem_alloc);
2144 sk_mem_uncharge(sk, len);
2145 }
2146 EXPORT_SYMBOL(sock_rfree);
2147
2148 /*
2149 * Buffer destructor for skbs that are not used directly in read or write
2150 * path, e.g. for error handler skbs. Automatically called from kfree_skb.
2151 */
sock_efree(struct sk_buff * skb)2152 void sock_efree(struct sk_buff *skb)
2153 {
2154 sock_put(skb->sk);
2155 }
2156 EXPORT_SYMBOL(sock_efree);
2157
2158 /* Buffer destructor for prefetch/receive path where reference count may
2159 * not be held, e.g. for listen sockets.
2160 */
2161 #ifdef CONFIG_INET
sock_pfree(struct sk_buff * skb)2162 void sock_pfree(struct sk_buff *skb)
2163 {
2164 if (sk_is_refcounted(skb->sk))
2165 sock_gen_put(skb->sk);
2166 }
2167 EXPORT_SYMBOL(sock_pfree);
2168 #endif /* CONFIG_INET */
2169
sock_i_uid(struct sock * sk)2170 kuid_t sock_i_uid(struct sock *sk)
2171 {
2172 kuid_t uid;
2173
2174 read_lock_bh(&sk->sk_callback_lock);
2175 uid = sk->sk_socket ? SOCK_INODE(sk->sk_socket)->i_uid : GLOBAL_ROOT_UID;
2176 read_unlock_bh(&sk->sk_callback_lock);
2177 return uid;
2178 }
2179 EXPORT_SYMBOL(sock_i_uid);
2180
__sock_i_ino(struct sock * sk)2181 unsigned long __sock_i_ino(struct sock *sk)
2182 {
2183 unsigned long ino;
2184
2185 read_lock(&sk->sk_callback_lock);
2186 ino = sk->sk_socket ? SOCK_INODE(sk->sk_socket)->i_ino : 0;
2187 read_unlock(&sk->sk_callback_lock);
2188 return ino;
2189 }
2190 EXPORT_SYMBOL(__sock_i_ino);
2191
sock_i_ino(struct sock * sk)2192 unsigned long sock_i_ino(struct sock *sk)
2193 {
2194 unsigned long ino;
2195
2196 local_bh_disable();
2197 ino = __sock_i_ino(sk);
2198 local_bh_enable();
2199 return ino;
2200 }
2201 EXPORT_SYMBOL(sock_i_ino);
2202
2203 /*
2204 * Allocate a skb from the socket's send buffer.
2205 */
sock_wmalloc(struct sock * sk,unsigned long size,int force,gfp_t priority)2206 struct sk_buff *sock_wmalloc(struct sock *sk, unsigned long size, int force,
2207 gfp_t priority)
2208 {
2209 if (force ||
2210 refcount_read(&sk->sk_wmem_alloc) < READ_ONCE(sk->sk_sndbuf)) {
2211 struct sk_buff *skb = alloc_skb(size, priority);
2212
2213 if (skb) {
2214 skb_set_owner_w(skb, sk);
2215 return skb;
2216 }
2217 }
2218 return NULL;
2219 }
2220 EXPORT_SYMBOL(sock_wmalloc);
2221
sock_ofree(struct sk_buff * skb)2222 static void sock_ofree(struct sk_buff *skb)
2223 {
2224 struct sock *sk = skb->sk;
2225
2226 atomic_sub(skb->truesize, &sk->sk_omem_alloc);
2227 }
2228
sock_omalloc(struct sock * sk,unsigned long size,gfp_t priority)2229 struct sk_buff *sock_omalloc(struct sock *sk, unsigned long size,
2230 gfp_t priority)
2231 {
2232 struct sk_buff *skb;
2233
2234 /* small safe race: SKB_TRUESIZE may differ from final skb->truesize */
2235 if (atomic_read(&sk->sk_omem_alloc) + SKB_TRUESIZE(size) >
2236 READ_ONCE(sysctl_optmem_max))
2237 return NULL;
2238
2239 skb = alloc_skb(size, priority);
2240 if (!skb)
2241 return NULL;
2242
2243 atomic_add(skb->truesize, &sk->sk_omem_alloc);
2244 skb->sk = sk;
2245 skb->destructor = sock_ofree;
2246 return skb;
2247 }
2248
2249 /*
2250 * Allocate a memory block from the socket's option memory buffer.
2251 */
sock_kmalloc(struct sock * sk,int size,gfp_t priority)2252 void *sock_kmalloc(struct sock *sk, int size, gfp_t priority)
2253 {
2254 int optmem_max = READ_ONCE(sysctl_optmem_max);
2255
2256 if ((unsigned int)size <= optmem_max &&
2257 atomic_read(&sk->sk_omem_alloc) + size < optmem_max) {
2258 void *mem;
2259 /* First do the add, to avoid the race if kmalloc
2260 * might sleep.
2261 */
2262 atomic_add(size, &sk->sk_omem_alloc);
2263 mem = kmalloc(size, priority);
2264 if (mem)
2265 return mem;
2266 atomic_sub(size, &sk->sk_omem_alloc);
2267 }
2268 return NULL;
2269 }
2270 EXPORT_SYMBOL(sock_kmalloc);
2271
2272 /* Free an option memory block. Note, we actually want the inline
2273 * here as this allows gcc to detect the nullify and fold away the
2274 * condition entirely.
2275 */
__sock_kfree_s(struct sock * sk,void * mem,int size,const bool nullify)2276 static inline void __sock_kfree_s(struct sock *sk, void *mem, int size,
2277 const bool nullify)
2278 {
2279 if (WARN_ON_ONCE(!mem))
2280 return;
2281 if (nullify)
2282 kfree_sensitive(mem);
2283 else
2284 kfree(mem);
2285 atomic_sub(size, &sk->sk_omem_alloc);
2286 }
2287
sock_kfree_s(struct sock * sk,void * mem,int size)2288 void sock_kfree_s(struct sock *sk, void *mem, int size)
2289 {
2290 __sock_kfree_s(sk, mem, size, false);
2291 }
2292 EXPORT_SYMBOL(sock_kfree_s);
2293
sock_kzfree_s(struct sock * sk,void * mem,int size)2294 void sock_kzfree_s(struct sock *sk, void *mem, int size)
2295 {
2296 __sock_kfree_s(sk, mem, size, true);
2297 }
2298 EXPORT_SYMBOL(sock_kzfree_s);
2299
2300 /* It is almost wait_for_tcp_memory minus release_sock/lock_sock.
2301 I think, these locks should be removed for datagram sockets.
2302 */
sock_wait_for_wmem(struct sock * sk,long timeo)2303 static long sock_wait_for_wmem(struct sock *sk, long timeo)
2304 {
2305 DEFINE_WAIT(wait);
2306
2307 sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk);
2308 for (;;) {
2309 if (!timeo)
2310 break;
2311 if (signal_pending(current))
2312 break;
2313 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
2314 prepare_to_wait(sk_sleep(sk), &wait, TASK_INTERRUPTIBLE);
2315 if (refcount_read(&sk->sk_wmem_alloc) < READ_ONCE(sk->sk_sndbuf))
2316 break;
2317 if (sk->sk_shutdown & SEND_SHUTDOWN)
2318 break;
2319 if (sk->sk_err)
2320 break;
2321 timeo = schedule_timeout(timeo);
2322 }
2323 finish_wait(sk_sleep(sk), &wait);
2324 return timeo;
2325 }
2326
2327
2328 /*
2329 * Generic send/receive buffer handlers
2330 */
2331
sock_alloc_send_pskb(struct sock * sk,unsigned long header_len,unsigned long data_len,int noblock,int * errcode,int max_page_order)2332 struct sk_buff *sock_alloc_send_pskb(struct sock *sk, unsigned long header_len,
2333 unsigned long data_len, int noblock,
2334 int *errcode, int max_page_order)
2335 {
2336 struct sk_buff *skb;
2337 long timeo;
2338 int err;
2339
2340 timeo = sock_sndtimeo(sk, noblock);
2341 for (;;) {
2342 err = sock_error(sk);
2343 if (err != 0)
2344 goto failure;
2345
2346 err = -EPIPE;
2347 if (sk->sk_shutdown & SEND_SHUTDOWN)
2348 goto failure;
2349
2350 if (sk_wmem_alloc_get(sk) < READ_ONCE(sk->sk_sndbuf))
2351 break;
2352
2353 sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk);
2354 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
2355 err = -EAGAIN;
2356 if (!timeo)
2357 goto failure;
2358 if (signal_pending(current))
2359 goto interrupted;
2360 timeo = sock_wait_for_wmem(sk, timeo);
2361 }
2362 skb = alloc_skb_with_frags(header_len, data_len, max_page_order,
2363 errcode, sk->sk_allocation);
2364 if (skb)
2365 skb_set_owner_w(skb, sk);
2366 return skb;
2367
2368 interrupted:
2369 err = sock_intr_errno(timeo);
2370 failure:
2371 *errcode = err;
2372 return NULL;
2373 }
2374 EXPORT_SYMBOL(sock_alloc_send_pskb);
2375
sock_alloc_send_skb(struct sock * sk,unsigned long size,int noblock,int * errcode)2376 struct sk_buff *sock_alloc_send_skb(struct sock *sk, unsigned long size,
2377 int noblock, int *errcode)
2378 {
2379 return sock_alloc_send_pskb(sk, size, 0, noblock, errcode, 0);
2380 }
2381 EXPORT_SYMBOL(sock_alloc_send_skb);
2382
__sock_cmsg_send(struct sock * sk,struct msghdr * msg,struct cmsghdr * cmsg,struct sockcm_cookie * sockc)2383 int __sock_cmsg_send(struct sock *sk, struct msghdr *msg, struct cmsghdr *cmsg,
2384 struct sockcm_cookie *sockc)
2385 {
2386 u32 tsflags;
2387
2388 switch (cmsg->cmsg_type) {
2389 case SO_MARK:
2390 if (!ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN))
2391 return -EPERM;
2392 if (cmsg->cmsg_len != CMSG_LEN(sizeof(u32)))
2393 return -EINVAL;
2394 sockc->mark = *(u32 *)CMSG_DATA(cmsg);
2395 break;
2396 case SO_TIMESTAMPING_OLD:
2397 if (cmsg->cmsg_len != CMSG_LEN(sizeof(u32)))
2398 return -EINVAL;
2399
2400 tsflags = *(u32 *)CMSG_DATA(cmsg);
2401 if (tsflags & ~SOF_TIMESTAMPING_TX_RECORD_MASK)
2402 return -EINVAL;
2403
2404 sockc->tsflags &= ~SOF_TIMESTAMPING_TX_RECORD_MASK;
2405 sockc->tsflags |= tsflags;
2406 break;
2407 case SCM_TXTIME:
2408 if (!sock_flag(sk, SOCK_TXTIME))
2409 return -EINVAL;
2410 if (cmsg->cmsg_len != CMSG_LEN(sizeof(u64)))
2411 return -EINVAL;
2412 sockc->transmit_time = get_unaligned((u64 *)CMSG_DATA(cmsg));
2413 break;
2414 /* SCM_RIGHTS and SCM_CREDENTIALS are semantically in SOL_UNIX. */
2415 case SCM_RIGHTS:
2416 case SCM_CREDENTIALS:
2417 break;
2418 default:
2419 return -EINVAL;
2420 }
2421 return 0;
2422 }
2423 EXPORT_SYMBOL(__sock_cmsg_send);
2424
sock_cmsg_send(struct sock * sk,struct msghdr * msg,struct sockcm_cookie * sockc)2425 int sock_cmsg_send(struct sock *sk, struct msghdr *msg,
2426 struct sockcm_cookie *sockc)
2427 {
2428 struct cmsghdr *cmsg;
2429 int ret;
2430
2431 for_each_cmsghdr(cmsg, msg) {
2432 if (!CMSG_OK(msg, cmsg))
2433 return -EINVAL;
2434 if (cmsg->cmsg_level != SOL_SOCKET)
2435 continue;
2436 ret = __sock_cmsg_send(sk, msg, cmsg, sockc);
2437 if (ret)
2438 return ret;
2439 }
2440 return 0;
2441 }
2442 EXPORT_SYMBOL(sock_cmsg_send);
2443
sk_enter_memory_pressure(struct sock * sk)2444 static void sk_enter_memory_pressure(struct sock *sk)
2445 {
2446 if (!sk->sk_prot->enter_memory_pressure)
2447 return;
2448
2449 sk->sk_prot->enter_memory_pressure(sk);
2450 }
2451
sk_leave_memory_pressure(struct sock * sk)2452 static void sk_leave_memory_pressure(struct sock *sk)
2453 {
2454 if (sk->sk_prot->leave_memory_pressure) {
2455 sk->sk_prot->leave_memory_pressure(sk);
2456 } else {
2457 unsigned long *memory_pressure = sk->sk_prot->memory_pressure;
2458
2459 if (memory_pressure && READ_ONCE(*memory_pressure))
2460 WRITE_ONCE(*memory_pressure, 0);
2461 }
2462 }
2463
2464 DEFINE_STATIC_KEY_FALSE(net_high_order_alloc_disable_key);
2465
2466 /**
2467 * skb_page_frag_refill - check that a page_frag contains enough room
2468 * @sz: minimum size of the fragment we want to get
2469 * @pfrag: pointer to page_frag
2470 * @gfp: priority for memory allocation
2471 *
2472 * Note: While this allocator tries to use high order pages, there is
2473 * no guarantee that allocations succeed. Therefore, @sz MUST be
2474 * less or equal than PAGE_SIZE.
2475 */
skb_page_frag_refill(unsigned int sz,struct page_frag * pfrag,gfp_t gfp)2476 bool skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t gfp)
2477 {
2478 if (pfrag->page) {
2479 if (page_ref_count(pfrag->page) == 1) {
2480 pfrag->offset = 0;
2481 return true;
2482 }
2483 if (pfrag->offset + sz <= pfrag->size)
2484 return true;
2485 put_page(pfrag->page);
2486 }
2487
2488 pfrag->offset = 0;
2489 if (SKB_FRAG_PAGE_ORDER &&
2490 !static_branch_unlikely(&net_high_order_alloc_disable_key)) {
2491 /* Avoid direct reclaim but allow kswapd to wake */
2492 pfrag->page = alloc_pages((gfp & ~__GFP_DIRECT_RECLAIM) |
2493 __GFP_COMP | __GFP_NOWARN |
2494 __GFP_NORETRY,
2495 SKB_FRAG_PAGE_ORDER);
2496 if (likely(pfrag->page)) {
2497 pfrag->size = PAGE_SIZE << SKB_FRAG_PAGE_ORDER;
2498 return true;
2499 }
2500 }
2501 pfrag->page = alloc_page(gfp);
2502 if (likely(pfrag->page)) {
2503 pfrag->size = PAGE_SIZE;
2504 return true;
2505 }
2506 return false;
2507 }
2508 EXPORT_SYMBOL(skb_page_frag_refill);
2509
sk_page_frag_refill(struct sock * sk,struct page_frag * pfrag)2510 bool sk_page_frag_refill(struct sock *sk, struct page_frag *pfrag)
2511 {
2512 if (likely(skb_page_frag_refill(32U, pfrag, sk->sk_allocation)))
2513 return true;
2514
2515 sk_enter_memory_pressure(sk);
2516 sk_stream_moderate_sndbuf(sk);
2517 return false;
2518 }
2519 EXPORT_SYMBOL(sk_page_frag_refill);
2520
__lock_sock(struct sock * sk)2521 static void __lock_sock(struct sock *sk)
2522 __releases(&sk->sk_lock.slock)
2523 __acquires(&sk->sk_lock.slock)
2524 {
2525 DEFINE_WAIT(wait);
2526
2527 for (;;) {
2528 prepare_to_wait_exclusive(&sk->sk_lock.wq, &wait,
2529 TASK_UNINTERRUPTIBLE);
2530 spin_unlock_bh(&sk->sk_lock.slock);
2531 schedule();
2532 spin_lock_bh(&sk->sk_lock.slock);
2533 if (!sock_owned_by_user(sk))
2534 break;
2535 }
2536 finish_wait(&sk->sk_lock.wq, &wait);
2537 }
2538
__release_sock(struct sock * sk)2539 void __release_sock(struct sock *sk)
2540 __releases(&sk->sk_lock.slock)
2541 __acquires(&sk->sk_lock.slock)
2542 {
2543 struct sk_buff *skb, *next;
2544
2545 while ((skb = sk->sk_backlog.head) != NULL) {
2546 sk->sk_backlog.head = sk->sk_backlog.tail = NULL;
2547
2548 spin_unlock_bh(&sk->sk_lock.slock);
2549
2550 do {
2551 next = skb->next;
2552 prefetch(next);
2553 WARN_ON_ONCE(skb_dst_is_noref(skb));
2554 skb_mark_not_on_list(skb);
2555 sk_backlog_rcv(sk, skb);
2556
2557 cond_resched();
2558
2559 skb = next;
2560 } while (skb != NULL);
2561
2562 spin_lock_bh(&sk->sk_lock.slock);
2563 }
2564
2565 /*
2566 * Doing the zeroing here guarantee we can not loop forever
2567 * while a wild producer attempts to flood us.
2568 */
2569 sk->sk_backlog.len = 0;
2570 }
2571
__sk_flush_backlog(struct sock * sk)2572 void __sk_flush_backlog(struct sock *sk)
2573 {
2574 spin_lock_bh(&sk->sk_lock.slock);
2575 __release_sock(sk);
2576 spin_unlock_bh(&sk->sk_lock.slock);
2577 }
2578
2579 /**
2580 * sk_wait_data - wait for data to arrive at sk_receive_queue
2581 * @sk: sock to wait on
2582 * @timeo: for how long
2583 * @skb: last skb seen on sk_receive_queue
2584 *
2585 * Now socket state including sk->sk_err is changed only under lock,
2586 * hence we may omit checks after joining wait queue.
2587 * We check receive queue before schedule() only as optimization;
2588 * it is very likely that release_sock() added new data.
2589 */
sk_wait_data(struct sock * sk,long * timeo,const struct sk_buff * skb)2590 int sk_wait_data(struct sock *sk, long *timeo, const struct sk_buff *skb)
2591 {
2592 DEFINE_WAIT_FUNC(wait, woken_wake_function);
2593 int rc;
2594
2595 add_wait_queue(sk_sleep(sk), &wait);
2596 sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk);
2597 rc = sk_wait_event(sk, timeo, skb_peek_tail(&sk->sk_receive_queue) != skb, &wait);
2598 sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk);
2599 remove_wait_queue(sk_sleep(sk), &wait);
2600 return rc;
2601 }
2602 EXPORT_SYMBOL(sk_wait_data);
2603
2604 /**
2605 * __sk_mem_raise_allocated - increase memory_allocated
2606 * @sk: socket
2607 * @size: memory size to allocate
2608 * @amt: pages to allocate
2609 * @kind: allocation type
2610 *
2611 * Similar to __sk_mem_schedule(), but does not update sk_forward_alloc
2612 */
__sk_mem_raise_allocated(struct sock * sk,int size,int amt,int kind)2613 int __sk_mem_raise_allocated(struct sock *sk, int size, int amt, int kind)
2614 {
2615 struct proto *prot = sk->sk_prot;
2616 long allocated = sk_memory_allocated_add(sk, amt);
2617 bool charged = true;
2618
2619 if (mem_cgroup_sockets_enabled && sk->sk_memcg &&
2620 !(charged = mem_cgroup_charge_skmem(sk->sk_memcg, amt)))
2621 goto suppress_allocation;
2622
2623 /* Under limit. */
2624 if (allocated <= sk_prot_mem_limits(sk, 0)) {
2625 sk_leave_memory_pressure(sk);
2626 return 1;
2627 }
2628
2629 /* Under pressure. */
2630 if (allocated > sk_prot_mem_limits(sk, 1))
2631 sk_enter_memory_pressure(sk);
2632
2633 /* Over hard limit. */
2634 if (allocated > sk_prot_mem_limits(sk, 2))
2635 goto suppress_allocation;
2636
2637 /* guarantee minimum buffer size under pressure */
2638 if (kind == SK_MEM_RECV) {
2639 if (atomic_read(&sk->sk_rmem_alloc) < sk_get_rmem0(sk, prot))
2640 return 1;
2641
2642 } else { /* SK_MEM_SEND */
2643 int wmem0 = sk_get_wmem0(sk, prot);
2644
2645 if (sk->sk_type == SOCK_STREAM) {
2646 if (sk->sk_wmem_queued < wmem0)
2647 return 1;
2648 } else if (refcount_read(&sk->sk_wmem_alloc) < wmem0) {
2649 return 1;
2650 }
2651 }
2652
2653 if (sk_has_memory_pressure(sk)) {
2654 u64 alloc;
2655
2656 if (!sk_under_memory_pressure(sk))
2657 return 1;
2658 alloc = sk_sockets_allocated_read_positive(sk);
2659 if (sk_prot_mem_limits(sk, 2) > alloc *
2660 sk_mem_pages(sk->sk_wmem_queued +
2661 atomic_read(&sk->sk_rmem_alloc) +
2662 sk->sk_forward_alloc))
2663 return 1;
2664 }
2665
2666 suppress_allocation:
2667
2668 if (kind == SK_MEM_SEND && sk->sk_type == SOCK_STREAM) {
2669 sk_stream_moderate_sndbuf(sk);
2670
2671 /* Fail only if socket is _under_ its sndbuf.
2672 * In this case we cannot block, so that we have to fail.
2673 */
2674 if (sk->sk_wmem_queued + size >= sk->sk_sndbuf)
2675 return 1;
2676 }
2677
2678 if (kind == SK_MEM_SEND || (kind == SK_MEM_RECV && charged))
2679 trace_sock_exceed_buf_limit(sk, prot, allocated, kind);
2680
2681 sk_memory_allocated_sub(sk, amt);
2682
2683 if (mem_cgroup_sockets_enabled && sk->sk_memcg)
2684 mem_cgroup_uncharge_skmem(sk->sk_memcg, amt);
2685
2686 return 0;
2687 }
2688 EXPORT_SYMBOL(__sk_mem_raise_allocated);
2689
2690 /**
2691 * __sk_mem_schedule - increase sk_forward_alloc and memory_allocated
2692 * @sk: socket
2693 * @size: memory size to allocate
2694 * @kind: allocation type
2695 *
2696 * If kind is SK_MEM_SEND, it means wmem allocation. Otherwise it means
2697 * rmem allocation. This function assumes that protocols which have
2698 * memory_pressure use sk_wmem_queued as write buffer accounting.
2699 */
__sk_mem_schedule(struct sock * sk,int size,int kind)2700 int __sk_mem_schedule(struct sock *sk, int size, int kind)
2701 {
2702 int ret, amt = sk_mem_pages(size);
2703
2704 sk->sk_forward_alloc += amt << SK_MEM_QUANTUM_SHIFT;
2705 ret = __sk_mem_raise_allocated(sk, size, amt, kind);
2706 if (!ret)
2707 sk->sk_forward_alloc -= amt << SK_MEM_QUANTUM_SHIFT;
2708 return ret;
2709 }
2710 EXPORT_SYMBOL(__sk_mem_schedule);
2711
2712 /**
2713 * __sk_mem_reduce_allocated - reclaim memory_allocated
2714 * @sk: socket
2715 * @amount: number of quanta
2716 *
2717 * Similar to __sk_mem_reclaim(), but does not update sk_forward_alloc
2718 */
__sk_mem_reduce_allocated(struct sock * sk,int amount)2719 void __sk_mem_reduce_allocated(struct sock *sk, int amount)
2720 {
2721 sk_memory_allocated_sub(sk, amount);
2722
2723 if (mem_cgroup_sockets_enabled && sk->sk_memcg)
2724 mem_cgroup_uncharge_skmem(sk->sk_memcg, amount);
2725
2726 if (sk_under_global_memory_pressure(sk) &&
2727 (sk_memory_allocated(sk) < sk_prot_mem_limits(sk, 0)))
2728 sk_leave_memory_pressure(sk);
2729 }
2730 EXPORT_SYMBOL(__sk_mem_reduce_allocated);
2731
2732 /**
2733 * __sk_mem_reclaim - reclaim sk_forward_alloc and memory_allocated
2734 * @sk: socket
2735 * @amount: number of bytes (rounded down to a SK_MEM_QUANTUM multiple)
2736 */
__sk_mem_reclaim(struct sock * sk,int amount)2737 void __sk_mem_reclaim(struct sock *sk, int amount)
2738 {
2739 amount >>= SK_MEM_QUANTUM_SHIFT;
2740 sk->sk_forward_alloc -= amount << SK_MEM_QUANTUM_SHIFT;
2741 __sk_mem_reduce_allocated(sk, amount);
2742 }
2743 EXPORT_SYMBOL(__sk_mem_reclaim);
2744
sk_set_peek_off(struct sock * sk,int val)2745 int sk_set_peek_off(struct sock *sk, int val)
2746 {
2747 WRITE_ONCE(sk->sk_peek_off, val);
2748 return 0;
2749 }
2750 EXPORT_SYMBOL_GPL(sk_set_peek_off);
2751
2752 /*
2753 * Set of default routines for initialising struct proto_ops when
2754 * the protocol does not support a particular function. In certain
2755 * cases where it makes no sense for a protocol to have a "do nothing"
2756 * function, some default processing is provided.
2757 */
2758
sock_no_bind(struct socket * sock,struct sockaddr * saddr,int len)2759 int sock_no_bind(struct socket *sock, struct sockaddr *saddr, int len)
2760 {
2761 return -EOPNOTSUPP;
2762 }
2763 EXPORT_SYMBOL(sock_no_bind);
2764
sock_no_connect(struct socket * sock,struct sockaddr * saddr,int len,int flags)2765 int sock_no_connect(struct socket *sock, struct sockaddr *saddr,
2766 int len, int flags)
2767 {
2768 return -EOPNOTSUPP;
2769 }
2770 EXPORT_SYMBOL(sock_no_connect);
2771
sock_no_socketpair(struct socket * sock1,struct socket * sock2)2772 int sock_no_socketpair(struct socket *sock1, struct socket *sock2)
2773 {
2774 return -EOPNOTSUPP;
2775 }
2776 EXPORT_SYMBOL(sock_no_socketpair);
2777
sock_no_accept(struct socket * sock,struct socket * newsock,int flags,bool kern)2778 int sock_no_accept(struct socket *sock, struct socket *newsock, int flags,
2779 bool kern)
2780 {
2781 return -EOPNOTSUPP;
2782 }
2783 EXPORT_SYMBOL(sock_no_accept);
2784
sock_no_getname(struct socket * sock,struct sockaddr * saddr,int peer)2785 int sock_no_getname(struct socket *sock, struct sockaddr *saddr,
2786 int peer)
2787 {
2788 return -EOPNOTSUPP;
2789 }
2790 EXPORT_SYMBOL(sock_no_getname);
2791
sock_no_ioctl(struct socket * sock,unsigned int cmd,unsigned long arg)2792 int sock_no_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg)
2793 {
2794 return -EOPNOTSUPP;
2795 }
2796 EXPORT_SYMBOL(sock_no_ioctl);
2797
sock_no_listen(struct socket * sock,int backlog)2798 int sock_no_listen(struct socket *sock, int backlog)
2799 {
2800 return -EOPNOTSUPP;
2801 }
2802 EXPORT_SYMBOL(sock_no_listen);
2803
sock_no_shutdown(struct socket * sock,int how)2804 int sock_no_shutdown(struct socket *sock, int how)
2805 {
2806 return -EOPNOTSUPP;
2807 }
2808 EXPORT_SYMBOL(sock_no_shutdown);
2809
sock_no_sendmsg(struct socket * sock,struct msghdr * m,size_t len)2810 int sock_no_sendmsg(struct socket *sock, struct msghdr *m, size_t len)
2811 {
2812 return -EOPNOTSUPP;
2813 }
2814 EXPORT_SYMBOL(sock_no_sendmsg);
2815
sock_no_sendmsg_locked(struct sock * sk,struct msghdr * m,size_t len)2816 int sock_no_sendmsg_locked(struct sock *sk, struct msghdr *m, size_t len)
2817 {
2818 return -EOPNOTSUPP;
2819 }
2820 EXPORT_SYMBOL(sock_no_sendmsg_locked);
2821
sock_no_recvmsg(struct socket * sock,struct msghdr * m,size_t len,int flags)2822 int sock_no_recvmsg(struct socket *sock, struct msghdr *m, size_t len,
2823 int flags)
2824 {
2825 return -EOPNOTSUPP;
2826 }
2827 EXPORT_SYMBOL(sock_no_recvmsg);
2828
sock_no_mmap(struct file * file,struct socket * sock,struct vm_area_struct * vma)2829 int sock_no_mmap(struct file *file, struct socket *sock, struct vm_area_struct *vma)
2830 {
2831 /* Mirror missing mmap method error code */
2832 return -ENODEV;
2833 }
2834 EXPORT_SYMBOL(sock_no_mmap);
2835
2836 /*
2837 * When a file is received (via SCM_RIGHTS, etc), we must bump the
2838 * various sock-based usage counts.
2839 */
__receive_sock(struct file * file)2840 void __receive_sock(struct file *file)
2841 {
2842 struct socket *sock;
2843 int error;
2844
2845 /*
2846 * The resulting value of "error" is ignored here since we only
2847 * need to take action when the file is a socket and testing
2848 * "sock" for NULL is sufficient.
2849 */
2850 sock = sock_from_file(file, &error);
2851 if (sock) {
2852 sock_update_netprioidx(&sock->sk->sk_cgrp_data);
2853 sock_update_classid(&sock->sk->sk_cgrp_data);
2854 }
2855 }
2856
sock_no_sendpage(struct socket * sock,struct page * page,int offset,size_t size,int flags)2857 ssize_t sock_no_sendpage(struct socket *sock, struct page *page, int offset, size_t size, int flags)
2858 {
2859 ssize_t res;
2860 struct msghdr msg = {.msg_flags = flags};
2861 struct kvec iov;
2862 char *kaddr = kmap(page);
2863 iov.iov_base = kaddr + offset;
2864 iov.iov_len = size;
2865 res = kernel_sendmsg(sock, &msg, &iov, 1, size);
2866 kunmap(page);
2867 return res;
2868 }
2869 EXPORT_SYMBOL(sock_no_sendpage);
2870
sock_no_sendpage_locked(struct sock * sk,struct page * page,int offset,size_t size,int flags)2871 ssize_t sock_no_sendpage_locked(struct sock *sk, struct page *page,
2872 int offset, size_t size, int flags)
2873 {
2874 ssize_t res;
2875 struct msghdr msg = {.msg_flags = flags};
2876 struct kvec iov;
2877 char *kaddr = kmap(page);
2878
2879 iov.iov_base = kaddr + offset;
2880 iov.iov_len = size;
2881 res = kernel_sendmsg_locked(sk, &msg, &iov, 1, size);
2882 kunmap(page);
2883 return res;
2884 }
2885 EXPORT_SYMBOL(sock_no_sendpage_locked);
2886
2887 /*
2888 * Default Socket Callbacks
2889 */
2890
sock_def_wakeup(struct sock * sk)2891 static void sock_def_wakeup(struct sock *sk)
2892 {
2893 struct socket_wq *wq;
2894
2895 rcu_read_lock();
2896 wq = rcu_dereference(sk->sk_wq);
2897 if (skwq_has_sleeper(wq))
2898 wake_up_interruptible_all(&wq->wait);
2899 rcu_read_unlock();
2900 }
2901
sock_def_error_report(struct sock * sk)2902 static void sock_def_error_report(struct sock *sk)
2903 {
2904 struct socket_wq *wq;
2905
2906 rcu_read_lock();
2907 wq = rcu_dereference(sk->sk_wq);
2908 if (skwq_has_sleeper(wq))
2909 wake_up_interruptible_poll(&wq->wait, EPOLLERR);
2910 sk_wake_async(sk, SOCK_WAKE_IO, POLL_ERR);
2911 rcu_read_unlock();
2912 }
2913
sock_def_readable(struct sock * sk)2914 void sock_def_readable(struct sock *sk)
2915 {
2916 struct socket_wq *wq;
2917
2918 rcu_read_lock();
2919 wq = rcu_dereference(sk->sk_wq);
2920 if (skwq_has_sleeper(wq))
2921 wake_up_interruptible_sync_poll(&wq->wait, EPOLLIN | EPOLLPRI |
2922 EPOLLRDNORM | EPOLLRDBAND);
2923 sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN);
2924 rcu_read_unlock();
2925 }
2926
sock_def_write_space(struct sock * sk)2927 static void sock_def_write_space(struct sock *sk)
2928 {
2929 struct socket_wq *wq;
2930
2931 rcu_read_lock();
2932
2933 /* Do not wake up a writer until he can make "significant"
2934 * progress. --DaveM
2935 */
2936 if ((refcount_read(&sk->sk_wmem_alloc) << 1) <= READ_ONCE(sk->sk_sndbuf)) {
2937 wq = rcu_dereference(sk->sk_wq);
2938 if (skwq_has_sleeper(wq))
2939 wake_up_interruptible_sync_poll(&wq->wait, EPOLLOUT |
2940 EPOLLWRNORM | EPOLLWRBAND);
2941
2942 /* Should agree with poll, otherwise some programs break */
2943 if (sock_writeable(sk))
2944 sk_wake_async(sk, SOCK_WAKE_SPACE, POLL_OUT);
2945 }
2946
2947 rcu_read_unlock();
2948 }
2949
sock_def_destruct(struct sock * sk)2950 static void sock_def_destruct(struct sock *sk)
2951 {
2952 }
2953
sk_send_sigurg(struct sock * sk)2954 void sk_send_sigurg(struct sock *sk)
2955 {
2956 if (sk->sk_socket && sk->sk_socket->file)
2957 if (send_sigurg(&sk->sk_socket->file->f_owner))
2958 sk_wake_async(sk, SOCK_WAKE_URG, POLL_PRI);
2959 }
2960 EXPORT_SYMBOL(sk_send_sigurg);
2961
sk_reset_timer(struct sock * sk,struct timer_list * timer,unsigned long expires)2962 void sk_reset_timer(struct sock *sk, struct timer_list* timer,
2963 unsigned long expires)
2964 {
2965 if (!mod_timer(timer, expires))
2966 sock_hold(sk);
2967 }
2968 EXPORT_SYMBOL(sk_reset_timer);
2969
sk_stop_timer(struct sock * sk,struct timer_list * timer)2970 void sk_stop_timer(struct sock *sk, struct timer_list* timer)
2971 {
2972 if (del_timer(timer))
2973 __sock_put(sk);
2974 }
2975 EXPORT_SYMBOL(sk_stop_timer);
2976
sk_stop_timer_sync(struct sock * sk,struct timer_list * timer)2977 void sk_stop_timer_sync(struct sock *sk, struct timer_list *timer)
2978 {
2979 if (del_timer_sync(timer))
2980 __sock_put(sk);
2981 }
2982 EXPORT_SYMBOL(sk_stop_timer_sync);
2983
sock_init_data_uid(struct socket * sock,struct sock * sk,kuid_t uid)2984 void sock_init_data_uid(struct socket *sock, struct sock *sk, kuid_t uid)
2985 {
2986 sk_init_common(sk);
2987 sk->sk_send_head = NULL;
2988
2989 timer_setup(&sk->sk_timer, NULL, 0);
2990
2991 sk->sk_allocation = GFP_KERNEL;
2992 sk->sk_rcvbuf = READ_ONCE(sysctl_rmem_default);
2993 sk->sk_sndbuf = READ_ONCE(sysctl_wmem_default);
2994 sk->sk_state = TCP_CLOSE;
2995 sk_set_socket(sk, sock);
2996
2997 sock_set_flag(sk, SOCK_ZAPPED);
2998
2999 if (sock) {
3000 sk->sk_type = sock->type;
3001 RCU_INIT_POINTER(sk->sk_wq, &sock->wq);
3002 sock->sk = sk;
3003 } else {
3004 RCU_INIT_POINTER(sk->sk_wq, NULL);
3005 }
3006 sk->sk_uid = uid;
3007
3008 rwlock_init(&sk->sk_callback_lock);
3009 if (sk->sk_kern_sock)
3010 lockdep_set_class_and_name(
3011 &sk->sk_callback_lock,
3012 af_kern_callback_keys + sk->sk_family,
3013 af_family_kern_clock_key_strings[sk->sk_family]);
3014 else
3015 lockdep_set_class_and_name(
3016 &sk->sk_callback_lock,
3017 af_callback_keys + sk->sk_family,
3018 af_family_clock_key_strings[sk->sk_family]);
3019
3020 sk->sk_state_change = sock_def_wakeup;
3021 sk->sk_data_ready = sock_def_readable;
3022 sk->sk_write_space = sock_def_write_space;
3023 sk->sk_error_report = sock_def_error_report;
3024 sk->sk_destruct = sock_def_destruct;
3025
3026 sk->sk_frag.page = NULL;
3027 sk->sk_frag.offset = 0;
3028 sk->sk_peek_off = -1;
3029
3030 sk->sk_peer_pid = NULL;
3031 sk->sk_peer_cred = NULL;
3032 spin_lock_init(&sk->sk_peer_lock);
3033
3034 sk->sk_write_pending = 0;
3035 sk->sk_rcvlowat = 1;
3036 sk->sk_rcvtimeo = MAX_SCHEDULE_TIMEOUT;
3037 sk->sk_sndtimeo = MAX_SCHEDULE_TIMEOUT;
3038
3039 sk->sk_stamp = SK_DEFAULT_STAMP;
3040 #if BITS_PER_LONG==32
3041 seqlock_init(&sk->sk_stamp_seq);
3042 #endif
3043 atomic_set(&sk->sk_zckey, 0);
3044
3045 #ifdef CONFIG_NET_RX_BUSY_POLL
3046 sk->sk_napi_id = 0;
3047 sk->sk_ll_usec = READ_ONCE(sysctl_net_busy_read);
3048 #endif
3049
3050 sk->sk_max_pacing_rate = ~0UL;
3051 sk->sk_pacing_rate = ~0UL;
3052 WRITE_ONCE(sk->sk_pacing_shift, 10);
3053 sk->sk_incoming_cpu = -1;
3054
3055 sk_rx_queue_clear(sk);
3056 /*
3057 * Before updating sk_refcnt, we must commit prior changes to memory
3058 * (Documentation/RCU/rculist_nulls.rst for details)
3059 */
3060 smp_wmb();
3061 refcount_set(&sk->sk_refcnt, 1);
3062 atomic_set(&sk->sk_drops, 0);
3063 }
3064 EXPORT_SYMBOL(sock_init_data_uid);
3065
sock_init_data(struct socket * sock,struct sock * sk)3066 void sock_init_data(struct socket *sock, struct sock *sk)
3067 {
3068 kuid_t uid = sock ?
3069 SOCK_INODE(sock)->i_uid :
3070 make_kuid(sock_net(sk)->user_ns, 0);
3071
3072 sock_init_data_uid(sock, sk, uid);
3073 }
3074 EXPORT_SYMBOL(sock_init_data);
3075
lock_sock_nested(struct sock * sk,int subclass)3076 void lock_sock_nested(struct sock *sk, int subclass)
3077 {
3078 might_sleep();
3079 spin_lock_bh(&sk->sk_lock.slock);
3080 if (sk->sk_lock.owned)
3081 __lock_sock(sk);
3082 sk->sk_lock.owned = 1;
3083 spin_unlock(&sk->sk_lock.slock);
3084 /*
3085 * The sk_lock has mutex_lock() semantics here:
3086 */
3087 mutex_acquire(&sk->sk_lock.dep_map, subclass, 0, _RET_IP_);
3088 local_bh_enable();
3089 }
3090 EXPORT_SYMBOL(lock_sock_nested);
3091
release_sock(struct sock * sk)3092 void release_sock(struct sock *sk)
3093 {
3094 spin_lock_bh(&sk->sk_lock.slock);
3095 if (sk->sk_backlog.tail)
3096 __release_sock(sk);
3097
3098 /* Warning : release_cb() might need to release sk ownership,
3099 * ie call sock_release_ownership(sk) before us.
3100 */
3101 if (sk->sk_prot->release_cb)
3102 sk->sk_prot->release_cb(sk);
3103
3104 sock_release_ownership(sk);
3105 if (waitqueue_active(&sk->sk_lock.wq))
3106 wake_up(&sk->sk_lock.wq);
3107 spin_unlock_bh(&sk->sk_lock.slock);
3108 }
3109 EXPORT_SYMBOL(release_sock);
3110
3111 /**
3112 * lock_sock_fast - fast version of lock_sock
3113 * @sk: socket
3114 *
3115 * This version should be used for very small section, where process wont block
3116 * return false if fast path is taken:
3117 *
3118 * sk_lock.slock locked, owned = 0, BH disabled
3119 *
3120 * return true if slow path is taken:
3121 *
3122 * sk_lock.slock unlocked, owned = 1, BH enabled
3123 */
lock_sock_fast(struct sock * sk)3124 bool lock_sock_fast(struct sock *sk)
3125 {
3126 might_sleep();
3127 spin_lock_bh(&sk->sk_lock.slock);
3128
3129 if (!sk->sk_lock.owned)
3130 /*
3131 * Note : We must disable BH
3132 */
3133 return false;
3134
3135 __lock_sock(sk);
3136 sk->sk_lock.owned = 1;
3137 spin_unlock(&sk->sk_lock.slock);
3138 /*
3139 * The sk_lock has mutex_lock() semantics here:
3140 */
3141 mutex_acquire(&sk->sk_lock.dep_map, 0, 0, _RET_IP_);
3142 local_bh_enable();
3143 return true;
3144 }
3145 EXPORT_SYMBOL(lock_sock_fast);
3146
sock_gettstamp(struct socket * sock,void __user * userstamp,bool timeval,bool time32)3147 int sock_gettstamp(struct socket *sock, void __user *userstamp,
3148 bool timeval, bool time32)
3149 {
3150 struct sock *sk = sock->sk;
3151 struct timespec64 ts;
3152
3153 sock_enable_timestamp(sk, SOCK_TIMESTAMP);
3154 ts = ktime_to_timespec64(sock_read_timestamp(sk));
3155 if (ts.tv_sec == -1)
3156 return -ENOENT;
3157 if (ts.tv_sec == 0) {
3158 ktime_t kt = ktime_get_real();
3159 sock_write_timestamp(sk, kt);
3160 ts = ktime_to_timespec64(kt);
3161 }
3162
3163 if (timeval)
3164 ts.tv_nsec /= 1000;
3165
3166 #ifdef CONFIG_COMPAT_32BIT_TIME
3167 if (time32)
3168 return put_old_timespec32(&ts, userstamp);
3169 #endif
3170 #ifdef CONFIG_SPARC64
3171 /* beware of padding in sparc64 timeval */
3172 if (timeval && !in_compat_syscall()) {
3173 struct __kernel_old_timeval __user tv = {
3174 .tv_sec = ts.tv_sec,
3175 .tv_usec = ts.tv_nsec,
3176 };
3177 if (copy_to_user(userstamp, &tv, sizeof(tv)))
3178 return -EFAULT;
3179 return 0;
3180 }
3181 #endif
3182 return put_timespec64(&ts, userstamp);
3183 }
3184 EXPORT_SYMBOL(sock_gettstamp);
3185
sock_enable_timestamp(struct sock * sk,enum sock_flags flag)3186 void sock_enable_timestamp(struct sock *sk, enum sock_flags flag)
3187 {
3188 if (!sock_flag(sk, flag)) {
3189 unsigned long previous_flags = sk->sk_flags;
3190
3191 sock_set_flag(sk, flag);
3192 /*
3193 * we just set one of the two flags which require net
3194 * time stamping, but time stamping might have been on
3195 * already because of the other one
3196 */
3197 if (sock_needs_netstamp(sk) &&
3198 !(previous_flags & SK_FLAGS_TIMESTAMP))
3199 net_enable_timestamp();
3200 }
3201 }
3202
sock_recv_errqueue(struct sock * sk,struct msghdr * msg,int len,int level,int type)3203 int sock_recv_errqueue(struct sock *sk, struct msghdr *msg, int len,
3204 int level, int type)
3205 {
3206 struct sock_exterr_skb *serr;
3207 struct sk_buff *skb;
3208 int copied, err;
3209
3210 err = -EAGAIN;
3211 skb = sock_dequeue_err_skb(sk);
3212 if (skb == NULL)
3213 goto out;
3214
3215 copied = skb->len;
3216 if (copied > len) {
3217 msg->msg_flags |= MSG_TRUNC;
3218 copied = len;
3219 }
3220 err = skb_copy_datagram_msg(skb, 0, msg, copied);
3221 if (err)
3222 goto out_free_skb;
3223
3224 sock_recv_timestamp(msg, sk, skb);
3225
3226 serr = SKB_EXT_ERR(skb);
3227 put_cmsg(msg, level, type, sizeof(serr->ee), &serr->ee);
3228
3229 msg->msg_flags |= MSG_ERRQUEUE;
3230 err = copied;
3231
3232 out_free_skb:
3233 kfree_skb(skb);
3234 out:
3235 return err;
3236 }
3237 EXPORT_SYMBOL(sock_recv_errqueue);
3238
3239 /*
3240 * Get a socket option on an socket.
3241 *
3242 * FIX: POSIX 1003.1g is very ambiguous here. It states that
3243 * asynchronous errors should be reported by getsockopt. We assume
3244 * this means if you specify SO_ERROR (otherwise whats the point of it).
3245 */
sock_common_getsockopt(struct socket * sock,int level,int optname,char __user * optval,int __user * optlen)3246 int sock_common_getsockopt(struct socket *sock, int level, int optname,
3247 char __user *optval, int __user *optlen)
3248 {
3249 struct sock *sk = sock->sk;
3250
3251 /* IPV6_ADDRFORM can change sk->sk_prot under us. */
3252 return READ_ONCE(sk->sk_prot)->getsockopt(sk, level, optname, optval, optlen);
3253 }
3254 EXPORT_SYMBOL(sock_common_getsockopt);
3255
sock_common_recvmsg(struct socket * sock,struct msghdr * msg,size_t size,int flags)3256 int sock_common_recvmsg(struct socket *sock, struct msghdr *msg, size_t size,
3257 int flags)
3258 {
3259 struct sock *sk = sock->sk;
3260 int addr_len = 0;
3261 int err;
3262
3263 err = sk->sk_prot->recvmsg(sk, msg, size, flags & MSG_DONTWAIT,
3264 flags & ~MSG_DONTWAIT, &addr_len);
3265 if (err >= 0)
3266 msg->msg_namelen = addr_len;
3267 return err;
3268 }
3269 EXPORT_SYMBOL(sock_common_recvmsg);
3270
3271 /*
3272 * Set socket options on an inet socket.
3273 */
sock_common_setsockopt(struct socket * sock,int level,int optname,sockptr_t optval,unsigned int optlen)3274 int sock_common_setsockopt(struct socket *sock, int level, int optname,
3275 sockptr_t optval, unsigned int optlen)
3276 {
3277 struct sock *sk = sock->sk;
3278
3279 /* IPV6_ADDRFORM can change sk->sk_prot under us. */
3280 return READ_ONCE(sk->sk_prot)->setsockopt(sk, level, optname, optval, optlen);
3281 }
3282 EXPORT_SYMBOL(sock_common_setsockopt);
3283
sk_common_release(struct sock * sk)3284 void sk_common_release(struct sock *sk)
3285 {
3286 if (sk->sk_prot->destroy)
3287 sk->sk_prot->destroy(sk);
3288
3289 /*
3290 * Observation: when sk_common_release is called, processes have
3291 * no access to socket. But net still has.
3292 * Step one, detach it from networking:
3293 *
3294 * A. Remove from hash tables.
3295 */
3296
3297 sk->sk_prot->unhash(sk);
3298
3299 /*
3300 * In this point socket cannot receive new packets, but it is possible
3301 * that some packets are in flight because some CPU runs receiver and
3302 * did hash table lookup before we unhashed socket. They will achieve
3303 * receive queue and will be purged by socket destructor.
3304 *
3305 * Also we still have packets pending on receive queue and probably,
3306 * our own packets waiting in device queues. sock_destroy will drain
3307 * receive queue, but transmitted packets will delay socket destruction
3308 * until the last reference will be released.
3309 */
3310
3311 sock_orphan(sk);
3312
3313 xfrm_sk_free_policy(sk);
3314
3315 sk_refcnt_debug_release(sk);
3316
3317 sock_put(sk);
3318 }
3319 EXPORT_SYMBOL(sk_common_release);
3320
sk_get_meminfo(const struct sock * sk,u32 * mem)3321 void sk_get_meminfo(const struct sock *sk, u32 *mem)
3322 {
3323 memset(mem, 0, sizeof(*mem) * SK_MEMINFO_VARS);
3324
3325 mem[SK_MEMINFO_RMEM_ALLOC] = sk_rmem_alloc_get(sk);
3326 mem[SK_MEMINFO_RCVBUF] = READ_ONCE(sk->sk_rcvbuf);
3327 mem[SK_MEMINFO_WMEM_ALLOC] = sk_wmem_alloc_get(sk);
3328 mem[SK_MEMINFO_SNDBUF] = READ_ONCE(sk->sk_sndbuf);
3329 mem[SK_MEMINFO_FWD_ALLOC] = sk->sk_forward_alloc;
3330 mem[SK_MEMINFO_WMEM_QUEUED] = READ_ONCE(sk->sk_wmem_queued);
3331 mem[SK_MEMINFO_OPTMEM] = atomic_read(&sk->sk_omem_alloc);
3332 mem[SK_MEMINFO_BACKLOG] = READ_ONCE(sk->sk_backlog.len);
3333 mem[SK_MEMINFO_DROPS] = atomic_read(&sk->sk_drops);
3334 }
3335
3336 #ifdef CONFIG_PROC_FS
3337 #define PROTO_INUSE_NR 64 /* should be enough for the first time */
3338 struct prot_inuse {
3339 int val[PROTO_INUSE_NR];
3340 };
3341
3342 static DECLARE_BITMAP(proto_inuse_idx, PROTO_INUSE_NR);
3343
sock_prot_inuse_add(struct net * net,struct proto * prot,int val)3344 void sock_prot_inuse_add(struct net *net, struct proto *prot, int val)
3345 {
3346 __this_cpu_add(net->core.prot_inuse->val[prot->inuse_idx], val);
3347 }
3348 EXPORT_SYMBOL_GPL(sock_prot_inuse_add);
3349
sock_prot_inuse_get(struct net * net,struct proto * prot)3350 int sock_prot_inuse_get(struct net *net, struct proto *prot)
3351 {
3352 int cpu, idx = prot->inuse_idx;
3353 int res = 0;
3354
3355 for_each_possible_cpu(cpu)
3356 res += per_cpu_ptr(net->core.prot_inuse, cpu)->val[idx];
3357
3358 return res >= 0 ? res : 0;
3359 }
3360 EXPORT_SYMBOL_GPL(sock_prot_inuse_get);
3361
sock_inuse_add(struct net * net,int val)3362 static void sock_inuse_add(struct net *net, int val)
3363 {
3364 this_cpu_add(*net->core.sock_inuse, val);
3365 }
3366
sock_inuse_get(struct net * net)3367 int sock_inuse_get(struct net *net)
3368 {
3369 int cpu, res = 0;
3370
3371 for_each_possible_cpu(cpu)
3372 res += *per_cpu_ptr(net->core.sock_inuse, cpu);
3373
3374 return res;
3375 }
3376
3377 EXPORT_SYMBOL_GPL(sock_inuse_get);
3378
sock_inuse_init_net(struct net * net)3379 static int __net_init sock_inuse_init_net(struct net *net)
3380 {
3381 net->core.prot_inuse = alloc_percpu(struct prot_inuse);
3382 if (net->core.prot_inuse == NULL)
3383 return -ENOMEM;
3384
3385 net->core.sock_inuse = alloc_percpu(int);
3386 if (net->core.sock_inuse == NULL)
3387 goto out;
3388
3389 return 0;
3390
3391 out:
3392 free_percpu(net->core.prot_inuse);
3393 return -ENOMEM;
3394 }
3395
sock_inuse_exit_net(struct net * net)3396 static void __net_exit sock_inuse_exit_net(struct net *net)
3397 {
3398 free_percpu(net->core.prot_inuse);
3399 free_percpu(net->core.sock_inuse);
3400 }
3401
3402 static struct pernet_operations net_inuse_ops = {
3403 .init = sock_inuse_init_net,
3404 .exit = sock_inuse_exit_net,
3405 };
3406
net_inuse_init(void)3407 static __init int net_inuse_init(void)
3408 {
3409 if (register_pernet_subsys(&net_inuse_ops))
3410 panic("Cannot initialize net inuse counters");
3411
3412 return 0;
3413 }
3414
3415 core_initcall(net_inuse_init);
3416
assign_proto_idx(struct proto * prot)3417 static int assign_proto_idx(struct proto *prot)
3418 {
3419 prot->inuse_idx = find_first_zero_bit(proto_inuse_idx, PROTO_INUSE_NR);
3420
3421 if (unlikely(prot->inuse_idx == PROTO_INUSE_NR - 1)) {
3422 pr_err("PROTO_INUSE_NR exhausted\n");
3423 return -ENOSPC;
3424 }
3425
3426 set_bit(prot->inuse_idx, proto_inuse_idx);
3427 return 0;
3428 }
3429
release_proto_idx(struct proto * prot)3430 static void release_proto_idx(struct proto *prot)
3431 {
3432 if (prot->inuse_idx != PROTO_INUSE_NR - 1)
3433 clear_bit(prot->inuse_idx, proto_inuse_idx);
3434 }
3435 #else
assign_proto_idx(struct proto * prot)3436 static inline int assign_proto_idx(struct proto *prot)
3437 {
3438 return 0;
3439 }
3440
release_proto_idx(struct proto * prot)3441 static inline void release_proto_idx(struct proto *prot)
3442 {
3443 }
3444
sock_inuse_add(struct net * net,int val)3445 static void sock_inuse_add(struct net *net, int val)
3446 {
3447 }
3448 #endif
3449
tw_prot_cleanup(struct timewait_sock_ops * twsk_prot)3450 static void tw_prot_cleanup(struct timewait_sock_ops *twsk_prot)
3451 {
3452 if (!twsk_prot)
3453 return;
3454 kfree(twsk_prot->twsk_slab_name);
3455 twsk_prot->twsk_slab_name = NULL;
3456 kmem_cache_destroy(twsk_prot->twsk_slab);
3457 twsk_prot->twsk_slab = NULL;
3458 }
3459
req_prot_cleanup(struct request_sock_ops * rsk_prot)3460 static void req_prot_cleanup(struct request_sock_ops *rsk_prot)
3461 {
3462 if (!rsk_prot)
3463 return;
3464 kfree(rsk_prot->slab_name);
3465 rsk_prot->slab_name = NULL;
3466 kmem_cache_destroy(rsk_prot->slab);
3467 rsk_prot->slab = NULL;
3468 }
3469
req_prot_init(const struct proto * prot)3470 static int req_prot_init(const struct proto *prot)
3471 {
3472 struct request_sock_ops *rsk_prot = prot->rsk_prot;
3473
3474 if (!rsk_prot)
3475 return 0;
3476
3477 rsk_prot->slab_name = kasprintf(GFP_KERNEL, "request_sock_%s",
3478 prot->name);
3479 if (!rsk_prot->slab_name)
3480 return -ENOMEM;
3481
3482 rsk_prot->slab = kmem_cache_create(rsk_prot->slab_name,
3483 rsk_prot->obj_size, 0,
3484 SLAB_ACCOUNT | prot->slab_flags,
3485 NULL);
3486
3487 if (!rsk_prot->slab) {
3488 pr_crit("%s: Can't create request sock SLAB cache!\n",
3489 prot->name);
3490 return -ENOMEM;
3491 }
3492 return 0;
3493 }
3494
proto_register(struct proto * prot,int alloc_slab)3495 int proto_register(struct proto *prot, int alloc_slab)
3496 {
3497 int ret = -ENOBUFS;
3498
3499 if (alloc_slab) {
3500 prot->slab = kmem_cache_create_usercopy(prot->name,
3501 prot->obj_size, 0,
3502 SLAB_HWCACHE_ALIGN | SLAB_ACCOUNT |
3503 prot->slab_flags,
3504 prot->useroffset, prot->usersize,
3505 NULL);
3506
3507 if (prot->slab == NULL) {
3508 pr_crit("%s: Can't create sock SLAB cache!\n",
3509 prot->name);
3510 goto out;
3511 }
3512
3513 if (req_prot_init(prot))
3514 goto out_free_request_sock_slab;
3515
3516 if (prot->twsk_prot != NULL) {
3517 prot->twsk_prot->twsk_slab_name = kasprintf(GFP_KERNEL, "tw_sock_%s", prot->name);
3518
3519 if (prot->twsk_prot->twsk_slab_name == NULL)
3520 goto out_free_request_sock_slab;
3521
3522 prot->twsk_prot->twsk_slab =
3523 kmem_cache_create(prot->twsk_prot->twsk_slab_name,
3524 prot->twsk_prot->twsk_obj_size,
3525 0,
3526 SLAB_ACCOUNT |
3527 prot->slab_flags,
3528 NULL);
3529 if (prot->twsk_prot->twsk_slab == NULL)
3530 goto out_free_timewait_sock_slab;
3531 }
3532 }
3533
3534 mutex_lock(&proto_list_mutex);
3535 ret = assign_proto_idx(prot);
3536 if (ret) {
3537 mutex_unlock(&proto_list_mutex);
3538 goto out_free_timewait_sock_slab;
3539 }
3540 list_add(&prot->node, &proto_list);
3541 mutex_unlock(&proto_list_mutex);
3542 return ret;
3543
3544 out_free_timewait_sock_slab:
3545 if (alloc_slab && prot->twsk_prot)
3546 tw_prot_cleanup(prot->twsk_prot);
3547 out_free_request_sock_slab:
3548 if (alloc_slab) {
3549 req_prot_cleanup(prot->rsk_prot);
3550
3551 kmem_cache_destroy(prot->slab);
3552 prot->slab = NULL;
3553 }
3554 out:
3555 return ret;
3556 }
3557 EXPORT_SYMBOL(proto_register);
3558
proto_unregister(struct proto * prot)3559 void proto_unregister(struct proto *prot)
3560 {
3561 mutex_lock(&proto_list_mutex);
3562 release_proto_idx(prot);
3563 list_del(&prot->node);
3564 mutex_unlock(&proto_list_mutex);
3565
3566 kmem_cache_destroy(prot->slab);
3567 prot->slab = NULL;
3568
3569 req_prot_cleanup(prot->rsk_prot);
3570 tw_prot_cleanup(prot->twsk_prot);
3571 }
3572 EXPORT_SYMBOL(proto_unregister);
3573
sock_load_diag_module(int family,int protocol)3574 int sock_load_diag_module(int family, int protocol)
3575 {
3576 if (!protocol) {
3577 if (!sock_is_registered(family))
3578 return -ENOENT;
3579
3580 return request_module("net-pf-%d-proto-%d-type-%d", PF_NETLINK,
3581 NETLINK_SOCK_DIAG, family);
3582 }
3583
3584 #ifdef CONFIG_INET
3585 if (family == AF_INET &&
3586 protocol != IPPROTO_RAW &&
3587 protocol < MAX_INET_PROTOS &&
3588 !rcu_access_pointer(inet_protos[protocol]))
3589 return -ENOENT;
3590 #endif
3591
3592 return request_module("net-pf-%d-proto-%d-type-%d-%d", PF_NETLINK,
3593 NETLINK_SOCK_DIAG, family, protocol);
3594 }
3595 EXPORT_SYMBOL(sock_load_diag_module);
3596
3597 #ifdef CONFIG_PROC_FS
proto_seq_start(struct seq_file * seq,loff_t * pos)3598 static void *proto_seq_start(struct seq_file *seq, loff_t *pos)
3599 __acquires(proto_list_mutex)
3600 {
3601 mutex_lock(&proto_list_mutex);
3602 return seq_list_start_head(&proto_list, *pos);
3603 }
3604
proto_seq_next(struct seq_file * seq,void * v,loff_t * pos)3605 static void *proto_seq_next(struct seq_file *seq, void *v, loff_t *pos)
3606 {
3607 return seq_list_next(v, &proto_list, pos);
3608 }
3609
proto_seq_stop(struct seq_file * seq,void * v)3610 static void proto_seq_stop(struct seq_file *seq, void *v)
3611 __releases(proto_list_mutex)
3612 {
3613 mutex_unlock(&proto_list_mutex);
3614 }
3615
proto_method_implemented(const void * method)3616 static char proto_method_implemented(const void *method)
3617 {
3618 return method == NULL ? 'n' : 'y';
3619 }
sock_prot_memory_allocated(struct proto * proto)3620 static long sock_prot_memory_allocated(struct proto *proto)
3621 {
3622 return proto->memory_allocated != NULL ? proto_memory_allocated(proto) : -1L;
3623 }
3624
sock_prot_memory_pressure(struct proto * proto)3625 static const char *sock_prot_memory_pressure(struct proto *proto)
3626 {
3627 return proto->memory_pressure != NULL ?
3628 proto_memory_pressure(proto) ? "yes" : "no" : "NI";
3629 }
3630
proto_seq_printf(struct seq_file * seq,struct proto * proto)3631 static void proto_seq_printf(struct seq_file *seq, struct proto *proto)
3632 {
3633
3634 seq_printf(seq, "%-9s %4u %6d %6ld %-3s %6u %-3s %-10s "
3635 "%2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c\n",
3636 proto->name,
3637 proto->obj_size,
3638 sock_prot_inuse_get(seq_file_net(seq), proto),
3639 sock_prot_memory_allocated(proto),
3640 sock_prot_memory_pressure(proto),
3641 proto->max_header,
3642 proto->slab == NULL ? "no" : "yes",
3643 module_name(proto->owner),
3644 proto_method_implemented(proto->close),
3645 proto_method_implemented(proto->connect),
3646 proto_method_implemented(proto->disconnect),
3647 proto_method_implemented(proto->accept),
3648 proto_method_implemented(proto->ioctl),
3649 proto_method_implemented(proto->init),
3650 proto_method_implemented(proto->destroy),
3651 proto_method_implemented(proto->shutdown),
3652 proto_method_implemented(proto->setsockopt),
3653 proto_method_implemented(proto->getsockopt),
3654 proto_method_implemented(proto->sendmsg),
3655 proto_method_implemented(proto->recvmsg),
3656 proto_method_implemented(proto->sendpage),
3657 proto_method_implemented(proto->bind),
3658 proto_method_implemented(proto->backlog_rcv),
3659 proto_method_implemented(proto->hash),
3660 proto_method_implemented(proto->unhash),
3661 proto_method_implemented(proto->get_port),
3662 proto_method_implemented(proto->enter_memory_pressure));
3663 }
3664
proto_seq_show(struct seq_file * seq,void * v)3665 static int proto_seq_show(struct seq_file *seq, void *v)
3666 {
3667 if (v == &proto_list)
3668 seq_printf(seq, "%-9s %-4s %-8s %-6s %-5s %-7s %-4s %-10s %s",
3669 "protocol",
3670 "size",
3671 "sockets",
3672 "memory",
3673 "press",
3674 "maxhdr",
3675 "slab",
3676 "module",
3677 "cl co di ac io in de sh ss gs se re sp bi br ha uh gp em\n");
3678 else
3679 proto_seq_printf(seq, list_entry(v, struct proto, node));
3680 return 0;
3681 }
3682
3683 static const struct seq_operations proto_seq_ops = {
3684 .start = proto_seq_start,
3685 .next = proto_seq_next,
3686 .stop = proto_seq_stop,
3687 .show = proto_seq_show,
3688 };
3689
proto_init_net(struct net * net)3690 static __net_init int proto_init_net(struct net *net)
3691 {
3692 if (!proc_create_net("protocols", 0444, net->proc_net, &proto_seq_ops,
3693 sizeof(struct seq_net_private)))
3694 return -ENOMEM;
3695
3696 return 0;
3697 }
3698
proto_exit_net(struct net * net)3699 static __net_exit void proto_exit_net(struct net *net)
3700 {
3701 remove_proc_entry("protocols", net->proc_net);
3702 }
3703
3704
3705 static __net_initdata struct pernet_operations proto_net_ops = {
3706 .init = proto_init_net,
3707 .exit = proto_exit_net,
3708 };
3709
proto_init(void)3710 static int __init proto_init(void)
3711 {
3712 return register_pernet_subsys(&proto_net_ops);
3713 }
3714
3715 subsys_initcall(proto_init);
3716
3717 #endif /* PROC_FS */
3718
3719 #ifdef CONFIG_NET_RX_BUSY_POLL
sk_busy_loop_end(void * p,unsigned long start_time)3720 bool sk_busy_loop_end(void *p, unsigned long start_time)
3721 {
3722 struct sock *sk = p;
3723
3724 return !skb_queue_empty_lockless(&sk->sk_receive_queue) ||
3725 sk_busy_loop_timeout(sk, start_time);
3726 }
3727 EXPORT_SYMBOL(sk_busy_loop_end);
3728 #endif /* CONFIG_NET_RX_BUSY_POLL */
3729
sock_bind_add(struct sock * sk,struct sockaddr * addr,int addr_len)3730 int sock_bind_add(struct sock *sk, struct sockaddr *addr, int addr_len)
3731 {
3732 if (!sk->sk_prot->bind_add)
3733 return -EOPNOTSUPP;
3734 return sk->sk_prot->bind_add(sk, addr, addr_len);
3735 }
3736 EXPORT_SYMBOL(sock_bind_add);
3737