1 /* SPDX-License-Identifier: GPL-2.0-or-later */
2 /*
3 * INET An implementation of the TCP/IP protocol suite for the LINUX
4 * operating system. INET is implemented using the BSD Socket
5 * interface as the means of communication with the user level.
6 *
7 * Definitions for the AF_INET socket handler.
8 *
9 * Version: @(#)sock.h 1.0.4 05/13/93
10 *
11 * Authors: Ross Biro
12 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
13 * Corey Minyard <wf-rch!minyard@relay.EU.net>
14 * Florian La Roche <flla@stud.uni-sb.de>
15 *
16 * Fixes:
17 * Alan Cox : Volatiles in skbuff pointers. See
18 * skbuff comments. May be overdone,
19 * better to prove they can be removed
20 * than the reverse.
21 * Alan Cox : Added a zapped field for tcp to note
22 * a socket is reset and must stay shut up
23 * Alan Cox : New fields for options
24 * Pauline Middelink : identd support
25 * Alan Cox : Eliminate low level recv/recvfrom
26 * David S. Miller : New socket lookup architecture.
27 * Steve Whitehouse: Default routines for sock_ops
28 * Arnaldo C. Melo : removed net_pinfo, tp_pinfo and made
29 * protinfo be just a void pointer, as the
30 * protocol specific parts were moved to
31 * respective headers and ipv4/v6, etc now
32 * use private slabcaches for its socks
33 * Pedro Hortas : New flags field for socket options
34 */
35 #ifndef _SOCK_H
36 #define _SOCK_H
37
38 #include <linux/hardirq.h>
39 #include <linux/kernel.h>
40 #include <linux/list.h>
41 #include <linux/list_nulls.h>
42 #include <linux/timer.h>
43 #include <linux/cache.h>
44 #include <linux/bitops.h>
45 #include <linux/lockdep.h>
46 #include <linux/netdevice.h>
47 #include <linux/skbuff.h> /* struct sk_buff */
48 #include <linux/mm.h>
49 #include <linux/security.h>
50 #include <linux/slab.h>
51 #include <linux/uaccess.h>
52 #include <linux/page_counter.h>
53 #include <linux/memcontrol.h>
54 #include <linux/static_key.h>
55 #include <linux/sched.h>
56 #include <linux/wait.h>
57 #include <linux/cgroup-defs.h>
58 #include <linux/rbtree.h>
59 #include <linux/filter.h>
60 #include <linux/rculist_nulls.h>
61 #include <linux/poll.h>
62
63 #include <linux/atomic.h>
64 #include <linux/refcount.h>
65 #include <net/dst.h>
66 #include <net/checksum.h>
67 #include <net/tcp_states.h>
68 #include <linux/net_tstamp.h>
69 #include <net/smc.h>
70 #include <net/l3mdev.h>
71 #include <linux/android_kabi.h>
72 #include <linux/android_vendor.h>
73
74 /*
75 * This structure really needs to be cleaned up.
76 * Most of it is for TCP, and not used by any of
77 * the other protocols.
78 */
79
80 /* Define this to get the SOCK_DBG debugging facility. */
81 #define SOCK_DEBUGGING
82 #ifdef SOCK_DEBUGGING
83 #define SOCK_DEBUG(sk, msg...) do { if ((sk) && sock_flag((sk), SOCK_DBG)) \
84 printk(KERN_DEBUG msg); } while (0)
85 #else
86 /* Validate arguments and do nothing */
87 static inline __printf(2, 3)
SOCK_DEBUG(const struct sock * sk,const char * msg,...)88 void SOCK_DEBUG(const struct sock *sk, const char *msg, ...)
89 {
90 }
91 #endif
92
93 /* This is the per-socket lock. The spinlock provides a synchronization
94 * between user contexts and software interrupt processing, whereas the
95 * mini-semaphore synchronizes multiple users amongst themselves.
96 */
97 typedef struct {
98 spinlock_t slock;
99 int owned;
100 wait_queue_head_t wq;
101 /*
102 * We express the mutex-alike socket_lock semantics
103 * to the lock validator by explicitly managing
104 * the slock as a lock variant (in addition to
105 * the slock itself):
106 */
107 #ifdef CONFIG_DEBUG_LOCK_ALLOC
108 struct lockdep_map dep_map;
109 #endif
110 } socket_lock_t;
111
112 struct sock;
113 struct proto;
114 struct net;
115
116 typedef __u32 __bitwise __portpair;
117 typedef __u64 __bitwise __addrpair;
118
119 /**
120 * struct sock_common - minimal network layer representation of sockets
121 * @skc_daddr: Foreign IPv4 addr
122 * @skc_rcv_saddr: Bound local IPv4 addr
123 * @skc_hash: hash value used with various protocol lookup tables
124 * @skc_u16hashes: two u16 hash values used by UDP lookup tables
125 * @skc_dport: placeholder for inet_dport/tw_dport
126 * @skc_num: placeholder for inet_num/tw_num
127 * @skc_family: network address family
128 * @skc_state: Connection state
129 * @skc_reuse: %SO_REUSEADDR setting
130 * @skc_reuseport: %SO_REUSEPORT setting
131 * @skc_bound_dev_if: bound device index if != 0
132 * @skc_bind_node: bind hash linkage for various protocol lookup tables
133 * @skc_portaddr_node: second hash linkage for UDP/UDP-Lite protocol
134 * @skc_prot: protocol handlers inside a network family
135 * @skc_net: reference to the network namespace of this socket
136 * @skc_node: main hash linkage for various protocol lookup tables
137 * @skc_nulls_node: main hash linkage for TCP/UDP/UDP-Lite protocol
138 * @skc_tx_queue_mapping: tx queue number for this connection
139 * @skc_rx_queue_mapping: rx queue number for this connection
140 * @skc_flags: place holder for sk_flags
141 * %SO_LINGER (l_onoff), %SO_BROADCAST, %SO_KEEPALIVE,
142 * %SO_OOBINLINE settings, %SO_TIMESTAMPING settings
143 * @skc_incoming_cpu: record/match cpu processing incoming packets
144 * @skc_refcnt: reference count
145 *
146 * This is the minimal network layer representation of sockets, the header
147 * for struct sock and struct inet_timewait_sock.
148 */
149 struct sock_common {
150 /* skc_daddr and skc_rcv_saddr must be grouped on a 8 bytes aligned
151 * address on 64bit arches : cf INET_MATCH()
152 */
153 union {
154 __addrpair skc_addrpair;
155 struct {
156 __be32 skc_daddr;
157 __be32 skc_rcv_saddr;
158 };
159 };
160 union {
161 unsigned int skc_hash;
162 __u16 skc_u16hashes[2];
163 };
164 /* skc_dport && skc_num must be grouped as well */
165 union {
166 __portpair skc_portpair;
167 struct {
168 __be16 skc_dport;
169 __u16 skc_num;
170 };
171 };
172
173 unsigned short skc_family;
174 volatile unsigned char skc_state;
175 unsigned char skc_reuse:4;
176 unsigned char skc_reuseport:1;
177 unsigned char skc_ipv6only:1;
178 unsigned char skc_net_refcnt:1;
179 int skc_bound_dev_if;
180 union {
181 struct hlist_node skc_bind_node;
182 struct hlist_node skc_portaddr_node;
183 };
184 struct proto *skc_prot;
185 possible_net_t skc_net;
186
187 #if IS_ENABLED(CONFIG_IPV6)
188 struct in6_addr skc_v6_daddr;
189 struct in6_addr skc_v6_rcv_saddr;
190 #endif
191
192 atomic64_t skc_cookie;
193
194 /* following fields are padding to force
195 * offset(struct sock, sk_refcnt) == 128 on 64bit arches
196 * assuming IPV6 is enabled. We use this padding differently
197 * for different kind of 'sockets'
198 */
199 union {
200 unsigned long skc_flags;
201 struct sock *skc_listener; /* request_sock */
202 struct inet_timewait_death_row *skc_tw_dr; /* inet_timewait_sock */
203 };
204 /*
205 * fields between dontcopy_begin/dontcopy_end
206 * are not copied in sock_copy()
207 */
208 /* private: */
209 int skc_dontcopy_begin[0];
210 /* public: */
211 union {
212 struct hlist_node skc_node;
213 struct hlist_nulls_node skc_nulls_node;
214 };
215 unsigned short skc_tx_queue_mapping;
216 #ifdef CONFIG_XPS
217 unsigned short skc_rx_queue_mapping;
218 #endif
219 union {
220 int skc_incoming_cpu;
221 u32 skc_rcv_wnd;
222 u32 skc_tw_rcv_nxt; /* struct tcp_timewait_sock */
223 };
224
225 refcount_t skc_refcnt;
226 /* private: */
227 int skc_dontcopy_end[0];
228 union {
229 u32 skc_rxhash;
230 u32 skc_window_clamp;
231 u32 skc_tw_snd_nxt; /* struct tcp_timewait_sock */
232 };
233 /* public: */
234 };
235
236 struct bpf_sk_storage;
237
238 /**
239 * struct sock - network layer representation of sockets
240 * @__sk_common: shared layout with inet_timewait_sock
241 * @sk_shutdown: mask of %SEND_SHUTDOWN and/or %RCV_SHUTDOWN
242 * @sk_userlocks: %SO_SNDBUF and %SO_RCVBUF settings
243 * @sk_lock: synchronizer
244 * @sk_kern_sock: True if sock is using kernel lock classes
245 * @sk_rcvbuf: size of receive buffer in bytes
246 * @sk_wq: sock wait queue and async head
247 * @sk_rx_dst: receive input route used by early demux
248 * @sk_dst_cache: destination cache
249 * @sk_dst_pending_confirm: need to confirm neighbour
250 * @sk_policy: flow policy
251 * @sk_receive_queue: incoming packets
252 * @sk_wmem_alloc: transmit queue bytes committed
253 * @sk_tsq_flags: TCP Small Queues flags
254 * @sk_write_queue: Packet sending queue
255 * @sk_omem_alloc: "o" is "option" or "other"
256 * @sk_wmem_queued: persistent queue size
257 * @sk_forward_alloc: space allocated forward
258 * @sk_napi_id: id of the last napi context to receive data for sk
259 * @sk_ll_usec: usecs to busypoll when there is no data
260 * @sk_allocation: allocation mode
261 * @sk_pacing_rate: Pacing rate (if supported by transport/packet scheduler)
262 * @sk_pacing_status: Pacing status (requested, handled by sch_fq)
263 * @sk_max_pacing_rate: Maximum pacing rate (%SO_MAX_PACING_RATE)
264 * @sk_sndbuf: size of send buffer in bytes
265 * @__sk_flags_offset: empty field used to determine location of bitfield
266 * @sk_padding: unused element for alignment
267 * @sk_no_check_tx: %SO_NO_CHECK setting, set checksum in TX packets
268 * @sk_no_check_rx: allow zero checksum in RX packets
269 * @sk_route_caps: route capabilities (e.g. %NETIF_F_TSO)
270 * @sk_route_nocaps: forbidden route capabilities (e.g NETIF_F_GSO_MASK)
271 * @sk_gso_type: GSO type (e.g. %SKB_GSO_TCPV4)
272 * @sk_gso_max_size: Maximum GSO segment size to build
273 * @sk_gso_max_segs: Maximum number of GSO segments
274 * @sk_pacing_shift: scaling factor for TCP Small Queues
275 * @sk_lingertime: %SO_LINGER l_linger setting
276 * @sk_backlog: always used with the per-socket spinlock held
277 * @sk_callback_lock: used with the callbacks in the end of this struct
278 * @sk_error_queue: rarely used
279 * @sk_prot_creator: sk_prot of original sock creator (see ipv6_setsockopt,
280 * IPV6_ADDRFORM for instance)
281 * @sk_err: last error
282 * @sk_err_soft: errors that don't cause failure but are the cause of a
283 * persistent failure not just 'timed out'
284 * @sk_drops: raw/udp drops counter
285 * @sk_ack_backlog: current listen backlog
286 * @sk_max_ack_backlog: listen backlog set in listen()
287 * @sk_uid: user id of owner
288 * @sk_priority: %SO_PRIORITY setting
289 * @sk_type: socket type (%SOCK_STREAM, etc)
290 * @sk_protocol: which protocol this socket belongs in this network family
291 * @sk_peer_pid: &struct pid for this socket's peer
292 * @sk_peer_cred: %SO_PEERCRED setting
293 * @sk_rcvlowat: %SO_RCVLOWAT setting
294 * @sk_rcvtimeo: %SO_RCVTIMEO setting
295 * @sk_sndtimeo: %SO_SNDTIMEO setting
296 * @sk_txhash: computed flow hash for use on transmit
297 * @sk_filter: socket filtering instructions
298 * @sk_timer: sock cleanup timer
299 * @sk_stamp: time stamp of last packet received
300 * @sk_stamp_seq: lock for accessing sk_stamp on 32 bit architectures only
301 * @sk_tsflags: SO_TIMESTAMPING socket options
302 * @sk_tskey: counter to disambiguate concurrent tstamp requests
303 * @sk_zckey: counter to order MSG_ZEROCOPY notifications
304 * @sk_socket: Identd and reporting IO signals
305 * @sk_user_data: RPC layer private data. Write-protected by @sk_callback_lock.
306 * @sk_frag: cached page frag
307 * @sk_peek_off: current peek_offset value
308 * @sk_send_head: front of stuff to transmit
309 * @sk_security: used by security modules
310 * @sk_mark: generic packet mark
311 * @sk_cgrp_data: cgroup data for this cgroup
312 * @sk_memcg: this socket's memory cgroup association
313 * @sk_write_pending: a write to stream socket waits to start
314 * @sk_state_change: callback to indicate change in the state of the sock
315 * @sk_data_ready: callback to indicate there is data to be processed
316 * @sk_write_space: callback to indicate there is bf sending space available
317 * @sk_error_report: callback to indicate errors (e.g. %MSG_ERRQUEUE)
318 * @sk_backlog_rcv: callback to process the backlog
319 * @sk_destruct: called at sock freeing time, i.e. when all refcnt == 0
320 * @sk_reuseport_cb: reuseport group container
321 * @sk_rcu: used during RCU grace period
322 * @sk_clockid: clockid used by time-based scheduling (SO_TXTIME)
323 * @sk_txtime_deadline_mode: set deadline mode for SO_TXTIME
324 * @sk_txtime_unused: unused txtime flags
325 */
326 struct sock {
327 /*
328 * Now struct inet_timewait_sock also uses sock_common, so please just
329 * don't add nothing before this first member (__sk_common) --acme
330 */
331 struct sock_common __sk_common;
332 #define sk_node __sk_common.skc_node
333 #define sk_nulls_node __sk_common.skc_nulls_node
334 #define sk_refcnt __sk_common.skc_refcnt
335 #define sk_tx_queue_mapping __sk_common.skc_tx_queue_mapping
336 #ifdef CONFIG_XPS
337 #define sk_rx_queue_mapping __sk_common.skc_rx_queue_mapping
338 #endif
339
340 #define sk_dontcopy_begin __sk_common.skc_dontcopy_begin
341 #define sk_dontcopy_end __sk_common.skc_dontcopy_end
342 #define sk_hash __sk_common.skc_hash
343 #define sk_portpair __sk_common.skc_portpair
344 #define sk_num __sk_common.skc_num
345 #define sk_dport __sk_common.skc_dport
346 #define sk_addrpair __sk_common.skc_addrpair
347 #define sk_daddr __sk_common.skc_daddr
348 #define sk_rcv_saddr __sk_common.skc_rcv_saddr
349 #define sk_family __sk_common.skc_family
350 #define sk_state __sk_common.skc_state
351 #define sk_reuse __sk_common.skc_reuse
352 #define sk_reuseport __sk_common.skc_reuseport
353 #define sk_ipv6only __sk_common.skc_ipv6only
354 #define sk_net_refcnt __sk_common.skc_net_refcnt
355 #define sk_bound_dev_if __sk_common.skc_bound_dev_if
356 #define sk_bind_node __sk_common.skc_bind_node
357 #define sk_prot __sk_common.skc_prot
358 #define sk_net __sk_common.skc_net
359 #define sk_v6_daddr __sk_common.skc_v6_daddr
360 #define sk_v6_rcv_saddr __sk_common.skc_v6_rcv_saddr
361 #define sk_cookie __sk_common.skc_cookie
362 #define sk_incoming_cpu __sk_common.skc_incoming_cpu
363 #define sk_flags __sk_common.skc_flags
364 #define sk_rxhash __sk_common.skc_rxhash
365
366 socket_lock_t sk_lock;
367 atomic_t sk_drops;
368 int sk_rcvlowat;
369 struct sk_buff_head sk_error_queue;
370 struct sk_buff *sk_rx_skb_cache;
371 struct sk_buff_head sk_receive_queue;
372 /*
373 * The backlog queue is special, it is always used with
374 * the per-socket spinlock held and requires low latency
375 * access. Therefore we special case it's implementation.
376 * Note : rmem_alloc is in this structure to fill a hole
377 * on 64bit arches, not because its logically part of
378 * backlog.
379 */
380 struct {
381 atomic_t rmem_alloc;
382 int len;
383 struct sk_buff *head;
384 struct sk_buff *tail;
385 } sk_backlog;
386 #define sk_rmem_alloc sk_backlog.rmem_alloc
387
388 int sk_forward_alloc;
389 #ifdef CONFIG_NET_RX_BUSY_POLL
390 unsigned int sk_ll_usec;
391 /* ===== mostly read cache line ===== */
392 unsigned int sk_napi_id;
393 #endif
394 int sk_rcvbuf;
395
396 struct sk_filter __rcu *sk_filter;
397 union {
398 struct socket_wq __rcu *sk_wq;
399 struct socket_wq *sk_wq_raw;
400 };
401 #ifdef CONFIG_XFRM
402 struct xfrm_policy __rcu *sk_policy[2];
403 #endif
404 struct dst_entry __rcu *sk_rx_dst;
405 struct dst_entry __rcu *sk_dst_cache;
406 atomic_t sk_omem_alloc;
407 int sk_sndbuf;
408
409 /* ===== cache line for TX ===== */
410 int sk_wmem_queued;
411 refcount_t sk_wmem_alloc;
412 unsigned long sk_tsq_flags;
413 union {
414 struct sk_buff *sk_send_head;
415 struct rb_root tcp_rtx_queue;
416 };
417 struct sk_buff *sk_tx_skb_cache;
418 struct sk_buff_head sk_write_queue;
419 __s32 sk_peek_off;
420 int sk_write_pending;
421 __u32 sk_dst_pending_confirm;
422 u32 sk_pacing_status; /* see enum sk_pacing */
423 long sk_sndtimeo;
424 struct timer_list sk_timer;
425 __u32 sk_priority;
426 __u32 sk_mark;
427 unsigned long sk_pacing_rate; /* bytes per second */
428 unsigned long sk_max_pacing_rate;
429 struct page_frag sk_frag;
430 netdev_features_t sk_route_caps;
431 netdev_features_t sk_route_nocaps;
432 netdev_features_t sk_route_forced_caps;
433 int sk_gso_type;
434 unsigned int sk_gso_max_size;
435 gfp_t sk_allocation;
436 __u32 sk_txhash;
437
438 /*
439 * Because of non atomicity rules, all
440 * changes are protected by socket lock.
441 */
442 unsigned int __sk_flags_offset[0];
443 #ifdef __BIG_ENDIAN_BITFIELD
444 #define SK_FL_PROTO_SHIFT 16
445 #define SK_FL_PROTO_MASK 0x00ff0000
446
447 #define SK_FL_TYPE_SHIFT 0
448 #define SK_FL_TYPE_MASK 0x0000ffff
449 #else
450 #define SK_FL_PROTO_SHIFT 8
451 #define SK_FL_PROTO_MASK 0x0000ff00
452
453 #define SK_FL_TYPE_SHIFT 16
454 #define SK_FL_TYPE_MASK 0xffff0000
455 #endif
456
457 unsigned int sk_padding : 1,
458 sk_kern_sock : 1,
459 sk_no_check_tx : 1,
460 sk_no_check_rx : 1,
461 sk_userlocks : 4,
462 sk_protocol : 8,
463 sk_type : 16;
464 #define SK_PROTOCOL_MAX U8_MAX
465 u16 sk_gso_max_segs;
466 u8 sk_pacing_shift;
467 unsigned long sk_lingertime;
468 struct proto *sk_prot_creator;
469 rwlock_t sk_callback_lock;
470 int sk_err,
471 sk_err_soft;
472 u32 sk_ack_backlog;
473 u32 sk_max_ack_backlog;
474 kuid_t sk_uid;
475 #if IS_ENABLED(CONFIG_DEBUG_SPINLOCK) || IS_ENABLED(CONFIG_DEBUG_LOCK_ALLOC)
476 spinlock_t sk_peer_lock;
477 #else
478 /* sk_peer_lock is in the ANDROID_KABI_RESERVE(1) field below */
479 #endif
480 struct pid *sk_peer_pid;
481 const struct cred *sk_peer_cred;
482
483 long sk_rcvtimeo;
484 ktime_t sk_stamp;
485 #if BITS_PER_LONG==32
486 seqlock_t sk_stamp_seq;
487 #endif
488 u16 sk_tsflags;
489 u8 sk_shutdown;
490 u32 sk_tskey;
491 atomic_t sk_zckey;
492
493 u8 sk_clockid;
494 u8 sk_txtime_deadline_mode : 1,
495 sk_txtime_report_errors : 1,
496 sk_txtime_unused : 6;
497
498 struct socket *sk_socket;
499 void *sk_user_data;
500 #ifdef CONFIG_SECURITY
501 void *sk_security;
502 #endif
503 struct sock_cgroup_data sk_cgrp_data;
504 struct mem_cgroup *sk_memcg;
505 void (*sk_state_change)(struct sock *sk);
506 void (*sk_data_ready)(struct sock *sk);
507 void (*sk_write_space)(struct sock *sk);
508 void (*sk_error_report)(struct sock *sk);
509 int (*sk_backlog_rcv)(struct sock *sk,
510 struct sk_buff *skb);
511 #ifdef CONFIG_SOCK_VALIDATE_XMIT
512 struct sk_buff* (*sk_validate_xmit_skb)(struct sock *sk,
513 struct net_device *dev,
514 struct sk_buff *skb);
515 #endif
516 void (*sk_destruct)(struct sock *sk);
517 struct sock_reuseport __rcu *sk_reuseport_cb;
518 #ifdef CONFIG_BPF_SYSCALL
519 struct bpf_sk_storage __rcu *sk_bpf_storage;
520 #endif
521 struct rcu_head sk_rcu;
522
523 #if IS_ENABLED(CONFIG_DEBUG_SPINLOCK) || IS_ENABLED(CONFIG_DEBUG_LOCK_ALLOC)
524 ANDROID_KABI_RESERVE(1);
525 #else
526 ANDROID_KABI_USE(1, spinlock_t sk_peer_lock);
527 #endif
528 ANDROID_KABI_RESERVE(2);
529 ANDROID_KABI_RESERVE(3);
530 ANDROID_KABI_RESERVE(4);
531 ANDROID_KABI_RESERVE(5);
532 ANDROID_KABI_RESERVE(6);
533 ANDROID_KABI_RESERVE(7);
534 ANDROID_KABI_RESERVE(8);
535
536 ANDROID_VENDOR_DATA(1);
537 };
538
539 enum sk_pacing {
540 SK_PACING_NONE = 0,
541 SK_PACING_NEEDED = 1,
542 SK_PACING_FQ = 2,
543 };
544
545 #define __sk_user_data(sk) ((*((void __rcu **)&(sk)->sk_user_data)))
546
547 #define rcu_dereference_sk_user_data(sk) rcu_dereference(__sk_user_data((sk)))
548 #define rcu_assign_sk_user_data(sk, ptr) rcu_assign_pointer(__sk_user_data((sk)), ptr)
549
550 /*
551 * SK_CAN_REUSE and SK_NO_REUSE on a socket mean that the socket is OK
552 * or not whether his port will be reused by someone else. SK_FORCE_REUSE
553 * on a socket means that the socket will reuse everybody else's port
554 * without looking at the other's sk_reuse value.
555 */
556
557 #define SK_NO_REUSE 0
558 #define SK_CAN_REUSE 1
559 #define SK_FORCE_REUSE 2
560
561 int sk_set_peek_off(struct sock *sk, int val);
562
sk_peek_offset(struct sock * sk,int flags)563 static inline int sk_peek_offset(struct sock *sk, int flags)
564 {
565 if (unlikely(flags & MSG_PEEK)) {
566 return READ_ONCE(sk->sk_peek_off);
567 }
568
569 return 0;
570 }
571
sk_peek_offset_bwd(struct sock * sk,int val)572 static inline void sk_peek_offset_bwd(struct sock *sk, int val)
573 {
574 s32 off = READ_ONCE(sk->sk_peek_off);
575
576 if (unlikely(off >= 0)) {
577 off = max_t(s32, off - val, 0);
578 WRITE_ONCE(sk->sk_peek_off, off);
579 }
580 }
581
sk_peek_offset_fwd(struct sock * sk,int val)582 static inline void sk_peek_offset_fwd(struct sock *sk, int val)
583 {
584 sk_peek_offset_bwd(sk, -val);
585 }
586
587 /*
588 * Hashed lists helper routines
589 */
sk_entry(const struct hlist_node * node)590 static inline struct sock *sk_entry(const struct hlist_node *node)
591 {
592 return hlist_entry(node, struct sock, sk_node);
593 }
594
__sk_head(const struct hlist_head * head)595 static inline struct sock *__sk_head(const struct hlist_head *head)
596 {
597 return hlist_entry(head->first, struct sock, sk_node);
598 }
599
sk_head(const struct hlist_head * head)600 static inline struct sock *sk_head(const struct hlist_head *head)
601 {
602 return hlist_empty(head) ? NULL : __sk_head(head);
603 }
604
__sk_nulls_head(const struct hlist_nulls_head * head)605 static inline struct sock *__sk_nulls_head(const struct hlist_nulls_head *head)
606 {
607 return hlist_nulls_entry(head->first, struct sock, sk_nulls_node);
608 }
609
sk_nulls_head(const struct hlist_nulls_head * head)610 static inline struct sock *sk_nulls_head(const struct hlist_nulls_head *head)
611 {
612 return hlist_nulls_empty(head) ? NULL : __sk_nulls_head(head);
613 }
614
sk_next(const struct sock * sk)615 static inline struct sock *sk_next(const struct sock *sk)
616 {
617 return hlist_entry_safe(sk->sk_node.next, struct sock, sk_node);
618 }
619
sk_nulls_next(const struct sock * sk)620 static inline struct sock *sk_nulls_next(const struct sock *sk)
621 {
622 return (!is_a_nulls(sk->sk_nulls_node.next)) ?
623 hlist_nulls_entry(sk->sk_nulls_node.next,
624 struct sock, sk_nulls_node) :
625 NULL;
626 }
627
sk_unhashed(const struct sock * sk)628 static inline bool sk_unhashed(const struct sock *sk)
629 {
630 return hlist_unhashed(&sk->sk_node);
631 }
632
sk_hashed(const struct sock * sk)633 static inline bool sk_hashed(const struct sock *sk)
634 {
635 return !sk_unhashed(sk);
636 }
637
sk_node_init(struct hlist_node * node)638 static inline void sk_node_init(struct hlist_node *node)
639 {
640 node->pprev = NULL;
641 }
642
sk_nulls_node_init(struct hlist_nulls_node * node)643 static inline void sk_nulls_node_init(struct hlist_nulls_node *node)
644 {
645 node->pprev = NULL;
646 }
647
__sk_del_node(struct sock * sk)648 static inline void __sk_del_node(struct sock *sk)
649 {
650 __hlist_del(&sk->sk_node);
651 }
652
653 /* NB: equivalent to hlist_del_init_rcu */
__sk_del_node_init(struct sock * sk)654 static inline bool __sk_del_node_init(struct sock *sk)
655 {
656 if (sk_hashed(sk)) {
657 __sk_del_node(sk);
658 sk_node_init(&sk->sk_node);
659 return true;
660 }
661 return false;
662 }
663
664 /* Grab socket reference count. This operation is valid only
665 when sk is ALREADY grabbed f.e. it is found in hash table
666 or a list and the lookup is made under lock preventing hash table
667 modifications.
668 */
669
sock_hold(struct sock * sk)670 static __always_inline void sock_hold(struct sock *sk)
671 {
672 refcount_inc(&sk->sk_refcnt);
673 }
674
675 /* Ungrab socket in the context, which assumes that socket refcnt
676 cannot hit zero, f.e. it is true in context of any socketcall.
677 */
__sock_put(struct sock * sk)678 static __always_inline void __sock_put(struct sock *sk)
679 {
680 refcount_dec(&sk->sk_refcnt);
681 }
682
sk_del_node_init(struct sock * sk)683 static inline bool sk_del_node_init(struct sock *sk)
684 {
685 bool rc = __sk_del_node_init(sk);
686
687 if (rc) {
688 /* paranoid for a while -acme */
689 WARN_ON(refcount_read(&sk->sk_refcnt) == 1);
690 __sock_put(sk);
691 }
692 return rc;
693 }
694 #define sk_del_node_init_rcu(sk) sk_del_node_init(sk)
695
__sk_nulls_del_node_init_rcu(struct sock * sk)696 static inline bool __sk_nulls_del_node_init_rcu(struct sock *sk)
697 {
698 if (sk_hashed(sk)) {
699 hlist_nulls_del_init_rcu(&sk->sk_nulls_node);
700 return true;
701 }
702 return false;
703 }
704
sk_nulls_del_node_init_rcu(struct sock * sk)705 static inline bool sk_nulls_del_node_init_rcu(struct sock *sk)
706 {
707 bool rc = __sk_nulls_del_node_init_rcu(sk);
708
709 if (rc) {
710 /* paranoid for a while -acme */
711 WARN_ON(refcount_read(&sk->sk_refcnt) == 1);
712 __sock_put(sk);
713 }
714 return rc;
715 }
716
__sk_add_node(struct sock * sk,struct hlist_head * list)717 static inline void __sk_add_node(struct sock *sk, struct hlist_head *list)
718 {
719 hlist_add_head(&sk->sk_node, list);
720 }
721
sk_add_node(struct sock * sk,struct hlist_head * list)722 static inline void sk_add_node(struct sock *sk, struct hlist_head *list)
723 {
724 sock_hold(sk);
725 __sk_add_node(sk, list);
726 }
727
sk_add_node_rcu(struct sock * sk,struct hlist_head * list)728 static inline void sk_add_node_rcu(struct sock *sk, struct hlist_head *list)
729 {
730 sock_hold(sk);
731 if (IS_ENABLED(CONFIG_IPV6) && sk->sk_reuseport &&
732 sk->sk_family == AF_INET6)
733 hlist_add_tail_rcu(&sk->sk_node, list);
734 else
735 hlist_add_head_rcu(&sk->sk_node, list);
736 }
737
sk_add_node_tail_rcu(struct sock * sk,struct hlist_head * list)738 static inline void sk_add_node_tail_rcu(struct sock *sk, struct hlist_head *list)
739 {
740 sock_hold(sk);
741 hlist_add_tail_rcu(&sk->sk_node, list);
742 }
743
__sk_nulls_add_node_rcu(struct sock * sk,struct hlist_nulls_head * list)744 static inline void __sk_nulls_add_node_rcu(struct sock *sk, struct hlist_nulls_head *list)
745 {
746 hlist_nulls_add_head_rcu(&sk->sk_nulls_node, list);
747 }
748
__sk_nulls_add_node_tail_rcu(struct sock * sk,struct hlist_nulls_head * list)749 static inline void __sk_nulls_add_node_tail_rcu(struct sock *sk, struct hlist_nulls_head *list)
750 {
751 hlist_nulls_add_tail_rcu(&sk->sk_nulls_node, list);
752 }
753
sk_nulls_add_node_rcu(struct sock * sk,struct hlist_nulls_head * list)754 static inline void sk_nulls_add_node_rcu(struct sock *sk, struct hlist_nulls_head *list)
755 {
756 sock_hold(sk);
757 __sk_nulls_add_node_rcu(sk, list);
758 }
759
__sk_del_bind_node(struct sock * sk)760 static inline void __sk_del_bind_node(struct sock *sk)
761 {
762 __hlist_del(&sk->sk_bind_node);
763 }
764
sk_add_bind_node(struct sock * sk,struct hlist_head * list)765 static inline void sk_add_bind_node(struct sock *sk,
766 struct hlist_head *list)
767 {
768 hlist_add_head(&sk->sk_bind_node, list);
769 }
770
771 #define sk_for_each(__sk, list) \
772 hlist_for_each_entry(__sk, list, sk_node)
773 #define sk_for_each_rcu(__sk, list) \
774 hlist_for_each_entry_rcu(__sk, list, sk_node)
775 #define sk_nulls_for_each(__sk, node, list) \
776 hlist_nulls_for_each_entry(__sk, node, list, sk_nulls_node)
777 #define sk_nulls_for_each_rcu(__sk, node, list) \
778 hlist_nulls_for_each_entry_rcu(__sk, node, list, sk_nulls_node)
779 #define sk_for_each_from(__sk) \
780 hlist_for_each_entry_from(__sk, sk_node)
781 #define sk_nulls_for_each_from(__sk, node) \
782 if (__sk && ({ node = &(__sk)->sk_nulls_node; 1; })) \
783 hlist_nulls_for_each_entry_from(__sk, node, sk_nulls_node)
784 #define sk_for_each_safe(__sk, tmp, list) \
785 hlist_for_each_entry_safe(__sk, tmp, list, sk_node)
786 #define sk_for_each_bound(__sk, list) \
787 hlist_for_each_entry(__sk, list, sk_bind_node)
788
789 /**
790 * sk_for_each_entry_offset_rcu - iterate over a list at a given struct offset
791 * @tpos: the type * to use as a loop cursor.
792 * @pos: the &struct hlist_node to use as a loop cursor.
793 * @head: the head for your list.
794 * @offset: offset of hlist_node within the struct.
795 *
796 */
797 #define sk_for_each_entry_offset_rcu(tpos, pos, head, offset) \
798 for (pos = rcu_dereference(hlist_first_rcu(head)); \
799 pos != NULL && \
800 ({ tpos = (typeof(*tpos) *)((void *)pos - offset); 1;}); \
801 pos = rcu_dereference(hlist_next_rcu(pos)))
802
sk_user_ns(struct sock * sk)803 static inline struct user_namespace *sk_user_ns(struct sock *sk)
804 {
805 /* Careful only use this in a context where these parameters
806 * can not change and must all be valid, such as recvmsg from
807 * userspace.
808 */
809 return sk->sk_socket->file->f_cred->user_ns;
810 }
811
812 /* Sock flags */
813 enum sock_flags {
814 SOCK_DEAD,
815 SOCK_DONE,
816 SOCK_URGINLINE,
817 SOCK_KEEPOPEN,
818 SOCK_LINGER,
819 SOCK_DESTROY,
820 SOCK_BROADCAST,
821 SOCK_TIMESTAMP,
822 SOCK_ZAPPED,
823 SOCK_USE_WRITE_QUEUE, /* whether to call sk->sk_write_space in sock_wfree */
824 SOCK_DBG, /* %SO_DEBUG setting */
825 SOCK_RCVTSTAMP, /* %SO_TIMESTAMP setting */
826 SOCK_RCVTSTAMPNS, /* %SO_TIMESTAMPNS setting */
827 SOCK_LOCALROUTE, /* route locally only, %SO_DONTROUTE setting */
828 SOCK_QUEUE_SHRUNK, /* write queue has been shrunk recently */
829 SOCK_MEMALLOC, /* VM depends on this socket for swapping */
830 SOCK_TIMESTAMPING_RX_SOFTWARE, /* %SOF_TIMESTAMPING_RX_SOFTWARE */
831 SOCK_FASYNC, /* fasync() active */
832 SOCK_RXQ_OVFL,
833 SOCK_ZEROCOPY, /* buffers from userspace */
834 SOCK_WIFI_STATUS, /* push wifi status to userspace */
835 SOCK_NOFCS, /* Tell NIC not to do the Ethernet FCS.
836 * Will use last 4 bytes of packet sent from
837 * user-space instead.
838 */
839 SOCK_FILTER_LOCKED, /* Filter cannot be changed anymore */
840 SOCK_SELECT_ERR_QUEUE, /* Wake select on error queue */
841 SOCK_RCU_FREE, /* wait rcu grace period in sk_destruct() */
842 SOCK_TXTIME,
843 SOCK_XDP, /* XDP is attached */
844 SOCK_TSTAMP_NEW, /* Indicates 64 bit timestamps always */
845 };
846
847 #define SK_FLAGS_TIMESTAMP ((1UL << SOCK_TIMESTAMP) | (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE))
848
sock_copy_flags(struct sock * nsk,struct sock * osk)849 static inline void sock_copy_flags(struct sock *nsk, struct sock *osk)
850 {
851 nsk->sk_flags = osk->sk_flags;
852 }
853
sock_set_flag(struct sock * sk,enum sock_flags flag)854 static inline void sock_set_flag(struct sock *sk, enum sock_flags flag)
855 {
856 __set_bit(flag, &sk->sk_flags);
857 }
858
sock_reset_flag(struct sock * sk,enum sock_flags flag)859 static inline void sock_reset_flag(struct sock *sk, enum sock_flags flag)
860 {
861 __clear_bit(flag, &sk->sk_flags);
862 }
863
sock_flag(const struct sock * sk,enum sock_flags flag)864 static inline bool sock_flag(const struct sock *sk, enum sock_flags flag)
865 {
866 return test_bit(flag, &sk->sk_flags);
867 }
868
869 #ifdef CONFIG_NET
870 DECLARE_STATIC_KEY_FALSE(memalloc_socks_key);
sk_memalloc_socks(void)871 static inline int sk_memalloc_socks(void)
872 {
873 return static_branch_unlikely(&memalloc_socks_key);
874 }
875
876 void __receive_sock(struct file *file);
877 #else
878
sk_memalloc_socks(void)879 static inline int sk_memalloc_socks(void)
880 {
881 return 0;
882 }
883
__receive_sock(struct file * file)884 static inline void __receive_sock(struct file *file)
885 { }
886 #endif
887
sk_gfp_mask(const struct sock * sk,gfp_t gfp_mask)888 static inline gfp_t sk_gfp_mask(const struct sock *sk, gfp_t gfp_mask)
889 {
890 return gfp_mask | (sk->sk_allocation & __GFP_MEMALLOC);
891 }
892
sk_acceptq_removed(struct sock * sk)893 static inline void sk_acceptq_removed(struct sock *sk)
894 {
895 sk->sk_ack_backlog--;
896 }
897
sk_acceptq_added(struct sock * sk)898 static inline void sk_acceptq_added(struct sock *sk)
899 {
900 sk->sk_ack_backlog++;
901 }
902
sk_acceptq_is_full(const struct sock * sk)903 static inline bool sk_acceptq_is_full(const struct sock *sk)
904 {
905 return sk->sk_ack_backlog > sk->sk_max_ack_backlog;
906 }
907
908 /*
909 * Compute minimal free write space needed to queue new packets.
910 */
sk_stream_min_wspace(const struct sock * sk)911 static inline int sk_stream_min_wspace(const struct sock *sk)
912 {
913 return READ_ONCE(sk->sk_wmem_queued) >> 1;
914 }
915
sk_stream_wspace(const struct sock * sk)916 static inline int sk_stream_wspace(const struct sock *sk)
917 {
918 return READ_ONCE(sk->sk_sndbuf) - READ_ONCE(sk->sk_wmem_queued);
919 }
920
sk_wmem_queued_add(struct sock * sk,int val)921 static inline void sk_wmem_queued_add(struct sock *sk, int val)
922 {
923 WRITE_ONCE(sk->sk_wmem_queued, sk->sk_wmem_queued + val);
924 }
925
926 void sk_stream_write_space(struct sock *sk);
927
928 /* OOB backlog add */
__sk_add_backlog(struct sock * sk,struct sk_buff * skb)929 static inline void __sk_add_backlog(struct sock *sk, struct sk_buff *skb)
930 {
931 /* dont let skb dst not refcounted, we are going to leave rcu lock */
932 skb_dst_force(skb);
933
934 if (!sk->sk_backlog.tail)
935 WRITE_ONCE(sk->sk_backlog.head, skb);
936 else
937 sk->sk_backlog.tail->next = skb;
938
939 WRITE_ONCE(sk->sk_backlog.tail, skb);
940 skb->next = NULL;
941 }
942
943 /*
944 * Take into account size of receive queue and backlog queue
945 * Do not take into account this skb truesize,
946 * to allow even a single big packet to come.
947 */
sk_rcvqueues_full(const struct sock * sk,unsigned int limit)948 static inline bool sk_rcvqueues_full(const struct sock *sk, unsigned int limit)
949 {
950 unsigned int qsize = sk->sk_backlog.len + atomic_read(&sk->sk_rmem_alloc);
951
952 return qsize > limit;
953 }
954
955 /* The per-socket spinlock must be held here. */
sk_add_backlog(struct sock * sk,struct sk_buff * skb,unsigned int limit)956 static inline __must_check int sk_add_backlog(struct sock *sk, struct sk_buff *skb,
957 unsigned int limit)
958 {
959 if (sk_rcvqueues_full(sk, limit))
960 return -ENOBUFS;
961
962 /*
963 * If the skb was allocated from pfmemalloc reserves, only
964 * allow SOCK_MEMALLOC sockets to use it as this socket is
965 * helping free memory
966 */
967 if (skb_pfmemalloc(skb) && !sock_flag(sk, SOCK_MEMALLOC))
968 return -ENOMEM;
969
970 __sk_add_backlog(sk, skb);
971 sk->sk_backlog.len += skb->truesize;
972 return 0;
973 }
974
975 int __sk_backlog_rcv(struct sock *sk, struct sk_buff *skb);
976
sk_backlog_rcv(struct sock * sk,struct sk_buff * skb)977 static inline int sk_backlog_rcv(struct sock *sk, struct sk_buff *skb)
978 {
979 if (sk_memalloc_socks() && skb_pfmemalloc(skb))
980 return __sk_backlog_rcv(sk, skb);
981
982 return sk->sk_backlog_rcv(sk, skb);
983 }
984
sk_incoming_cpu_update(struct sock * sk)985 static inline void sk_incoming_cpu_update(struct sock *sk)
986 {
987 int cpu = raw_smp_processor_id();
988
989 if (unlikely(READ_ONCE(sk->sk_incoming_cpu) != cpu))
990 WRITE_ONCE(sk->sk_incoming_cpu, cpu);
991 }
992
sock_rps_record_flow_hash(__u32 hash)993 static inline void sock_rps_record_flow_hash(__u32 hash)
994 {
995 #ifdef CONFIG_RPS
996 struct rps_sock_flow_table *sock_flow_table;
997
998 rcu_read_lock();
999 sock_flow_table = rcu_dereference(rps_sock_flow_table);
1000 rps_record_sock_flow(sock_flow_table, hash);
1001 rcu_read_unlock();
1002 #endif
1003 }
1004
sock_rps_record_flow(const struct sock * sk)1005 static inline void sock_rps_record_flow(const struct sock *sk)
1006 {
1007 #ifdef CONFIG_RPS
1008 if (static_branch_unlikely(&rfs_needed)) {
1009 /* Reading sk->sk_rxhash might incur an expensive cache line
1010 * miss.
1011 *
1012 * TCP_ESTABLISHED does cover almost all states where RFS
1013 * might be useful, and is cheaper [1] than testing :
1014 * IPv4: inet_sk(sk)->inet_daddr
1015 * IPv6: ipv6_addr_any(&sk->sk_v6_daddr)
1016 * OR an additional socket flag
1017 * [1] : sk_state and sk_prot are in the same cache line.
1018 */
1019 if (sk->sk_state == TCP_ESTABLISHED) {
1020 /* This READ_ONCE() is paired with the WRITE_ONCE()
1021 * from sock_rps_save_rxhash() and sock_rps_reset_rxhash().
1022 */
1023 sock_rps_record_flow_hash(READ_ONCE(sk->sk_rxhash));
1024 }
1025 }
1026 #endif
1027 }
1028
sock_rps_save_rxhash(struct sock * sk,const struct sk_buff * skb)1029 static inline void sock_rps_save_rxhash(struct sock *sk,
1030 const struct sk_buff *skb)
1031 {
1032 #ifdef CONFIG_RPS
1033 /* The following WRITE_ONCE() is paired with the READ_ONCE()
1034 * here, and another one in sock_rps_record_flow().
1035 */
1036 if (unlikely(READ_ONCE(sk->sk_rxhash) != skb->hash))
1037 WRITE_ONCE(sk->sk_rxhash, skb->hash);
1038 #endif
1039 }
1040
sock_rps_reset_rxhash(struct sock * sk)1041 static inline void sock_rps_reset_rxhash(struct sock *sk)
1042 {
1043 #ifdef CONFIG_RPS
1044 /* Paired with READ_ONCE() in sock_rps_record_flow() */
1045 WRITE_ONCE(sk->sk_rxhash, 0);
1046 #endif
1047 }
1048
1049 #define sk_wait_event(__sk, __timeo, __condition, __wait) \
1050 ({ int __rc; \
1051 release_sock(__sk); \
1052 __rc = __condition; \
1053 if (!__rc) { \
1054 *(__timeo) = wait_woken(__wait, \
1055 TASK_INTERRUPTIBLE, \
1056 *(__timeo)); \
1057 } \
1058 sched_annotate_sleep(); \
1059 lock_sock(__sk); \
1060 __rc = __condition; \
1061 __rc; \
1062 })
1063
1064 int sk_stream_wait_connect(struct sock *sk, long *timeo_p);
1065 int sk_stream_wait_memory(struct sock *sk, long *timeo_p);
1066 void sk_stream_wait_close(struct sock *sk, long timeo_p);
1067 int sk_stream_error(struct sock *sk, int flags, int err);
1068 void sk_stream_kill_queues(struct sock *sk);
1069 void sk_set_memalloc(struct sock *sk);
1070 void sk_clear_memalloc(struct sock *sk);
1071
1072 void __sk_flush_backlog(struct sock *sk);
1073
sk_flush_backlog(struct sock * sk)1074 static inline bool sk_flush_backlog(struct sock *sk)
1075 {
1076 if (unlikely(READ_ONCE(sk->sk_backlog.tail))) {
1077 __sk_flush_backlog(sk);
1078 return true;
1079 }
1080 return false;
1081 }
1082
1083 int sk_wait_data(struct sock *sk, long *timeo, const struct sk_buff *skb);
1084
1085 struct request_sock_ops;
1086 struct timewait_sock_ops;
1087 struct inet_hashinfo;
1088 struct raw_hashinfo;
1089 struct smc_hashinfo;
1090 struct module;
1091
1092 /*
1093 * caches using SLAB_TYPESAFE_BY_RCU should let .next pointer from nulls nodes
1094 * un-modified. Special care is taken when initializing object to zero.
1095 */
sk_prot_clear_nulls(struct sock * sk,int size)1096 static inline void sk_prot_clear_nulls(struct sock *sk, int size)
1097 {
1098 if (offsetof(struct sock, sk_node.next) != 0)
1099 memset(sk, 0, offsetof(struct sock, sk_node.next));
1100 memset(&sk->sk_node.pprev, 0,
1101 size - offsetof(struct sock, sk_node.pprev));
1102 }
1103
1104 /* Networking protocol blocks we attach to sockets.
1105 * socket layer -> transport layer interface
1106 */
1107 struct proto {
1108 void (*close)(struct sock *sk,
1109 long timeout);
1110 int (*pre_connect)(struct sock *sk,
1111 struct sockaddr *uaddr,
1112 int addr_len);
1113 int (*connect)(struct sock *sk,
1114 struct sockaddr *uaddr,
1115 int addr_len);
1116 int (*disconnect)(struct sock *sk, int flags);
1117
1118 struct sock * (*accept)(struct sock *sk, int flags, int *err,
1119 bool kern);
1120
1121 int (*ioctl)(struct sock *sk, int cmd,
1122 unsigned long arg);
1123 int (*init)(struct sock *sk);
1124 void (*destroy)(struct sock *sk);
1125 void (*shutdown)(struct sock *sk, int how);
1126 int (*setsockopt)(struct sock *sk, int level,
1127 int optname, char __user *optval,
1128 unsigned int optlen);
1129 int (*getsockopt)(struct sock *sk, int level,
1130 int optname, char __user *optval,
1131 int __user *option);
1132 void (*keepalive)(struct sock *sk, int valbool);
1133 #ifdef CONFIG_COMPAT
1134 int (*compat_setsockopt)(struct sock *sk,
1135 int level,
1136 int optname, char __user *optval,
1137 unsigned int optlen);
1138 int (*compat_getsockopt)(struct sock *sk,
1139 int level,
1140 int optname, char __user *optval,
1141 int __user *option);
1142 int (*compat_ioctl)(struct sock *sk,
1143 unsigned int cmd, unsigned long arg);
1144 #endif
1145 int (*sendmsg)(struct sock *sk, struct msghdr *msg,
1146 size_t len);
1147 int (*recvmsg)(struct sock *sk, struct msghdr *msg,
1148 size_t len, int noblock, int flags,
1149 int *addr_len);
1150 int (*sendpage)(struct sock *sk, struct page *page,
1151 int offset, size_t size, int flags);
1152 int (*bind)(struct sock *sk,
1153 struct sockaddr *uaddr, int addr_len);
1154
1155 int (*backlog_rcv) (struct sock *sk,
1156 struct sk_buff *skb);
1157
1158 void (*release_cb)(struct sock *sk);
1159
1160 /* Keeping track of sk's, looking them up, and port selection methods. */
1161 int (*hash)(struct sock *sk);
1162 void (*unhash)(struct sock *sk);
1163 void (*rehash)(struct sock *sk);
1164 int (*get_port)(struct sock *sk, unsigned short snum);
1165
1166 /* Keeping track of sockets in use */
1167 #ifdef CONFIG_PROC_FS
1168 unsigned int inuse_idx;
1169 #endif
1170
1171 bool (*stream_memory_free)(const struct sock *sk, int wake);
1172 bool (*stream_memory_read)(const struct sock *sk);
1173 /* Memory pressure */
1174 void (*enter_memory_pressure)(struct sock *sk);
1175 void (*leave_memory_pressure)(struct sock *sk);
1176 atomic_long_t *memory_allocated; /* Current allocated memory. */
1177 struct percpu_counter *sockets_allocated; /* Current number of sockets. */
1178 /*
1179 * Pressure flag: try to collapse.
1180 * Technical note: it is used by multiple contexts non atomically.
1181 * Make sure to use READ_ONCE()/WRITE_ONCE() for all reads/writes.
1182 * All the __sk_mem_schedule() is of this nature: accounting
1183 * is strict, actions are advisory and have some latency.
1184 */
1185 unsigned long *memory_pressure;
1186 long *sysctl_mem;
1187
1188 int *sysctl_wmem;
1189 int *sysctl_rmem;
1190 u32 sysctl_wmem_offset;
1191 u32 sysctl_rmem_offset;
1192
1193 int max_header;
1194 bool no_autobind;
1195
1196 struct kmem_cache *slab;
1197 unsigned int obj_size;
1198 slab_flags_t slab_flags;
1199 unsigned int useroffset; /* Usercopy region offset */
1200 unsigned int usersize; /* Usercopy region size */
1201
1202 struct percpu_counter *orphan_count;
1203
1204 struct request_sock_ops *rsk_prot;
1205 struct timewait_sock_ops *twsk_prot;
1206
1207 union {
1208 struct inet_hashinfo *hashinfo;
1209 struct udp_table *udp_table;
1210 struct raw_hashinfo *raw_hash;
1211 struct smc_hashinfo *smc_hash;
1212 } h;
1213
1214 struct module *owner;
1215
1216 char name[32];
1217
1218 struct list_head node;
1219 #ifdef SOCK_REFCNT_DEBUG
1220 atomic_t socks;
1221 #endif
1222 int (*diag_destroy)(struct sock *sk, int err);
1223 } __randomize_layout;
1224
1225 int proto_register(struct proto *prot, int alloc_slab);
1226 void proto_unregister(struct proto *prot);
1227 int sock_load_diag_module(int family, int protocol);
1228
1229 #ifdef SOCK_REFCNT_DEBUG
sk_refcnt_debug_inc(struct sock * sk)1230 static inline void sk_refcnt_debug_inc(struct sock *sk)
1231 {
1232 atomic_inc(&sk->sk_prot->socks);
1233 }
1234
sk_refcnt_debug_dec(struct sock * sk)1235 static inline void sk_refcnt_debug_dec(struct sock *sk)
1236 {
1237 atomic_dec(&sk->sk_prot->socks);
1238 printk(KERN_DEBUG "%s socket %p released, %d are still alive\n",
1239 sk->sk_prot->name, sk, atomic_read(&sk->sk_prot->socks));
1240 }
1241
sk_refcnt_debug_release(const struct sock * sk)1242 static inline void sk_refcnt_debug_release(const struct sock *sk)
1243 {
1244 if (refcount_read(&sk->sk_refcnt) != 1)
1245 printk(KERN_DEBUG "Destruction of the %s socket %p delayed, refcnt=%d\n",
1246 sk->sk_prot->name, sk, refcount_read(&sk->sk_refcnt));
1247 }
1248 #else /* SOCK_REFCNT_DEBUG */
1249 #define sk_refcnt_debug_inc(sk) do { } while (0)
1250 #define sk_refcnt_debug_dec(sk) do { } while (0)
1251 #define sk_refcnt_debug_release(sk) do { } while (0)
1252 #endif /* SOCK_REFCNT_DEBUG */
1253
__sk_stream_memory_free(const struct sock * sk,int wake)1254 static inline bool __sk_stream_memory_free(const struct sock *sk, int wake)
1255 {
1256 if (READ_ONCE(sk->sk_wmem_queued) >= READ_ONCE(sk->sk_sndbuf))
1257 return false;
1258
1259 return sk->sk_prot->stream_memory_free ?
1260 sk->sk_prot->stream_memory_free(sk, wake) : true;
1261 }
1262
sk_stream_memory_free(const struct sock * sk)1263 static inline bool sk_stream_memory_free(const struct sock *sk)
1264 {
1265 return __sk_stream_memory_free(sk, 0);
1266 }
1267
__sk_stream_is_writeable(const struct sock * sk,int wake)1268 static inline bool __sk_stream_is_writeable(const struct sock *sk, int wake)
1269 {
1270 return sk_stream_wspace(sk) >= sk_stream_min_wspace(sk) &&
1271 __sk_stream_memory_free(sk, wake);
1272 }
1273
sk_stream_is_writeable(const struct sock * sk)1274 static inline bool sk_stream_is_writeable(const struct sock *sk)
1275 {
1276 return __sk_stream_is_writeable(sk, 0);
1277 }
1278
sk_under_cgroup_hierarchy(struct sock * sk,struct cgroup * ancestor)1279 static inline int sk_under_cgroup_hierarchy(struct sock *sk,
1280 struct cgroup *ancestor)
1281 {
1282 #ifdef CONFIG_SOCK_CGROUP_DATA
1283 return cgroup_is_descendant(sock_cgroup_ptr(&sk->sk_cgrp_data),
1284 ancestor);
1285 #else
1286 return -ENOTSUPP;
1287 #endif
1288 }
1289
sk_has_memory_pressure(const struct sock * sk)1290 static inline bool sk_has_memory_pressure(const struct sock *sk)
1291 {
1292 return sk->sk_prot->memory_pressure != NULL;
1293 }
1294
sk_under_global_memory_pressure(const struct sock * sk)1295 static inline bool sk_under_global_memory_pressure(const struct sock *sk)
1296 {
1297 return sk->sk_prot->memory_pressure &&
1298 !!READ_ONCE(*sk->sk_prot->memory_pressure);
1299 }
1300
sk_under_memory_pressure(const struct sock * sk)1301 static inline bool sk_under_memory_pressure(const struct sock *sk)
1302 {
1303 if (!sk->sk_prot->memory_pressure)
1304 return false;
1305
1306 if (mem_cgroup_sockets_enabled && sk->sk_memcg &&
1307 mem_cgroup_under_socket_pressure(sk->sk_memcg))
1308 return true;
1309
1310 return !!READ_ONCE(*sk->sk_prot->memory_pressure);
1311 }
1312
1313 static inline long
sk_memory_allocated(const struct sock * sk)1314 sk_memory_allocated(const struct sock *sk)
1315 {
1316 return atomic_long_read(sk->sk_prot->memory_allocated);
1317 }
1318
1319 static inline long
sk_memory_allocated_add(struct sock * sk,int amt)1320 sk_memory_allocated_add(struct sock *sk, int amt)
1321 {
1322 return atomic_long_add_return(amt, sk->sk_prot->memory_allocated);
1323 }
1324
1325 static inline void
sk_memory_allocated_sub(struct sock * sk,int amt)1326 sk_memory_allocated_sub(struct sock *sk, int amt)
1327 {
1328 atomic_long_sub(amt, sk->sk_prot->memory_allocated);
1329 }
1330
sk_sockets_allocated_dec(struct sock * sk)1331 static inline void sk_sockets_allocated_dec(struct sock *sk)
1332 {
1333 percpu_counter_dec(sk->sk_prot->sockets_allocated);
1334 }
1335
sk_sockets_allocated_inc(struct sock * sk)1336 static inline void sk_sockets_allocated_inc(struct sock *sk)
1337 {
1338 percpu_counter_inc(sk->sk_prot->sockets_allocated);
1339 }
1340
1341 static inline u64
sk_sockets_allocated_read_positive(struct sock * sk)1342 sk_sockets_allocated_read_positive(struct sock *sk)
1343 {
1344 return percpu_counter_read_positive(sk->sk_prot->sockets_allocated);
1345 }
1346
1347 static inline int
proto_sockets_allocated_sum_positive(struct proto * prot)1348 proto_sockets_allocated_sum_positive(struct proto *prot)
1349 {
1350 return percpu_counter_sum_positive(prot->sockets_allocated);
1351 }
1352
1353 static inline long
proto_memory_allocated(struct proto * prot)1354 proto_memory_allocated(struct proto *prot)
1355 {
1356 return atomic_long_read(prot->memory_allocated);
1357 }
1358
1359 static inline bool
proto_memory_pressure(struct proto * prot)1360 proto_memory_pressure(struct proto *prot)
1361 {
1362 if (!prot->memory_pressure)
1363 return false;
1364 return !!READ_ONCE(*prot->memory_pressure);
1365 }
1366
1367
1368 #ifdef CONFIG_PROC_FS
1369 /* Called with local bh disabled */
1370 void sock_prot_inuse_add(struct net *net, struct proto *prot, int inc);
1371 int sock_prot_inuse_get(struct net *net, struct proto *proto);
1372 int sock_inuse_get(struct net *net);
1373 #else
sock_prot_inuse_add(struct net * net,struct proto * prot,int inc)1374 static inline void sock_prot_inuse_add(struct net *net, struct proto *prot,
1375 int inc)
1376 {
1377 }
1378 #endif
1379
1380
1381 /* With per-bucket locks this operation is not-atomic, so that
1382 * this version is not worse.
1383 */
__sk_prot_rehash(struct sock * sk)1384 static inline int __sk_prot_rehash(struct sock *sk)
1385 {
1386 sk->sk_prot->unhash(sk);
1387 return sk->sk_prot->hash(sk);
1388 }
1389
1390 /* About 10 seconds */
1391 #define SOCK_DESTROY_TIME (10*HZ)
1392
1393 /* Sockets 0-1023 can't be bound to unless you are superuser */
1394 #define PROT_SOCK 1024
1395
1396 #define SHUTDOWN_MASK 3
1397 #define RCV_SHUTDOWN 1
1398 #define SEND_SHUTDOWN 2
1399
1400 #define SOCK_SNDBUF_LOCK 1
1401 #define SOCK_RCVBUF_LOCK 2
1402 #define SOCK_BINDADDR_LOCK 4
1403 #define SOCK_BINDPORT_LOCK 8
1404
1405 struct socket_alloc {
1406 struct socket socket;
1407 struct inode vfs_inode;
1408 };
1409
SOCKET_I(struct inode * inode)1410 static inline struct socket *SOCKET_I(struct inode *inode)
1411 {
1412 return &container_of(inode, struct socket_alloc, vfs_inode)->socket;
1413 }
1414
SOCK_INODE(struct socket * socket)1415 static inline struct inode *SOCK_INODE(struct socket *socket)
1416 {
1417 return &container_of(socket, struct socket_alloc, socket)->vfs_inode;
1418 }
1419
1420 /*
1421 * Functions for memory accounting
1422 */
1423 int __sk_mem_raise_allocated(struct sock *sk, int size, int amt, int kind);
1424 int __sk_mem_schedule(struct sock *sk, int size, int kind);
1425 void __sk_mem_reduce_allocated(struct sock *sk, int amount);
1426 void __sk_mem_reclaim(struct sock *sk, int amount);
1427
1428 /* We used to have PAGE_SIZE here, but systems with 64KB pages
1429 * do not necessarily have 16x time more memory than 4KB ones.
1430 */
1431 #define SK_MEM_QUANTUM 4096
1432 #define SK_MEM_QUANTUM_SHIFT ilog2(SK_MEM_QUANTUM)
1433 #define SK_MEM_SEND 0
1434 #define SK_MEM_RECV 1
1435
1436 /* sysctl_mem values are in pages, we convert them in SK_MEM_QUANTUM units */
sk_prot_mem_limits(const struct sock * sk,int index)1437 static inline long sk_prot_mem_limits(const struct sock *sk, int index)
1438 {
1439 long val = READ_ONCE(sk->sk_prot->sysctl_mem[index]);
1440
1441 #if PAGE_SIZE > SK_MEM_QUANTUM
1442 val <<= PAGE_SHIFT - SK_MEM_QUANTUM_SHIFT;
1443 #elif PAGE_SIZE < SK_MEM_QUANTUM
1444 val >>= SK_MEM_QUANTUM_SHIFT - PAGE_SHIFT;
1445 #endif
1446 return val;
1447 }
1448
sk_mem_pages(int amt)1449 static inline int sk_mem_pages(int amt)
1450 {
1451 return (amt + SK_MEM_QUANTUM - 1) >> SK_MEM_QUANTUM_SHIFT;
1452 }
1453
sk_has_account(struct sock * sk)1454 static inline bool sk_has_account(struct sock *sk)
1455 {
1456 /* return true if protocol supports memory accounting */
1457 return !!sk->sk_prot->memory_allocated;
1458 }
1459
sk_wmem_schedule(struct sock * sk,int size)1460 static inline bool sk_wmem_schedule(struct sock *sk, int size)
1461 {
1462 if (!sk_has_account(sk))
1463 return true;
1464 return size <= sk->sk_forward_alloc ||
1465 __sk_mem_schedule(sk, size, SK_MEM_SEND);
1466 }
1467
1468 static inline bool
sk_rmem_schedule(struct sock * sk,struct sk_buff * skb,int size)1469 sk_rmem_schedule(struct sock *sk, struct sk_buff *skb, int size)
1470 {
1471 if (!sk_has_account(sk))
1472 return true;
1473 return size<= sk->sk_forward_alloc ||
1474 __sk_mem_schedule(sk, size, SK_MEM_RECV) ||
1475 skb_pfmemalloc(skb);
1476 }
1477
sk_mem_reclaim(struct sock * sk)1478 static inline void sk_mem_reclaim(struct sock *sk)
1479 {
1480 if (!sk_has_account(sk))
1481 return;
1482 if (sk->sk_forward_alloc >= SK_MEM_QUANTUM)
1483 __sk_mem_reclaim(sk, sk->sk_forward_alloc);
1484 }
1485
sk_mem_reclaim_partial(struct sock * sk)1486 static inline void sk_mem_reclaim_partial(struct sock *sk)
1487 {
1488 if (!sk_has_account(sk))
1489 return;
1490 if (sk->sk_forward_alloc > SK_MEM_QUANTUM)
1491 __sk_mem_reclaim(sk, sk->sk_forward_alloc - 1);
1492 }
1493
sk_mem_charge(struct sock * sk,int size)1494 static inline void sk_mem_charge(struct sock *sk, int size)
1495 {
1496 if (!sk_has_account(sk))
1497 return;
1498 sk->sk_forward_alloc -= size;
1499 }
1500
sk_mem_uncharge(struct sock * sk,int size)1501 static inline void sk_mem_uncharge(struct sock *sk, int size)
1502 {
1503 if (!sk_has_account(sk))
1504 return;
1505 sk->sk_forward_alloc += size;
1506
1507 /* Avoid a possible overflow.
1508 * TCP send queues can make this happen, if sk_mem_reclaim()
1509 * is not called and more than 2 GBytes are released at once.
1510 *
1511 * If we reach 2 MBytes, reclaim 1 MBytes right now, there is
1512 * no need to hold that much forward allocation anyway.
1513 */
1514 if (unlikely(sk->sk_forward_alloc >= 1 << 21))
1515 __sk_mem_reclaim(sk, 1 << 20);
1516 }
1517
1518 DECLARE_STATIC_KEY_FALSE(tcp_tx_skb_cache_key);
sk_wmem_free_skb(struct sock * sk,struct sk_buff * skb)1519 static inline void sk_wmem_free_skb(struct sock *sk, struct sk_buff *skb)
1520 {
1521 sock_set_flag(sk, SOCK_QUEUE_SHRUNK);
1522 sk_wmem_queued_add(sk, -skb->truesize);
1523 sk_mem_uncharge(sk, skb->truesize);
1524 if (static_branch_unlikely(&tcp_tx_skb_cache_key) &&
1525 !sk->sk_tx_skb_cache && !skb_cloned(skb)) {
1526 skb_zcopy_clear(skb, true);
1527 sk->sk_tx_skb_cache = skb;
1528 return;
1529 }
1530 __kfree_skb(skb);
1531 }
1532
sock_release_ownership(struct sock * sk)1533 static inline void sock_release_ownership(struct sock *sk)
1534 {
1535 if (sk->sk_lock.owned) {
1536 sk->sk_lock.owned = 0;
1537
1538 /* The sk_lock has mutex_unlock() semantics: */
1539 mutex_release(&sk->sk_lock.dep_map, 1, _RET_IP_);
1540 }
1541 }
1542
1543 /*
1544 * Macro so as to not evaluate some arguments when
1545 * lockdep is not enabled.
1546 *
1547 * Mark both the sk_lock and the sk_lock.slock as a
1548 * per-address-family lock class.
1549 */
1550 #define sock_lock_init_class_and_name(sk, sname, skey, name, key) \
1551 do { \
1552 sk->sk_lock.owned = 0; \
1553 init_waitqueue_head(&sk->sk_lock.wq); \
1554 spin_lock_init(&(sk)->sk_lock.slock); \
1555 debug_check_no_locks_freed((void *)&(sk)->sk_lock, \
1556 sizeof((sk)->sk_lock)); \
1557 lockdep_set_class_and_name(&(sk)->sk_lock.slock, \
1558 (skey), (sname)); \
1559 lockdep_init_map(&(sk)->sk_lock.dep_map, (name), (key), 0); \
1560 } while (0)
1561
1562 #ifdef CONFIG_LOCKDEP
lockdep_sock_is_held(const struct sock * sk)1563 static inline bool lockdep_sock_is_held(const struct sock *sk)
1564 {
1565 return lockdep_is_held(&sk->sk_lock) ||
1566 lockdep_is_held(&sk->sk_lock.slock);
1567 }
1568 #endif
1569
1570 void lock_sock_nested(struct sock *sk, int subclass);
1571
lock_sock(struct sock * sk)1572 static inline void lock_sock(struct sock *sk)
1573 {
1574 lock_sock_nested(sk, 0);
1575 }
1576
1577 void __release_sock(struct sock *sk);
1578 void release_sock(struct sock *sk);
1579
1580 /* BH context may only use the following locking interface. */
1581 #define bh_lock_sock(__sk) spin_lock(&((__sk)->sk_lock.slock))
1582 #define bh_lock_sock_nested(__sk) \
1583 spin_lock_nested(&((__sk)->sk_lock.slock), \
1584 SINGLE_DEPTH_NESTING)
1585 #define bh_unlock_sock(__sk) spin_unlock(&((__sk)->sk_lock.slock))
1586
1587 bool lock_sock_fast(struct sock *sk);
1588 /**
1589 * unlock_sock_fast - complement of lock_sock_fast
1590 * @sk: socket
1591 * @slow: slow mode
1592 *
1593 * fast unlock socket for user context.
1594 * If slow mode is on, we call regular release_sock()
1595 */
unlock_sock_fast(struct sock * sk,bool slow)1596 static inline void unlock_sock_fast(struct sock *sk, bool slow)
1597 {
1598 if (slow)
1599 release_sock(sk);
1600 else
1601 spin_unlock_bh(&sk->sk_lock.slock);
1602 }
1603
1604 /* Used by processes to "lock" a socket state, so that
1605 * interrupts and bottom half handlers won't change it
1606 * from under us. It essentially blocks any incoming
1607 * packets, so that we won't get any new data or any
1608 * packets that change the state of the socket.
1609 *
1610 * While locked, BH processing will add new packets to
1611 * the backlog queue. This queue is processed by the
1612 * owner of the socket lock right before it is released.
1613 *
1614 * Since ~2.3.5 it is also exclusive sleep lock serializing
1615 * accesses from user process context.
1616 */
1617
sock_owned_by_me(const struct sock * sk)1618 static inline void sock_owned_by_me(const struct sock *sk)
1619 {
1620 #ifdef CONFIG_LOCKDEP
1621 WARN_ON_ONCE(!lockdep_sock_is_held(sk) && debug_locks);
1622 #endif
1623 }
1624
sock_not_owned_by_me(const struct sock * sk)1625 static inline void sock_not_owned_by_me(const struct sock *sk)
1626 {
1627 #ifdef CONFIG_LOCKDEP
1628 WARN_ON_ONCE(lockdep_sock_is_held(sk) && debug_locks);
1629 #endif
1630 }
1631
sock_owned_by_user(const struct sock * sk)1632 static inline bool sock_owned_by_user(const struct sock *sk)
1633 {
1634 sock_owned_by_me(sk);
1635 return sk->sk_lock.owned;
1636 }
1637
sock_owned_by_user_nocheck(const struct sock * sk)1638 static inline bool sock_owned_by_user_nocheck(const struct sock *sk)
1639 {
1640 return sk->sk_lock.owned;
1641 }
1642
1643 /* no reclassification while locks are held */
sock_allow_reclassification(const struct sock * csk)1644 static inline bool sock_allow_reclassification(const struct sock *csk)
1645 {
1646 struct sock *sk = (struct sock *)csk;
1647
1648 return !sk->sk_lock.owned && !spin_is_locked(&sk->sk_lock.slock);
1649 }
1650
1651 struct sock *sk_alloc(struct net *net, int family, gfp_t priority,
1652 struct proto *prot, int kern);
1653 void sk_free(struct sock *sk);
1654 void sk_destruct(struct sock *sk);
1655 struct sock *sk_clone_lock(const struct sock *sk, const gfp_t priority);
1656 void sk_free_unlock_clone(struct sock *sk);
1657
1658 struct sk_buff *sock_wmalloc(struct sock *sk, unsigned long size, int force,
1659 gfp_t priority);
1660 void __sock_wfree(struct sk_buff *skb);
1661 void sock_wfree(struct sk_buff *skb);
1662 struct sk_buff *sock_omalloc(struct sock *sk, unsigned long size,
1663 gfp_t priority);
1664 void skb_orphan_partial(struct sk_buff *skb);
1665 void sock_rfree(struct sk_buff *skb);
1666 void sock_efree(struct sk_buff *skb);
1667 #ifdef CONFIG_INET
1668 void sock_edemux(struct sk_buff *skb);
1669 #else
1670 #define sock_edemux sock_efree
1671 #endif
1672
1673 int sock_setsockopt(struct socket *sock, int level, int op,
1674 char __user *optval, unsigned int optlen);
1675
1676 int sock_getsockopt(struct socket *sock, int level, int op,
1677 char __user *optval, int __user *optlen);
1678 int sock_gettstamp(struct socket *sock, void __user *userstamp,
1679 bool timeval, bool time32);
1680 struct sk_buff *sock_alloc_send_skb(struct sock *sk, unsigned long size,
1681 int noblock, int *errcode);
1682 struct sk_buff *sock_alloc_send_pskb(struct sock *sk, unsigned long header_len,
1683 unsigned long data_len, int noblock,
1684 int *errcode, int max_page_order);
1685 void *sock_kmalloc(struct sock *sk, int size, gfp_t priority);
1686 void sock_kfree_s(struct sock *sk, void *mem, int size);
1687 void sock_kzfree_s(struct sock *sk, void *mem, int size);
1688 void sk_send_sigurg(struct sock *sk);
1689
1690 struct sockcm_cookie {
1691 u64 transmit_time;
1692 u32 mark;
1693 u16 tsflags;
1694 };
1695
sockcm_init(struct sockcm_cookie * sockc,const struct sock * sk)1696 static inline void sockcm_init(struct sockcm_cookie *sockc,
1697 const struct sock *sk)
1698 {
1699 *sockc = (struct sockcm_cookie) { .tsflags = sk->sk_tsflags };
1700 }
1701
1702 int __sock_cmsg_send(struct sock *sk, struct msghdr *msg, struct cmsghdr *cmsg,
1703 struct sockcm_cookie *sockc);
1704 int sock_cmsg_send(struct sock *sk, struct msghdr *msg,
1705 struct sockcm_cookie *sockc);
1706
1707 /*
1708 * Functions to fill in entries in struct proto_ops when a protocol
1709 * does not implement a particular function.
1710 */
1711 int sock_no_bind(struct socket *, struct sockaddr *, int);
1712 int sock_no_connect(struct socket *, struct sockaddr *, int, int);
1713 int sock_no_socketpair(struct socket *, struct socket *);
1714 int sock_no_accept(struct socket *, struct socket *, int, bool);
1715 int sock_no_getname(struct socket *, struct sockaddr *, int);
1716 int sock_no_ioctl(struct socket *, unsigned int, unsigned long);
1717 int sock_no_listen(struct socket *, int);
1718 int sock_no_shutdown(struct socket *, int);
1719 int sock_no_getsockopt(struct socket *, int , int, char __user *, int __user *);
1720 int sock_no_setsockopt(struct socket *, int, int, char __user *, unsigned int);
1721 int sock_no_sendmsg(struct socket *, struct msghdr *, size_t);
1722 int sock_no_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t len);
1723 int sock_no_recvmsg(struct socket *, struct msghdr *, size_t, int);
1724 int sock_no_mmap(struct file *file, struct socket *sock,
1725 struct vm_area_struct *vma);
1726 ssize_t sock_no_sendpage(struct socket *sock, struct page *page, int offset,
1727 size_t size, int flags);
1728 ssize_t sock_no_sendpage_locked(struct sock *sk, struct page *page,
1729 int offset, size_t size, int flags);
1730
1731 /*
1732 * Functions to fill in entries in struct proto_ops when a protocol
1733 * uses the inet style.
1734 */
1735 int sock_common_getsockopt(struct socket *sock, int level, int optname,
1736 char __user *optval, int __user *optlen);
1737 int sock_common_recvmsg(struct socket *sock, struct msghdr *msg, size_t size,
1738 int flags);
1739 int sock_common_setsockopt(struct socket *sock, int level, int optname,
1740 char __user *optval, unsigned int optlen);
1741 int compat_sock_common_getsockopt(struct socket *sock, int level,
1742 int optname, char __user *optval, int __user *optlen);
1743 int compat_sock_common_setsockopt(struct socket *sock, int level,
1744 int optname, char __user *optval, unsigned int optlen);
1745
1746 void sk_common_release(struct sock *sk);
1747
1748 /*
1749 * Default socket callbacks and setup code
1750 */
1751
1752 /* Initialise core socket variables using an explicit uid. */
1753 void sock_init_data_uid(struct socket *sock, struct sock *sk, kuid_t uid);
1754
1755 /* Initialise core socket variables.
1756 * Assumes struct socket *sock is embedded in a struct socket_alloc.
1757 */
1758 void sock_init_data(struct socket *sock, struct sock *sk);
1759
1760 /*
1761 * Socket reference counting postulates.
1762 *
1763 * * Each user of socket SHOULD hold a reference count.
1764 * * Each access point to socket (an hash table bucket, reference from a list,
1765 * running timer, skb in flight MUST hold a reference count.
1766 * * When reference count hits 0, it means it will never increase back.
1767 * * When reference count hits 0, it means that no references from
1768 * outside exist to this socket and current process on current CPU
1769 * is last user and may/should destroy this socket.
1770 * * sk_free is called from any context: process, BH, IRQ. When
1771 * it is called, socket has no references from outside -> sk_free
1772 * may release descendant resources allocated by the socket, but
1773 * to the time when it is called, socket is NOT referenced by any
1774 * hash tables, lists etc.
1775 * * Packets, delivered from outside (from network or from another process)
1776 * and enqueued on receive/error queues SHOULD NOT grab reference count,
1777 * when they sit in queue. Otherwise, packets will leak to hole, when
1778 * socket is looked up by one cpu and unhasing is made by another CPU.
1779 * It is true for udp/raw, netlink (leak to receive and error queues), tcp
1780 * (leak to backlog). Packet socket does all the processing inside
1781 * BR_NETPROTO_LOCK, so that it has not this race condition. UNIX sockets
1782 * use separate SMP lock, so that they are prone too.
1783 */
1784
1785 /* Ungrab socket and destroy it, if it was the last reference. */
sock_put(struct sock * sk)1786 static inline void sock_put(struct sock *sk)
1787 {
1788 if (refcount_dec_and_test(&sk->sk_refcnt))
1789 sk_free(sk);
1790 }
1791 /* Generic version of sock_put(), dealing with all sockets
1792 * (TCP_TIMEWAIT, TCP_NEW_SYN_RECV, ESTABLISHED...)
1793 */
1794 void sock_gen_put(struct sock *sk);
1795
1796 int __sk_receive_skb(struct sock *sk, struct sk_buff *skb, const int nested,
1797 unsigned int trim_cap, bool refcounted);
sk_receive_skb(struct sock * sk,struct sk_buff * skb,const int nested)1798 static inline int sk_receive_skb(struct sock *sk, struct sk_buff *skb,
1799 const int nested)
1800 {
1801 return __sk_receive_skb(sk, skb, nested, 1, true);
1802 }
1803
sk_tx_queue_set(struct sock * sk,int tx_queue)1804 static inline void sk_tx_queue_set(struct sock *sk, int tx_queue)
1805 {
1806 /* sk_tx_queue_mapping accept only upto a 16-bit value */
1807 if (WARN_ON_ONCE((unsigned short)tx_queue >= USHRT_MAX))
1808 return;
1809 /* Paired with READ_ONCE() in sk_tx_queue_get() and
1810 * other WRITE_ONCE() because socket lock might be not held.
1811 */
1812 WRITE_ONCE(sk->sk_tx_queue_mapping, tx_queue);
1813 }
1814
1815 #define NO_QUEUE_MAPPING USHRT_MAX
1816
sk_tx_queue_clear(struct sock * sk)1817 static inline void sk_tx_queue_clear(struct sock *sk)
1818 {
1819 /* Paired with READ_ONCE() in sk_tx_queue_get() and
1820 * other WRITE_ONCE() because socket lock might be not held.
1821 */
1822 WRITE_ONCE(sk->sk_tx_queue_mapping, NO_QUEUE_MAPPING);
1823 }
1824
sk_tx_queue_get(const struct sock * sk)1825 static inline int sk_tx_queue_get(const struct sock *sk)
1826 {
1827 if (sk) {
1828 /* Paired with WRITE_ONCE() in sk_tx_queue_clear()
1829 * and sk_tx_queue_set().
1830 */
1831 int val = READ_ONCE(sk->sk_tx_queue_mapping);
1832
1833 if (val != NO_QUEUE_MAPPING)
1834 return val;
1835 }
1836 return -1;
1837 }
1838
sk_rx_queue_set(struct sock * sk,const struct sk_buff * skb)1839 static inline void sk_rx_queue_set(struct sock *sk, const struct sk_buff *skb)
1840 {
1841 #ifdef CONFIG_XPS
1842 if (skb_rx_queue_recorded(skb)) {
1843 u16 rx_queue = skb_get_rx_queue(skb);
1844
1845 if (WARN_ON_ONCE(rx_queue == NO_QUEUE_MAPPING))
1846 return;
1847
1848 sk->sk_rx_queue_mapping = rx_queue;
1849 }
1850 #endif
1851 }
1852
sk_rx_queue_clear(struct sock * sk)1853 static inline void sk_rx_queue_clear(struct sock *sk)
1854 {
1855 #ifdef CONFIG_XPS
1856 sk->sk_rx_queue_mapping = NO_QUEUE_MAPPING;
1857 #endif
1858 }
1859
1860 #ifdef CONFIG_XPS
sk_rx_queue_get(const struct sock * sk)1861 static inline int sk_rx_queue_get(const struct sock *sk)
1862 {
1863 if (sk && sk->sk_rx_queue_mapping != NO_QUEUE_MAPPING)
1864 return sk->sk_rx_queue_mapping;
1865
1866 return -1;
1867 }
1868 #endif
1869
sk_set_socket(struct sock * sk,struct socket * sock)1870 static inline void sk_set_socket(struct sock *sk, struct socket *sock)
1871 {
1872 sk->sk_socket = sock;
1873 }
1874
sk_sleep(struct sock * sk)1875 static inline wait_queue_head_t *sk_sleep(struct sock *sk)
1876 {
1877 BUILD_BUG_ON(offsetof(struct socket_wq, wait) != 0);
1878 return &rcu_dereference_raw(sk->sk_wq)->wait;
1879 }
1880 /* Detach socket from process context.
1881 * Announce socket dead, detach it from wait queue and inode.
1882 * Note that parent inode held reference count on this struct sock,
1883 * we do not release it in this function, because protocol
1884 * probably wants some additional cleanups or even continuing
1885 * to work with this socket (TCP).
1886 */
sock_orphan(struct sock * sk)1887 static inline void sock_orphan(struct sock *sk)
1888 {
1889 write_lock_bh(&sk->sk_callback_lock);
1890 sock_set_flag(sk, SOCK_DEAD);
1891 sk_set_socket(sk, NULL);
1892 sk->sk_wq = NULL;
1893 write_unlock_bh(&sk->sk_callback_lock);
1894 }
1895
sock_graft(struct sock * sk,struct socket * parent)1896 static inline void sock_graft(struct sock *sk, struct socket *parent)
1897 {
1898 WARN_ON(parent->sk);
1899 write_lock_bh(&sk->sk_callback_lock);
1900 rcu_assign_pointer(sk->sk_wq, &parent->wq);
1901 parent->sk = sk;
1902 sk_set_socket(sk, parent);
1903 sk->sk_uid = SOCK_INODE(parent)->i_uid;
1904 security_sock_graft(sk, parent);
1905 write_unlock_bh(&sk->sk_callback_lock);
1906 }
1907
1908 kuid_t sock_i_uid(struct sock *sk);
1909 unsigned long __sock_i_ino(struct sock *sk);
1910 unsigned long sock_i_ino(struct sock *sk);
1911
sock_net_uid(const struct net * net,const struct sock * sk)1912 static inline kuid_t sock_net_uid(const struct net *net, const struct sock *sk)
1913 {
1914 return sk ? sk->sk_uid : make_kuid(net->user_ns, 0);
1915 }
1916
net_tx_rndhash(void)1917 static inline u32 net_tx_rndhash(void)
1918 {
1919 u32 v = prandom_u32();
1920
1921 return v ?: 1;
1922 }
1923
sk_set_txhash(struct sock * sk)1924 static inline void sk_set_txhash(struct sock *sk)
1925 {
1926 /* This pairs with READ_ONCE() in skb_set_hash_from_sk() */
1927 WRITE_ONCE(sk->sk_txhash, net_tx_rndhash());
1928 }
1929
sk_rethink_txhash(struct sock * sk)1930 static inline void sk_rethink_txhash(struct sock *sk)
1931 {
1932 if (sk->sk_txhash)
1933 sk_set_txhash(sk);
1934 }
1935
1936 static inline struct dst_entry *
__sk_dst_get(struct sock * sk)1937 __sk_dst_get(struct sock *sk)
1938 {
1939 return rcu_dereference_check(sk->sk_dst_cache,
1940 lockdep_sock_is_held(sk));
1941 }
1942
1943 static inline struct dst_entry *
sk_dst_get(struct sock * sk)1944 sk_dst_get(struct sock *sk)
1945 {
1946 struct dst_entry *dst;
1947
1948 rcu_read_lock();
1949 dst = rcu_dereference(sk->sk_dst_cache);
1950 if (dst && !atomic_inc_not_zero(&dst->__refcnt))
1951 dst = NULL;
1952 rcu_read_unlock();
1953 return dst;
1954 }
1955
dst_negative_advice(struct sock * sk)1956 static inline void dst_negative_advice(struct sock *sk)
1957 {
1958 struct dst_entry *ndst, *dst = __sk_dst_get(sk);
1959
1960 sk_rethink_txhash(sk);
1961
1962 if (dst && dst->ops->negative_advice) {
1963 ndst = dst->ops->negative_advice(dst);
1964
1965 if (ndst != dst) {
1966 rcu_assign_pointer(sk->sk_dst_cache, ndst);
1967 sk_tx_queue_clear(sk);
1968 WRITE_ONCE(sk->sk_dst_pending_confirm, 0);
1969 }
1970 }
1971 }
1972
1973 static inline void
__sk_dst_set(struct sock * sk,struct dst_entry * dst)1974 __sk_dst_set(struct sock *sk, struct dst_entry *dst)
1975 {
1976 struct dst_entry *old_dst;
1977
1978 sk_tx_queue_clear(sk);
1979 WRITE_ONCE(sk->sk_dst_pending_confirm, 0);
1980 old_dst = rcu_dereference_protected(sk->sk_dst_cache,
1981 lockdep_sock_is_held(sk));
1982 rcu_assign_pointer(sk->sk_dst_cache, dst);
1983 dst_release(old_dst);
1984 }
1985
1986 static inline void
sk_dst_set(struct sock * sk,struct dst_entry * dst)1987 sk_dst_set(struct sock *sk, struct dst_entry *dst)
1988 {
1989 struct dst_entry *old_dst;
1990
1991 sk_tx_queue_clear(sk);
1992 WRITE_ONCE(sk->sk_dst_pending_confirm, 0);
1993 old_dst = xchg((__force struct dst_entry **)&sk->sk_dst_cache, dst);
1994 dst_release(old_dst);
1995 }
1996
1997 static inline void
__sk_dst_reset(struct sock * sk)1998 __sk_dst_reset(struct sock *sk)
1999 {
2000 __sk_dst_set(sk, NULL);
2001 }
2002
2003 static inline void
sk_dst_reset(struct sock * sk)2004 sk_dst_reset(struct sock *sk)
2005 {
2006 sk_dst_set(sk, NULL);
2007 }
2008
2009 struct dst_entry *__sk_dst_check(struct sock *sk, u32 cookie);
2010
2011 struct dst_entry *sk_dst_check(struct sock *sk, u32 cookie);
2012
sk_dst_confirm(struct sock * sk)2013 static inline void sk_dst_confirm(struct sock *sk)
2014 {
2015 if (!READ_ONCE(sk->sk_dst_pending_confirm))
2016 WRITE_ONCE(sk->sk_dst_pending_confirm, 1);
2017 }
2018
sock_confirm_neigh(struct sk_buff * skb,struct neighbour * n)2019 static inline void sock_confirm_neigh(struct sk_buff *skb, struct neighbour *n)
2020 {
2021 if (skb_get_dst_pending_confirm(skb)) {
2022 struct sock *sk = skb->sk;
2023 unsigned long now = jiffies;
2024
2025 /* avoid dirtying neighbour */
2026 if (READ_ONCE(n->confirmed) != now)
2027 WRITE_ONCE(n->confirmed, now);
2028 if (sk && READ_ONCE(sk->sk_dst_pending_confirm))
2029 WRITE_ONCE(sk->sk_dst_pending_confirm, 0);
2030 }
2031 }
2032
2033 bool sk_mc_loop(struct sock *sk);
2034
sk_can_gso(const struct sock * sk)2035 static inline bool sk_can_gso(const struct sock *sk)
2036 {
2037 return net_gso_ok(sk->sk_route_caps, sk->sk_gso_type);
2038 }
2039
2040 void sk_setup_caps(struct sock *sk, struct dst_entry *dst);
2041
sk_nocaps_add(struct sock * sk,netdev_features_t flags)2042 static inline void sk_nocaps_add(struct sock *sk, netdev_features_t flags)
2043 {
2044 sk->sk_route_nocaps |= flags;
2045 sk->sk_route_caps &= ~flags;
2046 }
2047
skb_do_copy_data_nocache(struct sock * sk,struct sk_buff * skb,struct iov_iter * from,char * to,int copy,int offset)2048 static inline int skb_do_copy_data_nocache(struct sock *sk, struct sk_buff *skb,
2049 struct iov_iter *from, char *to,
2050 int copy, int offset)
2051 {
2052 if (skb->ip_summed == CHECKSUM_NONE) {
2053 __wsum csum = 0;
2054 if (!csum_and_copy_from_iter_full(to, copy, &csum, from))
2055 return -EFAULT;
2056 skb->csum = csum_block_add(skb->csum, csum, offset);
2057 } else if (sk->sk_route_caps & NETIF_F_NOCACHE_COPY) {
2058 if (!copy_from_iter_full_nocache(to, copy, from))
2059 return -EFAULT;
2060 } else if (!copy_from_iter_full(to, copy, from))
2061 return -EFAULT;
2062
2063 return 0;
2064 }
2065
skb_add_data_nocache(struct sock * sk,struct sk_buff * skb,struct iov_iter * from,int copy)2066 static inline int skb_add_data_nocache(struct sock *sk, struct sk_buff *skb,
2067 struct iov_iter *from, int copy)
2068 {
2069 int err, offset = skb->len;
2070
2071 err = skb_do_copy_data_nocache(sk, skb, from, skb_put(skb, copy),
2072 copy, offset);
2073 if (err)
2074 __skb_trim(skb, offset);
2075
2076 return err;
2077 }
2078
skb_copy_to_page_nocache(struct sock * sk,struct iov_iter * from,struct sk_buff * skb,struct page * page,int off,int copy)2079 static inline int skb_copy_to_page_nocache(struct sock *sk, struct iov_iter *from,
2080 struct sk_buff *skb,
2081 struct page *page,
2082 int off, int copy)
2083 {
2084 int err;
2085
2086 err = skb_do_copy_data_nocache(sk, skb, from, page_address(page) + off,
2087 copy, skb->len);
2088 if (err)
2089 return err;
2090
2091 skb->len += copy;
2092 skb->data_len += copy;
2093 skb->truesize += copy;
2094 sk_wmem_queued_add(sk, copy);
2095 sk_mem_charge(sk, copy);
2096 return 0;
2097 }
2098
2099 /**
2100 * sk_wmem_alloc_get - returns write allocations
2101 * @sk: socket
2102 *
2103 * Returns sk_wmem_alloc minus initial offset of one
2104 */
sk_wmem_alloc_get(const struct sock * sk)2105 static inline int sk_wmem_alloc_get(const struct sock *sk)
2106 {
2107 return refcount_read(&sk->sk_wmem_alloc) - 1;
2108 }
2109
2110 /**
2111 * sk_rmem_alloc_get - returns read allocations
2112 * @sk: socket
2113 *
2114 * Returns sk_rmem_alloc
2115 */
sk_rmem_alloc_get(const struct sock * sk)2116 static inline int sk_rmem_alloc_get(const struct sock *sk)
2117 {
2118 return atomic_read(&sk->sk_rmem_alloc);
2119 }
2120
2121 /**
2122 * sk_has_allocations - check if allocations are outstanding
2123 * @sk: socket
2124 *
2125 * Returns true if socket has write or read allocations
2126 */
sk_has_allocations(const struct sock * sk)2127 static inline bool sk_has_allocations(const struct sock *sk)
2128 {
2129 return sk_wmem_alloc_get(sk) || sk_rmem_alloc_get(sk);
2130 }
2131
2132 /**
2133 * skwq_has_sleeper - check if there are any waiting processes
2134 * @wq: struct socket_wq
2135 *
2136 * Returns true if socket_wq has waiting processes
2137 *
2138 * The purpose of the skwq_has_sleeper and sock_poll_wait is to wrap the memory
2139 * barrier call. They were added due to the race found within the tcp code.
2140 *
2141 * Consider following tcp code paths::
2142 *
2143 * CPU1 CPU2
2144 * sys_select receive packet
2145 * ... ...
2146 * __add_wait_queue update tp->rcv_nxt
2147 * ... ...
2148 * tp->rcv_nxt check sock_def_readable
2149 * ... {
2150 * schedule rcu_read_lock();
2151 * wq = rcu_dereference(sk->sk_wq);
2152 * if (wq && waitqueue_active(&wq->wait))
2153 * wake_up_interruptible(&wq->wait)
2154 * ...
2155 * }
2156 *
2157 * The race for tcp fires when the __add_wait_queue changes done by CPU1 stay
2158 * in its cache, and so does the tp->rcv_nxt update on CPU2 side. The CPU1
2159 * could then endup calling schedule and sleep forever if there are no more
2160 * data on the socket.
2161 *
2162 */
skwq_has_sleeper(struct socket_wq * wq)2163 static inline bool skwq_has_sleeper(struct socket_wq *wq)
2164 {
2165 return wq && wq_has_sleeper(&wq->wait);
2166 }
2167
2168 /**
2169 * sock_poll_wait - place memory barrier behind the poll_wait call.
2170 * @filp: file
2171 * @sock: socket to wait on
2172 * @p: poll_table
2173 *
2174 * See the comments in the wq_has_sleeper function.
2175 */
sock_poll_wait(struct file * filp,struct socket * sock,poll_table * p)2176 static inline void sock_poll_wait(struct file *filp, struct socket *sock,
2177 poll_table *p)
2178 {
2179 if (!poll_does_not_wait(p)) {
2180 poll_wait(filp, &sock->wq.wait, p);
2181 /* We need to be sure we are in sync with the
2182 * socket flags modification.
2183 *
2184 * This memory barrier is paired in the wq_has_sleeper.
2185 */
2186 smp_mb();
2187 }
2188 }
2189
skb_set_hash_from_sk(struct sk_buff * skb,struct sock * sk)2190 static inline void skb_set_hash_from_sk(struct sk_buff *skb, struct sock *sk)
2191 {
2192 /* This pairs with WRITE_ONCE() in sk_set_txhash() */
2193 u32 txhash = READ_ONCE(sk->sk_txhash);
2194
2195 if (txhash) {
2196 skb->l4_hash = 1;
2197 skb->hash = txhash;
2198 }
2199 }
2200
2201 void skb_set_owner_w(struct sk_buff *skb, struct sock *sk);
2202
2203 /*
2204 * Queue a received datagram if it will fit. Stream and sequenced
2205 * protocols can't normally use this as they need to fit buffers in
2206 * and play with them.
2207 *
2208 * Inlined as it's very short and called for pretty much every
2209 * packet ever received.
2210 */
skb_set_owner_r(struct sk_buff * skb,struct sock * sk)2211 static inline void skb_set_owner_r(struct sk_buff *skb, struct sock *sk)
2212 {
2213 skb_orphan(skb);
2214 skb->sk = sk;
2215 skb->destructor = sock_rfree;
2216 atomic_add(skb->truesize, &sk->sk_rmem_alloc);
2217 sk_mem_charge(sk, skb->truesize);
2218 }
2219
skb_set_owner_sk_safe(struct sk_buff * skb,struct sock * sk)2220 static inline __must_check bool skb_set_owner_sk_safe(struct sk_buff *skb, struct sock *sk)
2221 {
2222 if (sk && refcount_inc_not_zero(&sk->sk_refcnt)) {
2223 skb_orphan(skb);
2224 skb->destructor = sock_efree;
2225 skb->sk = sk;
2226 return true;
2227 }
2228 return false;
2229 }
2230
skb_clone_and_charge_r(struct sk_buff * skb,struct sock * sk)2231 static inline struct sk_buff *skb_clone_and_charge_r(struct sk_buff *skb, struct sock *sk)
2232 {
2233 skb = skb_clone(skb, sk_gfp_mask(sk, GFP_ATOMIC));
2234 if (skb) {
2235 if (sk_rmem_schedule(sk, skb, skb->truesize)) {
2236 skb_set_owner_r(skb, sk);
2237 return skb;
2238 }
2239 __kfree_skb(skb);
2240 }
2241 return NULL;
2242 }
2243
2244 void sk_reset_timer(struct sock *sk, struct timer_list *timer,
2245 unsigned long expires);
2246
2247 void sk_stop_timer(struct sock *sk, struct timer_list *timer);
2248
2249 void sk_stop_timer_sync(struct sock *sk, struct timer_list *timer);
2250
2251 int __sk_queue_drop_skb(struct sock *sk, struct sk_buff_head *sk_queue,
2252 struct sk_buff *skb, unsigned int flags,
2253 void (*destructor)(struct sock *sk,
2254 struct sk_buff *skb));
2255 int __sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb);
2256 int sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb);
2257
2258 int sock_queue_err_skb(struct sock *sk, struct sk_buff *skb);
2259 struct sk_buff *sock_dequeue_err_skb(struct sock *sk);
2260
2261 /*
2262 * Recover an error report and clear atomically
2263 */
2264
sock_error(struct sock * sk)2265 static inline int sock_error(struct sock *sk)
2266 {
2267 int err;
2268 if (likely(!sk->sk_err))
2269 return 0;
2270 err = xchg(&sk->sk_err, 0);
2271 return -err;
2272 }
2273
sock_wspace(struct sock * sk)2274 static inline unsigned long sock_wspace(struct sock *sk)
2275 {
2276 int amt = 0;
2277
2278 if (!(sk->sk_shutdown & SEND_SHUTDOWN)) {
2279 amt = sk->sk_sndbuf - refcount_read(&sk->sk_wmem_alloc);
2280 if (amt < 0)
2281 amt = 0;
2282 }
2283 return amt;
2284 }
2285
2286 /* Note:
2287 * We use sk->sk_wq_raw, from contexts knowing this
2288 * pointer is not NULL and cannot disappear/change.
2289 */
sk_set_bit(int nr,struct sock * sk)2290 static inline void sk_set_bit(int nr, struct sock *sk)
2291 {
2292 if ((nr == SOCKWQ_ASYNC_NOSPACE || nr == SOCKWQ_ASYNC_WAITDATA) &&
2293 !sock_flag(sk, SOCK_FASYNC))
2294 return;
2295
2296 set_bit(nr, &sk->sk_wq_raw->flags);
2297 }
2298
sk_clear_bit(int nr,struct sock * sk)2299 static inline void sk_clear_bit(int nr, struct sock *sk)
2300 {
2301 if ((nr == SOCKWQ_ASYNC_NOSPACE || nr == SOCKWQ_ASYNC_WAITDATA) &&
2302 !sock_flag(sk, SOCK_FASYNC))
2303 return;
2304
2305 clear_bit(nr, &sk->sk_wq_raw->flags);
2306 }
2307
sk_wake_async(const struct sock * sk,int how,int band)2308 static inline void sk_wake_async(const struct sock *sk, int how, int band)
2309 {
2310 if (sock_flag(sk, SOCK_FASYNC)) {
2311 rcu_read_lock();
2312 sock_wake_async(rcu_dereference(sk->sk_wq), how, band);
2313 rcu_read_unlock();
2314 }
2315 }
2316
2317 /* Since sk_{r,w}mem_alloc sums skb->truesize, even a small frame might
2318 * need sizeof(sk_buff) + MTU + padding, unless net driver perform copybreak.
2319 * Note: for send buffers, TCP works better if we can build two skbs at
2320 * minimum.
2321 */
2322 #define TCP_SKB_MIN_TRUESIZE (2048 + SKB_DATA_ALIGN(sizeof(struct sk_buff)))
2323
2324 #define SOCK_MIN_SNDBUF (TCP_SKB_MIN_TRUESIZE * 2)
2325 #define SOCK_MIN_RCVBUF TCP_SKB_MIN_TRUESIZE
2326
sk_stream_moderate_sndbuf(struct sock * sk)2327 static inline void sk_stream_moderate_sndbuf(struct sock *sk)
2328 {
2329 u32 val;
2330
2331 if (sk->sk_userlocks & SOCK_SNDBUF_LOCK)
2332 return;
2333
2334 val = min(sk->sk_sndbuf, sk->sk_wmem_queued >> 1);
2335
2336 WRITE_ONCE(sk->sk_sndbuf, max_t(u32, val, SOCK_MIN_SNDBUF));
2337 }
2338
2339 struct sk_buff *sk_stream_alloc_skb(struct sock *sk, int size, gfp_t gfp,
2340 bool force_schedule);
2341
2342 /**
2343 * sk_page_frag - return an appropriate page_frag
2344 * @sk: socket
2345 *
2346 * Use the per task page_frag instead of the per socket one for
2347 * optimization when we know that we're in the normal context and owns
2348 * everything that's associated with %current.
2349 *
2350 * gfpflags_allow_blocking() isn't enough here as direct reclaim may nest
2351 * inside other socket operations and end up recursing into sk_page_frag()
2352 * while it's already in use.
2353 */
sk_page_frag(struct sock * sk)2354 static inline struct page_frag *sk_page_frag(struct sock *sk)
2355 {
2356 if (gfpflags_normal_context(sk->sk_allocation))
2357 return ¤t->task_frag;
2358
2359 return &sk->sk_frag;
2360 }
2361
2362 bool sk_page_frag_refill(struct sock *sk, struct page_frag *pfrag);
2363
2364 /*
2365 * Default write policy as shown to user space via poll/select/SIGIO
2366 */
sock_writeable(const struct sock * sk)2367 static inline bool sock_writeable(const struct sock *sk)
2368 {
2369 return refcount_read(&sk->sk_wmem_alloc) < (READ_ONCE(sk->sk_sndbuf) >> 1);
2370 }
2371
gfp_any(void)2372 static inline gfp_t gfp_any(void)
2373 {
2374 return in_softirq() ? GFP_ATOMIC : GFP_KERNEL;
2375 }
2376
sock_rcvtimeo(const struct sock * sk,bool noblock)2377 static inline long sock_rcvtimeo(const struct sock *sk, bool noblock)
2378 {
2379 return noblock ? 0 : sk->sk_rcvtimeo;
2380 }
2381
sock_sndtimeo(const struct sock * sk,bool noblock)2382 static inline long sock_sndtimeo(const struct sock *sk, bool noblock)
2383 {
2384 return noblock ? 0 : sk->sk_sndtimeo;
2385 }
2386
sock_rcvlowat(const struct sock * sk,int waitall,int len)2387 static inline int sock_rcvlowat(const struct sock *sk, int waitall, int len)
2388 {
2389 int v = waitall ? len : min_t(int, READ_ONCE(sk->sk_rcvlowat), len);
2390
2391 return v ?: 1;
2392 }
2393
2394 /* Alas, with timeout socket operations are not restartable.
2395 * Compare this to poll().
2396 */
sock_intr_errno(long timeo)2397 static inline int sock_intr_errno(long timeo)
2398 {
2399 return timeo == MAX_SCHEDULE_TIMEOUT ? -ERESTARTSYS : -EINTR;
2400 }
2401
2402 struct sock_skb_cb {
2403 u32 dropcount;
2404 };
2405
2406 /* Store sock_skb_cb at the end of skb->cb[] so protocol families
2407 * using skb->cb[] would keep using it directly and utilize its
2408 * alignement guarantee.
2409 */
2410 #define SOCK_SKB_CB_OFFSET ((FIELD_SIZEOF(struct sk_buff, cb) - \
2411 sizeof(struct sock_skb_cb)))
2412
2413 #define SOCK_SKB_CB(__skb) ((struct sock_skb_cb *)((__skb)->cb + \
2414 SOCK_SKB_CB_OFFSET))
2415
2416 #define sock_skb_cb_check_size(size) \
2417 BUILD_BUG_ON((size) > SOCK_SKB_CB_OFFSET)
2418
2419 static inline void
sock_skb_set_dropcount(const struct sock * sk,struct sk_buff * skb)2420 sock_skb_set_dropcount(const struct sock *sk, struct sk_buff *skb)
2421 {
2422 SOCK_SKB_CB(skb)->dropcount = sock_flag(sk, SOCK_RXQ_OVFL) ?
2423 atomic_read(&sk->sk_drops) : 0;
2424 }
2425
sk_drops_add(struct sock * sk,const struct sk_buff * skb)2426 static inline void sk_drops_add(struct sock *sk, const struct sk_buff *skb)
2427 {
2428 int segs = max_t(u16, 1, skb_shinfo(skb)->gso_segs);
2429
2430 atomic_add(segs, &sk->sk_drops);
2431 }
2432
sock_read_timestamp(struct sock * sk)2433 static inline ktime_t sock_read_timestamp(struct sock *sk)
2434 {
2435 #if BITS_PER_LONG==32
2436 unsigned int seq;
2437 ktime_t kt;
2438
2439 do {
2440 seq = read_seqbegin(&sk->sk_stamp_seq);
2441 kt = sk->sk_stamp;
2442 } while (read_seqretry(&sk->sk_stamp_seq, seq));
2443
2444 return kt;
2445 #else
2446 return READ_ONCE(sk->sk_stamp);
2447 #endif
2448 }
2449
sock_write_timestamp(struct sock * sk,ktime_t kt)2450 static inline void sock_write_timestamp(struct sock *sk, ktime_t kt)
2451 {
2452 #if BITS_PER_LONG==32
2453 write_seqlock(&sk->sk_stamp_seq);
2454 sk->sk_stamp = kt;
2455 write_sequnlock(&sk->sk_stamp_seq);
2456 #else
2457 WRITE_ONCE(sk->sk_stamp, kt);
2458 #endif
2459 }
2460
2461 void __sock_recv_timestamp(struct msghdr *msg, struct sock *sk,
2462 struct sk_buff *skb);
2463 void __sock_recv_wifi_status(struct msghdr *msg, struct sock *sk,
2464 struct sk_buff *skb);
2465
2466 static inline void
sock_recv_timestamp(struct msghdr * msg,struct sock * sk,struct sk_buff * skb)2467 sock_recv_timestamp(struct msghdr *msg, struct sock *sk, struct sk_buff *skb)
2468 {
2469 ktime_t kt = skb->tstamp;
2470 struct skb_shared_hwtstamps *hwtstamps = skb_hwtstamps(skb);
2471
2472 /*
2473 * generate control messages if
2474 * - receive time stamping in software requested
2475 * - software time stamp available and wanted
2476 * - hardware time stamps available and wanted
2477 */
2478 if (sock_flag(sk, SOCK_RCVTSTAMP) ||
2479 (sk->sk_tsflags & SOF_TIMESTAMPING_RX_SOFTWARE) ||
2480 (kt && sk->sk_tsflags & SOF_TIMESTAMPING_SOFTWARE) ||
2481 (hwtstamps->hwtstamp &&
2482 (sk->sk_tsflags & SOF_TIMESTAMPING_RAW_HARDWARE)))
2483 __sock_recv_timestamp(msg, sk, skb);
2484 else
2485 sock_write_timestamp(sk, kt);
2486
2487 if (sock_flag(sk, SOCK_WIFI_STATUS) && skb->wifi_acked_valid)
2488 __sock_recv_wifi_status(msg, sk, skb);
2489 }
2490
2491 void __sock_recv_ts_and_drops(struct msghdr *msg, struct sock *sk,
2492 struct sk_buff *skb);
2493
2494 #define SK_DEFAULT_STAMP (-1L * NSEC_PER_SEC)
sock_recv_ts_and_drops(struct msghdr * msg,struct sock * sk,struct sk_buff * skb)2495 static inline void sock_recv_ts_and_drops(struct msghdr *msg, struct sock *sk,
2496 struct sk_buff *skb)
2497 {
2498 #define FLAGS_TS_OR_DROPS ((1UL << SOCK_RXQ_OVFL) | \
2499 (1UL << SOCK_RCVTSTAMP))
2500 #define TSFLAGS_ANY (SOF_TIMESTAMPING_SOFTWARE | \
2501 SOF_TIMESTAMPING_RAW_HARDWARE)
2502
2503 if (sk->sk_flags & FLAGS_TS_OR_DROPS || sk->sk_tsflags & TSFLAGS_ANY)
2504 __sock_recv_ts_and_drops(msg, sk, skb);
2505 else if (unlikely(sock_flag(sk, SOCK_TIMESTAMP)))
2506 sock_write_timestamp(sk, skb->tstamp);
2507 else if (unlikely(sock_read_timestamp(sk) == SK_DEFAULT_STAMP))
2508 sock_write_timestamp(sk, 0);
2509 }
2510
2511 void __sock_tx_timestamp(__u16 tsflags, __u8 *tx_flags);
2512
2513 /**
2514 * _sock_tx_timestamp - checks whether the outgoing packet is to be time stamped
2515 * @sk: socket sending this packet
2516 * @tsflags: timestamping flags to use
2517 * @tx_flags: completed with instructions for time stamping
2518 * @tskey: filled in with next sk_tskey (not for TCP, which uses seqno)
2519 *
2520 * Note: callers should take care of initial ``*tx_flags`` value (usually 0)
2521 */
_sock_tx_timestamp(struct sock * sk,__u16 tsflags,__u8 * tx_flags,__u32 * tskey)2522 static inline void _sock_tx_timestamp(struct sock *sk, __u16 tsflags,
2523 __u8 *tx_flags, __u32 *tskey)
2524 {
2525 if (unlikely(tsflags)) {
2526 __sock_tx_timestamp(tsflags, tx_flags);
2527 if (tsflags & SOF_TIMESTAMPING_OPT_ID && tskey &&
2528 tsflags & SOF_TIMESTAMPING_TX_RECORD_MASK)
2529 *tskey = sk->sk_tskey++;
2530 }
2531 if (unlikely(sock_flag(sk, SOCK_WIFI_STATUS)))
2532 *tx_flags |= SKBTX_WIFI_STATUS;
2533 }
2534
sock_tx_timestamp(struct sock * sk,__u16 tsflags,__u8 * tx_flags)2535 static inline void sock_tx_timestamp(struct sock *sk, __u16 tsflags,
2536 __u8 *tx_flags)
2537 {
2538 _sock_tx_timestamp(sk, tsflags, tx_flags, NULL);
2539 }
2540
skb_setup_tx_timestamp(struct sk_buff * skb,__u16 tsflags)2541 static inline void skb_setup_tx_timestamp(struct sk_buff *skb, __u16 tsflags)
2542 {
2543 _sock_tx_timestamp(skb->sk, tsflags, &skb_shinfo(skb)->tx_flags,
2544 &skb_shinfo(skb)->tskey);
2545 }
2546
2547 /**
2548 * sk_eat_skb - Release a skb if it is no longer needed
2549 * @sk: socket to eat this skb from
2550 * @skb: socket buffer to eat
2551 *
2552 * This routine must be called with interrupts disabled or with the socket
2553 * locked so that the sk_buff queue operation is ok.
2554 */
2555 DECLARE_STATIC_KEY_FALSE(tcp_rx_skb_cache_key);
sk_eat_skb(struct sock * sk,struct sk_buff * skb)2556 static inline void sk_eat_skb(struct sock *sk, struct sk_buff *skb)
2557 {
2558 __skb_unlink(skb, &sk->sk_receive_queue);
2559 if (static_branch_unlikely(&tcp_rx_skb_cache_key) &&
2560 !sk->sk_rx_skb_cache) {
2561 sk->sk_rx_skb_cache = skb;
2562 skb_orphan(skb);
2563 return;
2564 }
2565 __kfree_skb(skb);
2566 }
2567
2568 static inline
sock_net(const struct sock * sk)2569 struct net *sock_net(const struct sock *sk)
2570 {
2571 return read_pnet(&sk->sk_net);
2572 }
2573
2574 static inline
sock_net_set(struct sock * sk,struct net * net)2575 void sock_net_set(struct sock *sk, struct net *net)
2576 {
2577 write_pnet(&sk->sk_net, net);
2578 }
2579
skb_steal_sock(struct sk_buff * skb)2580 static inline struct sock *skb_steal_sock(struct sk_buff *skb)
2581 {
2582 if (skb->sk) {
2583 struct sock *sk = skb->sk;
2584
2585 skb->destructor = NULL;
2586 skb->sk = NULL;
2587 return sk;
2588 }
2589 return NULL;
2590 }
2591
2592 /* This helper checks if a socket is a full socket,
2593 * ie _not_ a timewait or request socket.
2594 */
sk_fullsock(const struct sock * sk)2595 static inline bool sk_fullsock(const struct sock *sk)
2596 {
2597 return (1 << sk->sk_state) & ~(TCPF_TIME_WAIT | TCPF_NEW_SYN_RECV);
2598 }
2599
2600 /* Checks if this SKB belongs to an HW offloaded socket
2601 * and whether any SW fallbacks are required based on dev.
2602 * Check decrypted mark in case skb_orphan() cleared socket.
2603 */
sk_validate_xmit_skb(struct sk_buff * skb,struct net_device * dev)2604 static inline struct sk_buff *sk_validate_xmit_skb(struct sk_buff *skb,
2605 struct net_device *dev)
2606 {
2607 #ifdef CONFIG_SOCK_VALIDATE_XMIT
2608 struct sock *sk = skb->sk;
2609
2610 if (sk && sk_fullsock(sk) && sk->sk_validate_xmit_skb) {
2611 skb = sk->sk_validate_xmit_skb(sk, dev, skb);
2612 #ifdef CONFIG_TLS_DEVICE
2613 } else if (unlikely(skb->decrypted)) {
2614 pr_warn_ratelimited("unencrypted skb with no associated socket - dropping\n");
2615 kfree_skb(skb);
2616 skb = NULL;
2617 #endif
2618 }
2619 #endif
2620
2621 return skb;
2622 }
2623
2624 /* This helper checks if a socket is a LISTEN or NEW_SYN_RECV
2625 * SYNACK messages can be attached to either ones (depending on SYNCOOKIE)
2626 */
sk_listener(const struct sock * sk)2627 static inline bool sk_listener(const struct sock *sk)
2628 {
2629 return (1 << sk->sk_state) & (TCPF_LISTEN | TCPF_NEW_SYN_RECV);
2630 }
2631
2632 void sock_enable_timestamp(struct sock *sk, int flag);
2633 int sock_recv_errqueue(struct sock *sk, struct msghdr *msg, int len, int level,
2634 int type);
2635
2636 bool sk_ns_capable(const struct sock *sk,
2637 struct user_namespace *user_ns, int cap);
2638 bool sk_capable(const struct sock *sk, int cap);
2639 bool sk_net_capable(const struct sock *sk, int cap);
2640
2641 void sk_get_meminfo(const struct sock *sk, u32 *meminfo);
2642
2643 /* Take into consideration the size of the struct sk_buff overhead in the
2644 * determination of these values, since that is non-constant across
2645 * platforms. This makes socket queueing behavior and performance
2646 * not depend upon such differences.
2647 */
2648 #define _SK_MEM_PACKETS 256
2649 #define _SK_MEM_OVERHEAD SKB_TRUESIZE(256)
2650 #define SK_WMEM_MAX (_SK_MEM_OVERHEAD * _SK_MEM_PACKETS)
2651 #define SK_RMEM_MAX (_SK_MEM_OVERHEAD * _SK_MEM_PACKETS)
2652
2653 extern __u32 sysctl_wmem_max;
2654 extern __u32 sysctl_rmem_max;
2655
2656 extern int sysctl_tstamp_allow_data;
2657 extern int sysctl_optmem_max;
2658
2659 extern __u32 sysctl_wmem_default;
2660 extern __u32 sysctl_rmem_default;
2661
2662
2663 /* On 32bit arches, an skb frag is limited to 2^15 */
2664 #define SKB_FRAG_PAGE_ORDER get_order(32768)
2665 DECLARE_STATIC_KEY_FALSE(net_high_order_alloc_disable_key);
2666
sk_get_wmem0(const struct sock * sk,const struct proto * proto)2667 static inline int sk_get_wmem0(const struct sock *sk, const struct proto *proto)
2668 {
2669 /* Does this proto have per netns sysctl_wmem ? */
2670 if (proto->sysctl_wmem_offset)
2671 return *(int *)((void *)sock_net(sk) + proto->sysctl_wmem_offset);
2672
2673 return *proto->sysctl_wmem;
2674 }
2675
sk_get_rmem0(const struct sock * sk,const struct proto * proto)2676 static inline int sk_get_rmem0(const struct sock *sk, const struct proto *proto)
2677 {
2678 /* Does this proto have per netns sysctl_rmem ? */
2679 if (proto->sysctl_rmem_offset)
2680 return *(int *)((void *)sock_net(sk) + proto->sysctl_rmem_offset);
2681
2682 return *proto->sysctl_rmem;
2683 }
2684
2685 /* Default TCP Small queue budget is ~1 ms of data (1sec >> 10)
2686 * Some wifi drivers need to tweak it to get more chunks.
2687 * They can use this helper from their ndo_start_xmit()
2688 */
sk_pacing_shift_update(struct sock * sk,int val)2689 static inline void sk_pacing_shift_update(struct sock *sk, int val)
2690 {
2691 if (!sk || !sk_fullsock(sk) || READ_ONCE(sk->sk_pacing_shift) == val)
2692 return;
2693 WRITE_ONCE(sk->sk_pacing_shift, val);
2694 }
2695
2696 /* if a socket is bound to a device, check that the given device
2697 * index is either the same or that the socket is bound to an L3
2698 * master device and the given device index is also enslaved to
2699 * that L3 master
2700 */
sk_dev_equal_l3scope(struct sock * sk,int dif)2701 static inline bool sk_dev_equal_l3scope(struct sock *sk, int dif)
2702 {
2703 int mdif;
2704
2705 if (!sk->sk_bound_dev_if || sk->sk_bound_dev_if == dif)
2706 return true;
2707
2708 mdif = l3mdev_master_ifindex_by_index(sock_net(sk), dif);
2709 if (mdif && mdif == sk->sk_bound_dev_if)
2710 return true;
2711
2712 return false;
2713 }
2714
2715 #endif /* _SOCK_H */
2716