• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * INET		An implementation of the TCP/IP protocol suite for the LINUX
3  *		operating system.  INET is implemented using the  BSD Socket
4  *		interface as the means of communication with the user level.
5  *
6  *		Implementation of the Transmission Control Protocol(TCP).
7  *
8  * Authors:	Ross Biro
9  *		Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
10  *		Mark Evans, <evansmp@uhura.aston.ac.uk>
11  *		Corey Minyard <wf-rch!minyard@relay.EU.net>
12  *		Florian La Roche, <flla@stud.uni-sb.de>
13  *		Charles Hedrick, <hedrick@klinzhai.rutgers.edu>
14  *		Linus Torvalds, <torvalds@cs.helsinki.fi>
15  *		Alan Cox, <gw4pts@gw4pts.ampr.org>
16  *		Matthew Dillon, <dillon@apollo.west.oic.com>
17  *		Arnt Gulbrandsen, <agulbra@nvg.unit.no>
18  *		Jorge Cwik, <jorge@laser.satlink.net>
19  *
20  * Fixes:
21  *		Alan Cox	:	Numerous verify_area() calls
22  *		Alan Cox	:	Set the ACK bit on a reset
23  *		Alan Cox	:	Stopped it crashing if it closed while
24  *					sk->inuse=1 and was trying to connect
25  *					(tcp_err()).
26  *		Alan Cox	:	All icmp error handling was broken
27  *					pointers passed where wrong and the
28  *					socket was looked up backwards. Nobody
29  *					tested any icmp error code obviously.
30  *		Alan Cox	:	tcp_err() now handled properly. It
31  *					wakes people on errors. poll
32  *					behaves and the icmp error race
33  *					has gone by moving it into sock.c
34  *		Alan Cox	:	tcp_send_reset() fixed to work for
35  *					everything not just packets for
36  *					unknown sockets.
37  *		Alan Cox	:	tcp option processing.
38  *		Alan Cox	:	Reset tweaked (still not 100%) [Had
39  *					syn rule wrong]
40  *		Herp Rosmanith  :	More reset fixes
41  *		Alan Cox	:	No longer acks invalid rst frames.
42  *					Acking any kind of RST is right out.
43  *		Alan Cox	:	Sets an ignore me flag on an rst
44  *					receive otherwise odd bits of prattle
45  *					escape still
46  *		Alan Cox	:	Fixed another acking RST frame bug.
47  *					Should stop LAN workplace lockups.
48  *		Alan Cox	: 	Some tidyups using the new skb list
49  *					facilities
50  *		Alan Cox	:	sk->keepopen now seems to work
51  *		Alan Cox	:	Pulls options out correctly on accepts
52  *		Alan Cox	:	Fixed assorted sk->rqueue->next errors
53  *		Alan Cox	:	PSH doesn't end a TCP read. Switched a
54  *					bit to skb ops.
55  *		Alan Cox	:	Tidied tcp_data to avoid a potential
56  *					nasty.
57  *		Alan Cox	:	Added some better commenting, as the
58  *					tcp is hard to follow
59  *		Alan Cox	:	Removed incorrect check for 20 * psh
60  *	Michael O'Reilly	:	ack < copied bug fix.
61  *	Johannes Stille		:	Misc tcp fixes (not all in yet).
62  *		Alan Cox	:	FIN with no memory -> CRASH
63  *		Alan Cox	:	Added socket option proto entries.
64  *					Also added awareness of them to accept.
65  *		Alan Cox	:	Added TCP options (SOL_TCP)
66  *		Alan Cox	:	Switched wakeup calls to callbacks,
67  *					so the kernel can layer network
68  *					sockets.
69  *		Alan Cox	:	Use ip_tos/ip_ttl settings.
70  *		Alan Cox	:	Handle FIN (more) properly (we hope).
71  *		Alan Cox	:	RST frames sent on unsynchronised
72  *					state ack error.
73  *		Alan Cox	:	Put in missing check for SYN bit.
74  *		Alan Cox	:	Added tcp_select_window() aka NET2E
75  *					window non shrink trick.
76  *		Alan Cox	:	Added a couple of small NET2E timer
77  *					fixes
78  *		Charles Hedrick :	TCP fixes
79  *		Toomas Tamm	:	TCP window fixes
80  *		Alan Cox	:	Small URG fix to rlogin ^C ack fight
81  *		Charles Hedrick	:	Rewrote most of it to actually work
82  *		Linus		:	Rewrote tcp_read() and URG handling
83  *					completely
84  *		Gerhard Koerting:	Fixed some missing timer handling
85  *		Matthew Dillon  :	Reworked TCP machine states as per RFC
86  *		Gerhard Koerting:	PC/TCP workarounds
87  *		Adam Caldwell	:	Assorted timer/timing errors
88  *		Matthew Dillon	:	Fixed another RST bug
89  *		Alan Cox	:	Move to kernel side addressing changes.
90  *		Alan Cox	:	Beginning work on TCP fastpathing
91  *					(not yet usable)
92  *		Arnt Gulbrandsen:	Turbocharged tcp_check() routine.
93  *		Alan Cox	:	TCP fast path debugging
94  *		Alan Cox	:	Window clamping
95  *		Michael Riepe	:	Bug in tcp_check()
96  *		Matt Dillon	:	More TCP improvements and RST bug fixes
97  *		Matt Dillon	:	Yet more small nasties remove from the
98  *					TCP code (Be very nice to this man if
99  *					tcp finally works 100%) 8)
100  *		Alan Cox	:	BSD accept semantics.
101  *		Alan Cox	:	Reset on closedown bug.
102  *	Peter De Schrijver	:	ENOTCONN check missing in tcp_sendto().
103  *		Michael Pall	:	Handle poll() after URG properly in
104  *					all cases.
105  *		Michael Pall	:	Undo the last fix in tcp_read_urg()
106  *					(multi URG PUSH broke rlogin).
107  *		Michael Pall	:	Fix the multi URG PUSH problem in
108  *					tcp_readable(), poll() after URG
109  *					works now.
110  *		Michael Pall	:	recv(...,MSG_OOB) never blocks in the
111  *					BSD api.
112  *		Alan Cox	:	Changed the semantics of sk->socket to
113  *					fix a race and a signal problem with
114  *					accept() and async I/O.
115  *		Alan Cox	:	Relaxed the rules on tcp_sendto().
116  *		Yury Shevchuk	:	Really fixed accept() blocking problem.
117  *		Craig I. Hagan  :	Allow for BSD compatible TIME_WAIT for
118  *					clients/servers which listen in on
119  *					fixed ports.
120  *		Alan Cox	:	Cleaned the above up and shrank it to
121  *					a sensible code size.
122  *		Alan Cox	:	Self connect lockup fix.
123  *		Alan Cox	:	No connect to multicast.
124  *		Ross Biro	:	Close unaccepted children on master
125  *					socket close.
126  *		Alan Cox	:	Reset tracing code.
127  *		Alan Cox	:	Spurious resets on shutdown.
128  *		Alan Cox	:	Giant 15 minute/60 second timer error
129  *		Alan Cox	:	Small whoops in polling before an
130  *					accept.
131  *		Alan Cox	:	Kept the state trace facility since
132  *					it's handy for debugging.
133  *		Alan Cox	:	More reset handler fixes.
134  *		Alan Cox	:	Started rewriting the code based on
135  *					the RFC's for other useful protocol
136  *					references see: Comer, KA9Q NOS, and
137  *					for a reference on the difference
138  *					between specifications and how BSD
139  *					works see the 4.4lite source.
140  *		A.N.Kuznetsov	:	Don't time wait on completion of tidy
141  *					close.
142  *		Linus Torvalds	:	Fin/Shutdown & copied_seq changes.
143  *		Linus Torvalds	:	Fixed BSD port reuse to work first syn
144  *		Alan Cox	:	Reimplemented timers as per the RFC
145  *					and using multiple timers for sanity.
146  *		Alan Cox	:	Small bug fixes, and a lot of new
147  *					comments.
148  *		Alan Cox	:	Fixed dual reader crash by locking
149  *					the buffers (much like datagram.c)
150  *		Alan Cox	:	Fixed stuck sockets in probe. A probe
151  *					now gets fed up of retrying without
152  *					(even a no space) answer.
153  *		Alan Cox	:	Extracted closing code better
154  *		Alan Cox	:	Fixed the closing state machine to
155  *					resemble the RFC.
156  *		Alan Cox	:	More 'per spec' fixes.
157  *		Jorge Cwik	:	Even faster checksumming.
158  *		Alan Cox	:	tcp_data() doesn't ack illegal PSH
159  *					only frames. At least one pc tcp stack
160  *					generates them.
161  *		Alan Cox	:	Cache last socket.
162  *		Alan Cox	:	Per route irtt.
163  *		Matt Day	:	poll()->select() match BSD precisely on error
164  *		Alan Cox	:	New buffers
165  *		Marc Tamsky	:	Various sk->prot->retransmits and
166  *					sk->retransmits misupdating fixed.
167  *					Fixed tcp_write_timeout: stuck close,
168  *					and TCP syn retries gets used now.
169  *		Mark Yarvis	:	In tcp_read_wakeup(), don't send an
170  *					ack if state is TCP_CLOSED.
171  *		Alan Cox	:	Look up device on a retransmit - routes may
172  *					change. Doesn't yet cope with MSS shrink right
173  *					but it's a start!
174  *		Marc Tamsky	:	Closing in closing fixes.
175  *		Mike Shaver	:	RFC1122 verifications.
176  *		Alan Cox	:	rcv_saddr errors.
177  *		Alan Cox	:	Block double connect().
178  *		Alan Cox	:	Small hooks for enSKIP.
179  *		Alexey Kuznetsov:	Path MTU discovery.
180  *		Alan Cox	:	Support soft errors.
181  *		Alan Cox	:	Fix MTU discovery pathological case
182  *					when the remote claims no mtu!
183  *		Marc Tamsky	:	TCP_CLOSE fix.
184  *		Colin (G3TNE)	:	Send a reset on syn ack replies in
185  *					window but wrong (fixes NT lpd problems)
186  *		Pedro Roque	:	Better TCP window handling, delayed ack.
187  *		Joerg Reuter	:	No modification of locked buffers in
188  *					tcp_do_retransmit()
189  *		Eric Schenk	:	Changed receiver side silly window
190  *					avoidance algorithm to BSD style
191  *					algorithm. This doubles throughput
192  *					against machines running Solaris,
193  *					and seems to result in general
194  *					improvement.
195  *	Stefan Magdalinski	:	adjusted tcp_readable() to fix FIONREAD
196  *	Willy Konynenberg	:	Transparent proxying support.
197  *	Mike McLagan		:	Routing by source
198  *		Keith Owens	:	Do proper merging with partial SKB's in
199  *					tcp_do_sendmsg to avoid burstiness.
200  *		Eric Schenk	:	Fix fast close down bug with
201  *					shutdown() followed by close().
202  *		Andi Kleen 	:	Make poll agree with SIGIO
203  *	Salvatore Sanfilippo	:	Support SO_LINGER with linger == 1 and
204  *					lingertime == 0 (RFC 793 ABORT Call)
205  *	Hirokazu Takahashi	:	Use copy_from_user() instead of
206  *					csum_and_copy_from_user() if possible.
207  *
208  *		This program is free software; you can redistribute it and/or
209  *		modify it under the terms of the GNU General Public License
210  *		as published by the Free Software Foundation; either version
211  *		2 of the License, or(at your option) any later version.
212  *
213  * Description of States:
214  *
215  *	TCP_SYN_SENT		sent a connection request, waiting for ack
216  *
217  *	TCP_SYN_RECV		received a connection request, sent ack,
218  *				waiting for final ack in three-way handshake.
219  *
220  *	TCP_ESTABLISHED		connection established
221  *
222  *	TCP_FIN_WAIT1		our side has shutdown, waiting to complete
223  *				transmission of remaining buffered data
224  *
225  *	TCP_FIN_WAIT2		all buffered data sent, waiting for remote
226  *				to shutdown
227  *
228  *	TCP_CLOSING		both sides have shutdown but we still have
229  *				data we have to finish sending
230  *
231  *	TCP_TIME_WAIT		timeout to catch resent junk before entering
232  *				closed, can only be entered from FIN_WAIT2
233  *				or CLOSING.  Required because the other end
234  *				may not have gotten our last ACK causing it
235  *				to retransmit the data packet (which we ignore)
236  *
237  *	TCP_CLOSE_WAIT		remote side has shutdown and is waiting for
238  *				us to finish writing our data and to shutdown
239  *				(we have to close() to move on to LAST_ACK)
240  *
241  *	TCP_LAST_ACK		out side has shutdown after remote has
242  *				shutdown.  There may still be data in our
243  *				buffer that we have to finish sending
244  *
245  *	TCP_CLOSE		socket is finished
246  */
247 
248 #define pr_fmt(fmt) "TCP: " fmt
249 
250 #include <linux/kernel.h>
251 #include <linux/module.h>
252 #include <linux/types.h>
253 #include <linux/fcntl.h>
254 #include <linux/poll.h>
255 #include <linux/inet_diag.h>
256 #include <linux/init.h>
257 #include <linux/fs.h>
258 #include <linux/skbuff.h>
259 #include <linux/scatterlist.h>
260 #include <linux/splice.h>
261 #include <linux/net.h>
262 #include <linux/socket.h>
263 #include <linux/random.h>
264 #include <linux/bootmem.h>
265 #include <linux/highmem.h>
266 #include <linux/swap.h>
267 #include <linux/cache.h>
268 #include <linux/err.h>
269 #include <linux/crypto.h>
270 #include <linux/time.h>
271 #include <linux/slab.h>
272 
273 #include <net/icmp.h>
274 #include <net/inet_common.h>
275 #include <net/tcp.h>
276 #include <net/xfrm.h>
277 #include <net/ip.h>
278 #include <net/sock.h>
279 
280 #include <asm/uaccess.h>
281 #include <asm/ioctls.h>
282 #include <asm/unaligned.h>
283 #include <net/busy_poll.h>
284 
285 int sysctl_tcp_fin_timeout __read_mostly = TCP_FIN_TIMEOUT;
286 
287 int sysctl_tcp_min_tso_segs __read_mostly = 2;
288 
289 int sysctl_tcp_autocorking __read_mostly = 1;
290 
291 struct percpu_counter tcp_orphan_count;
292 EXPORT_SYMBOL_GPL(tcp_orphan_count);
293 
294 long sysctl_tcp_mem[3] __read_mostly;
295 int sysctl_tcp_wmem[3] __read_mostly;
296 int sysctl_tcp_rmem[3] __read_mostly;
297 
298 EXPORT_SYMBOL(sysctl_tcp_mem);
299 EXPORT_SYMBOL(sysctl_tcp_rmem);
300 EXPORT_SYMBOL(sysctl_tcp_wmem);
301 
302 atomic_long_t tcp_memory_allocated;	/* Current allocated memory. */
303 EXPORT_SYMBOL(tcp_memory_allocated);
304 
305 /*
306  * Current number of TCP sockets.
307  */
308 struct percpu_counter tcp_sockets_allocated;
309 EXPORT_SYMBOL(tcp_sockets_allocated);
310 
311 /*
312  * TCP splice context
313  */
314 struct tcp_splice_state {
315 	struct pipe_inode_info *pipe;
316 	size_t len;
317 	unsigned int flags;
318 };
319 
320 /*
321  * Pressure flag: try to collapse.
322  * Technical note: it is used by multiple contexts non atomically.
323  * All the __sk_mem_schedule() is of this nature: accounting
324  * is strict, actions are advisory and have some latency.
325  */
326 int tcp_memory_pressure __read_mostly;
327 EXPORT_SYMBOL(tcp_memory_pressure);
328 
tcp_enter_memory_pressure(struct sock * sk)329 void tcp_enter_memory_pressure(struct sock *sk)
330 {
331 	if (!tcp_memory_pressure) {
332 		NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMEMORYPRESSURES);
333 		tcp_memory_pressure = 1;
334 	}
335 }
336 EXPORT_SYMBOL(tcp_enter_memory_pressure);
337 
338 /* Convert seconds to retransmits based on initial and max timeout */
secs_to_retrans(int seconds,int timeout,int rto_max)339 static u8 secs_to_retrans(int seconds, int timeout, int rto_max)
340 {
341 	u8 res = 0;
342 
343 	if (seconds > 0) {
344 		int period = timeout;
345 
346 		res = 1;
347 		while (seconds > period && res < 255) {
348 			res++;
349 			timeout <<= 1;
350 			if (timeout > rto_max)
351 				timeout = rto_max;
352 			period += timeout;
353 		}
354 	}
355 	return res;
356 }
357 
358 /* Convert retransmits to seconds based on initial and max timeout */
retrans_to_secs(u8 retrans,int timeout,int rto_max)359 static int retrans_to_secs(u8 retrans, int timeout, int rto_max)
360 {
361 	int period = 0;
362 
363 	if (retrans > 0) {
364 		period = timeout;
365 		while (--retrans) {
366 			timeout <<= 1;
367 			if (timeout > rto_max)
368 				timeout = rto_max;
369 			period += timeout;
370 		}
371 	}
372 	return period;
373 }
374 
375 /* Address-family independent initialization for a tcp_sock.
376  *
377  * NOTE: A lot of things set to zero explicitly by call to
378  *       sk_alloc() so need not be done here.
379  */
tcp_init_sock(struct sock * sk)380 void tcp_init_sock(struct sock *sk)
381 {
382 	struct inet_connection_sock *icsk = inet_csk(sk);
383 	struct tcp_sock *tp = tcp_sk(sk);
384 
385 	tp->out_of_order_queue = RB_ROOT;
386 	tcp_init_xmit_timers(sk);
387 	tcp_prequeue_init(tp);
388 	INIT_LIST_HEAD(&tp->tsq_node);
389 
390 	icsk->icsk_rto = TCP_TIMEOUT_INIT;
391 	tp->mdev_us = jiffies_to_usecs(TCP_TIMEOUT_INIT);
392 	tp->rtt_min[0].rtt = ~0U;
393 
394 	/* So many TCP implementations out there (incorrectly) count the
395 	 * initial SYN frame in their delayed-ACK and congestion control
396 	 * algorithms that we must have the following bandaid to talk
397 	 * efficiently to them.  -DaveM
398 	 */
399 	tp->snd_cwnd = TCP_INIT_CWND;
400 
401 	/* See draft-stevens-tcpca-spec-01 for discussion of the
402 	 * initialization of these values.
403 	 */
404 	tp->snd_ssthresh = TCP_INFINITE_SSTHRESH;
405 	tp->snd_cwnd_clamp = ~0;
406 	tp->mss_cache = TCP_MSS_DEFAULT;
407 	u64_stats_init(&tp->syncp);
408 
409 	tp->reordering = sysctl_tcp_reordering;
410 	tcp_enable_early_retrans(tp);
411 	tcp_assign_congestion_control(sk);
412 
413 	tp->tsoffset = 0;
414 
415 	sk->sk_state = TCP_CLOSE;
416 
417 	sk->sk_write_space = sk_stream_write_space;
418 	sock_set_flag(sk, SOCK_USE_WRITE_QUEUE);
419 
420 	icsk->icsk_sync_mss = tcp_sync_mss;
421 
422 	sk->sk_sndbuf = sysctl_tcp_wmem[1];
423 	sk->sk_rcvbuf = sysctl_tcp_rmem[1];
424 
425 	local_bh_disable();
426 	sock_update_memcg(sk);
427 	sk_sockets_allocated_inc(sk);
428 	local_bh_enable();
429 }
430 EXPORT_SYMBOL(tcp_init_sock);
431 
tcp_tx_timestamp(struct sock * sk,struct sk_buff * skb)432 static void tcp_tx_timestamp(struct sock *sk, struct sk_buff *skb)
433 {
434 	if (sk->sk_tsflags) {
435 		struct skb_shared_info *shinfo = skb_shinfo(skb);
436 
437 		sock_tx_timestamp(sk, &shinfo->tx_flags);
438 		if (shinfo->tx_flags & SKBTX_ANY_TSTAMP)
439 			shinfo->tskey = TCP_SKB_CB(skb)->seq + skb->len - 1;
440 	}
441 }
442 
443 /*
444  *	Wait for a TCP event.
445  *
446  *	Note that we don't need to lock the socket, as the upper poll layers
447  *	take care of normal races (between the test and the event) and we don't
448  *	go look at any of the socket buffers directly.
449  */
tcp_poll(struct file * file,struct socket * sock,poll_table * wait)450 unsigned int tcp_poll(struct file *file, struct socket *sock, poll_table *wait)
451 {
452 	unsigned int mask;
453 	struct sock *sk = sock->sk;
454 	const struct tcp_sock *tp = tcp_sk(sk);
455 	int state;
456 
457 	sock_rps_record_flow(sk);
458 
459 	sock_poll_wait(file, sk_sleep(sk), wait);
460 
461 	state = sk_state_load(sk);
462 	if (state == TCP_LISTEN)
463 		return inet_csk_listen_poll(sk);
464 
465 	/* Socket is not locked. We are protected from async events
466 	 * by poll logic and correct handling of state changes
467 	 * made by other threads is impossible in any case.
468 	 */
469 
470 	mask = 0;
471 
472 	/*
473 	 * POLLHUP is certainly not done right. But poll() doesn't
474 	 * have a notion of HUP in just one direction, and for a
475 	 * socket the read side is more interesting.
476 	 *
477 	 * Some poll() documentation says that POLLHUP is incompatible
478 	 * with the POLLOUT/POLLWR flags, so somebody should check this
479 	 * all. But careful, it tends to be safer to return too many
480 	 * bits than too few, and you can easily break real applications
481 	 * if you don't tell them that something has hung up!
482 	 *
483 	 * Check-me.
484 	 *
485 	 * Check number 1. POLLHUP is _UNMASKABLE_ event (see UNIX98 and
486 	 * our fs/select.c). It means that after we received EOF,
487 	 * poll always returns immediately, making impossible poll() on write()
488 	 * in state CLOSE_WAIT. One solution is evident --- to set POLLHUP
489 	 * if and only if shutdown has been made in both directions.
490 	 * Actually, it is interesting to look how Solaris and DUX
491 	 * solve this dilemma. I would prefer, if POLLHUP were maskable,
492 	 * then we could set it on SND_SHUTDOWN. BTW examples given
493 	 * in Stevens' books assume exactly this behaviour, it explains
494 	 * why POLLHUP is incompatible with POLLOUT.	--ANK
495 	 *
496 	 * NOTE. Check for TCP_CLOSE is added. The goal is to prevent
497 	 * blocking on fresh not-connected or disconnected socket. --ANK
498 	 */
499 	if (sk->sk_shutdown == SHUTDOWN_MASK || state == TCP_CLOSE)
500 		mask |= POLLHUP;
501 	if (sk->sk_shutdown & RCV_SHUTDOWN)
502 		mask |= POLLIN | POLLRDNORM | POLLRDHUP;
503 
504 	/* Connected or passive Fast Open socket? */
505 	if (state != TCP_SYN_SENT &&
506 	    (state != TCP_SYN_RECV || tp->fastopen_rsk)) {
507 		int target = sock_rcvlowat(sk, 0, INT_MAX);
508 
509 		if (tp->urg_seq == tp->copied_seq &&
510 		    !sock_flag(sk, SOCK_URGINLINE) &&
511 		    tp->urg_data)
512 			target++;
513 
514 		if (tp->rcv_nxt - tp->copied_seq >= target)
515 			mask |= POLLIN | POLLRDNORM;
516 
517 		if (!(sk->sk_shutdown & SEND_SHUTDOWN)) {
518 			if (sk_stream_is_writeable(sk)) {
519 				mask |= POLLOUT | POLLWRNORM;
520 			} else {  /* send SIGIO later */
521 				sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk);
522 				set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
523 
524 				/* Race breaker. If space is freed after
525 				 * wspace test but before the flags are set,
526 				 * IO signal will be lost. Memory barrier
527 				 * pairs with the input side.
528 				 */
529 				smp_mb__after_atomic();
530 				if (sk_stream_is_writeable(sk))
531 					mask |= POLLOUT | POLLWRNORM;
532 			}
533 		} else
534 			mask |= POLLOUT | POLLWRNORM;
535 
536 		if (tp->urg_data & TCP_URG_VALID)
537 			mask |= POLLPRI;
538 	}
539 	/* This barrier is coupled with smp_wmb() in tcp_reset() */
540 	smp_rmb();
541 	if (sk->sk_err || !skb_queue_empty(&sk->sk_error_queue))
542 		mask |= POLLERR;
543 
544 	return mask;
545 }
546 EXPORT_SYMBOL(tcp_poll);
547 
tcp_ioctl(struct sock * sk,int cmd,unsigned long arg)548 int tcp_ioctl(struct sock *sk, int cmd, unsigned long arg)
549 {
550 	struct tcp_sock *tp = tcp_sk(sk);
551 	int answ;
552 	bool slow;
553 
554 	switch (cmd) {
555 	case SIOCINQ:
556 		if (sk->sk_state == TCP_LISTEN)
557 			return -EINVAL;
558 
559 		slow = lock_sock_fast(sk);
560 		if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV))
561 			answ = 0;
562 		else if (sock_flag(sk, SOCK_URGINLINE) ||
563 			 !tp->urg_data ||
564 			 before(tp->urg_seq, tp->copied_seq) ||
565 			 !before(tp->urg_seq, tp->rcv_nxt)) {
566 
567 			answ = tp->rcv_nxt - tp->copied_seq;
568 
569 			/* Subtract 1, if FIN was received */
570 			if (answ && sock_flag(sk, SOCK_DONE))
571 				answ--;
572 		} else
573 			answ = tp->urg_seq - tp->copied_seq;
574 		unlock_sock_fast(sk, slow);
575 		break;
576 	case SIOCATMARK:
577 		answ = tp->urg_data && tp->urg_seq == tp->copied_seq;
578 		break;
579 	case SIOCOUTQ:
580 		if (sk->sk_state == TCP_LISTEN)
581 			return -EINVAL;
582 
583 		if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV))
584 			answ = 0;
585 		else
586 			answ = tp->write_seq - tp->snd_una;
587 		break;
588 	case SIOCOUTQNSD:
589 		if (sk->sk_state == TCP_LISTEN)
590 			return -EINVAL;
591 
592 		if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV))
593 			answ = 0;
594 		else
595 			answ = tp->write_seq - tp->snd_nxt;
596 		break;
597 	default:
598 		return -ENOIOCTLCMD;
599 	}
600 
601 	return put_user(answ, (int __user *)arg);
602 }
603 EXPORT_SYMBOL(tcp_ioctl);
604 
tcp_mark_push(struct tcp_sock * tp,struct sk_buff * skb)605 static inline void tcp_mark_push(struct tcp_sock *tp, struct sk_buff *skb)
606 {
607 	TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_PSH;
608 	tp->pushed_seq = tp->write_seq;
609 }
610 
forced_push(const struct tcp_sock * tp)611 static inline bool forced_push(const struct tcp_sock *tp)
612 {
613 	return after(tp->write_seq, tp->pushed_seq + (tp->max_window >> 1));
614 }
615 
skb_entail(struct sock * sk,struct sk_buff * skb)616 static void skb_entail(struct sock *sk, struct sk_buff *skb)
617 {
618 	struct tcp_sock *tp = tcp_sk(sk);
619 	struct tcp_skb_cb *tcb = TCP_SKB_CB(skb);
620 
621 	skb->csum    = 0;
622 	tcb->seq     = tcb->end_seq = tp->write_seq;
623 	tcb->tcp_flags = TCPHDR_ACK;
624 	tcb->sacked  = 0;
625 	__skb_header_release(skb);
626 	tcp_add_write_queue_tail(sk, skb);
627 	sk->sk_wmem_queued += skb->truesize;
628 	sk_mem_charge(sk, skb->truesize);
629 	if (tp->nonagle & TCP_NAGLE_PUSH)
630 		tp->nonagle &= ~TCP_NAGLE_PUSH;
631 
632 	tcp_slow_start_after_idle_check(sk);
633 }
634 
tcp_mark_urg(struct tcp_sock * tp,int flags)635 static inline void tcp_mark_urg(struct tcp_sock *tp, int flags)
636 {
637 	if (flags & MSG_OOB)
638 		tp->snd_up = tp->write_seq;
639 }
640 
641 /* If a not yet filled skb is pushed, do not send it if
642  * we have data packets in Qdisc or NIC queues :
643  * Because TX completion will happen shortly, it gives a chance
644  * to coalesce future sendmsg() payload into this skb, without
645  * need for a timer, and with no latency trade off.
646  * As packets containing data payload have a bigger truesize
647  * than pure acks (dataless) packets, the last checks prevent
648  * autocorking if we only have an ACK in Qdisc/NIC queues,
649  * or if TX completion was delayed after we processed ACK packet.
650  */
tcp_should_autocork(struct sock * sk,struct sk_buff * skb,int size_goal)651 static bool tcp_should_autocork(struct sock *sk, struct sk_buff *skb,
652 				int size_goal)
653 {
654 	return skb->len < size_goal &&
655 	       sysctl_tcp_autocorking &&
656 	       skb != tcp_write_queue_head(sk) &&
657 	       atomic_read(&sk->sk_wmem_alloc) > skb->truesize;
658 }
659 
tcp_push(struct sock * sk,int flags,int mss_now,int nonagle,int size_goal)660 static void tcp_push(struct sock *sk, int flags, int mss_now,
661 		     int nonagle, int size_goal)
662 {
663 	struct tcp_sock *tp = tcp_sk(sk);
664 	struct sk_buff *skb;
665 
666 	if (!tcp_send_head(sk))
667 		return;
668 
669 	skb = tcp_write_queue_tail(sk);
670 	if (!(flags & MSG_MORE) || forced_push(tp))
671 		tcp_mark_push(tp, skb);
672 
673 	tcp_mark_urg(tp, flags);
674 
675 	if (tcp_should_autocork(sk, skb, size_goal)) {
676 
677 		/* avoid atomic op if TSQ_THROTTLED bit is already set */
678 		if (!test_bit(TSQ_THROTTLED, &tp->tsq_flags)) {
679 			NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPAUTOCORKING);
680 			set_bit(TSQ_THROTTLED, &tp->tsq_flags);
681 		}
682 		/* It is possible TX completion already happened
683 		 * before we set TSQ_THROTTLED.
684 		 */
685 		if (atomic_read(&sk->sk_wmem_alloc) > skb->truesize)
686 			return;
687 	}
688 
689 	if (flags & MSG_MORE)
690 		nonagle = TCP_NAGLE_CORK;
691 
692 	__tcp_push_pending_frames(sk, mss_now, nonagle);
693 }
694 
tcp_splice_data_recv(read_descriptor_t * rd_desc,struct sk_buff * skb,unsigned int offset,size_t len)695 static int tcp_splice_data_recv(read_descriptor_t *rd_desc, struct sk_buff *skb,
696 				unsigned int offset, size_t len)
697 {
698 	struct tcp_splice_state *tss = rd_desc->arg.data;
699 	int ret;
700 
701 	ret = skb_splice_bits(skb, skb->sk, offset, tss->pipe,
702 			      min(rd_desc->count, len), tss->flags,
703 			      skb_socket_splice);
704 	if (ret > 0)
705 		rd_desc->count -= ret;
706 	return ret;
707 }
708 
__tcp_splice_read(struct sock * sk,struct tcp_splice_state * tss)709 static int __tcp_splice_read(struct sock *sk, struct tcp_splice_state *tss)
710 {
711 	/* Store TCP splice context information in read_descriptor_t. */
712 	read_descriptor_t rd_desc = {
713 		.arg.data = tss,
714 		.count	  = tss->len,
715 	};
716 
717 	return tcp_read_sock(sk, &rd_desc, tcp_splice_data_recv);
718 }
719 
720 /**
721  *  tcp_splice_read - splice data from TCP socket to a pipe
722  * @sock:	socket to splice from
723  * @ppos:	position (not valid)
724  * @pipe:	pipe to splice to
725  * @len:	number of bytes to splice
726  * @flags:	splice modifier flags
727  *
728  * Description:
729  *    Will read pages from given socket and fill them into a pipe.
730  *
731  **/
tcp_splice_read(struct socket * sock,loff_t * ppos,struct pipe_inode_info * pipe,size_t len,unsigned int flags)732 ssize_t tcp_splice_read(struct socket *sock, loff_t *ppos,
733 			struct pipe_inode_info *pipe, size_t len,
734 			unsigned int flags)
735 {
736 	struct sock *sk = sock->sk;
737 	struct tcp_splice_state tss = {
738 		.pipe = pipe,
739 		.len = len,
740 		.flags = flags,
741 	};
742 	long timeo;
743 	ssize_t spliced;
744 	int ret;
745 
746 	sock_rps_record_flow(sk);
747 	/*
748 	 * We can't seek on a socket input
749 	 */
750 	if (unlikely(*ppos))
751 		return -ESPIPE;
752 
753 	ret = spliced = 0;
754 
755 	lock_sock(sk);
756 
757 	timeo = sock_rcvtimeo(sk, sock->file->f_flags & O_NONBLOCK);
758 	while (tss.len) {
759 		ret = __tcp_splice_read(sk, &tss);
760 		if (ret < 0)
761 			break;
762 		else if (!ret) {
763 			if (spliced)
764 				break;
765 			if (sock_flag(sk, SOCK_DONE))
766 				break;
767 			if (sk->sk_err) {
768 				ret = sock_error(sk);
769 				break;
770 			}
771 			if (sk->sk_shutdown & RCV_SHUTDOWN)
772 				break;
773 			if (sk->sk_state == TCP_CLOSE) {
774 				/*
775 				 * This occurs when user tries to read
776 				 * from never connected socket.
777 				 */
778 				if (!sock_flag(sk, SOCK_DONE))
779 					ret = -ENOTCONN;
780 				break;
781 			}
782 			if (!timeo) {
783 				ret = -EAGAIN;
784 				break;
785 			}
786 			/* if __tcp_splice_read() got nothing while we have
787 			 * an skb in receive queue, we do not want to loop.
788 			 * This might happen with URG data.
789 			 */
790 			if (!skb_queue_empty(&sk->sk_receive_queue))
791 				break;
792 			sk_wait_data(sk, &timeo, NULL);
793 			if (signal_pending(current)) {
794 				ret = sock_intr_errno(timeo);
795 				break;
796 			}
797 			continue;
798 		}
799 		tss.len -= ret;
800 		spliced += ret;
801 
802 		if (!timeo)
803 			break;
804 		release_sock(sk);
805 		lock_sock(sk);
806 
807 		if (sk->sk_err || sk->sk_state == TCP_CLOSE ||
808 		    (sk->sk_shutdown & RCV_SHUTDOWN) ||
809 		    signal_pending(current))
810 			break;
811 	}
812 
813 	release_sock(sk);
814 
815 	if (spliced)
816 		return spliced;
817 
818 	return ret;
819 }
820 EXPORT_SYMBOL(tcp_splice_read);
821 
sk_stream_alloc_skb(struct sock * sk,int size,gfp_t gfp,bool force_schedule)822 struct sk_buff *sk_stream_alloc_skb(struct sock *sk, int size, gfp_t gfp,
823 				    bool force_schedule)
824 {
825 	struct sk_buff *skb;
826 
827 	/* The TCP header must be at least 32-bit aligned.  */
828 	size = ALIGN(size, 4);
829 
830 	if (unlikely(tcp_under_memory_pressure(sk)))
831 		sk_mem_reclaim_partial(sk);
832 
833 	skb = alloc_skb_fclone(size + sk->sk_prot->max_header, gfp);
834 	if (likely(skb)) {
835 		bool mem_scheduled;
836 
837 		if (force_schedule) {
838 			mem_scheduled = true;
839 			sk_forced_mem_schedule(sk, skb->truesize);
840 		} else {
841 			mem_scheduled = sk_wmem_schedule(sk, skb->truesize);
842 		}
843 		if (likely(mem_scheduled)) {
844 			skb_reserve(skb, sk->sk_prot->max_header);
845 			/*
846 			 * Make sure that we have exactly size bytes
847 			 * available to the caller, no more, no less.
848 			 */
849 			skb->reserved_tailroom = skb->end - skb->tail - size;
850 			return skb;
851 		}
852 		__kfree_skb(skb);
853 	} else {
854 		sk->sk_prot->enter_memory_pressure(sk);
855 		sk_stream_moderate_sndbuf(sk);
856 	}
857 	return NULL;
858 }
859 
tcp_xmit_size_goal(struct sock * sk,u32 mss_now,int large_allowed)860 static unsigned int tcp_xmit_size_goal(struct sock *sk, u32 mss_now,
861 				       int large_allowed)
862 {
863 	struct tcp_sock *tp = tcp_sk(sk);
864 	u32 new_size_goal, size_goal;
865 
866 	if (!large_allowed || !sk_can_gso(sk))
867 		return mss_now;
868 
869 	/* Note : tcp_tso_autosize() will eventually split this later */
870 	new_size_goal = sk->sk_gso_max_size - 1 - MAX_TCP_HEADER;
871 	new_size_goal = tcp_bound_to_half_wnd(tp, new_size_goal);
872 
873 	/* We try hard to avoid divides here */
874 	size_goal = tp->gso_segs * mss_now;
875 	if (unlikely(new_size_goal < size_goal ||
876 		     new_size_goal >= size_goal + mss_now)) {
877 		tp->gso_segs = min_t(u16, new_size_goal / mss_now,
878 				     sk->sk_gso_max_segs);
879 		size_goal = tp->gso_segs * mss_now;
880 	}
881 
882 	return max(size_goal, mss_now);
883 }
884 
tcp_send_mss(struct sock * sk,int * size_goal,int flags)885 static int tcp_send_mss(struct sock *sk, int *size_goal, int flags)
886 {
887 	int mss_now;
888 
889 	mss_now = tcp_current_mss(sk);
890 	*size_goal = tcp_xmit_size_goal(sk, mss_now, !(flags & MSG_OOB));
891 
892 	return mss_now;
893 }
894 
do_tcp_sendpages(struct sock * sk,struct page * page,int offset,size_t size,int flags)895 static ssize_t do_tcp_sendpages(struct sock *sk, struct page *page, int offset,
896 				size_t size, int flags)
897 {
898 	struct tcp_sock *tp = tcp_sk(sk);
899 	int mss_now, size_goal;
900 	int err;
901 	ssize_t copied;
902 	long timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT);
903 
904 	/* Wait for a connection to finish. One exception is TCP Fast Open
905 	 * (passive side) where data is allowed to be sent before a connection
906 	 * is fully established.
907 	 */
908 	if (((1 << sk->sk_state) & ~(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)) &&
909 	    !tcp_passive_fastopen(sk)) {
910 		err = sk_stream_wait_connect(sk, &timeo);
911 		if (err != 0)
912 			goto out_err;
913 	}
914 
915 	sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk);
916 
917 	mss_now = tcp_send_mss(sk, &size_goal, flags);
918 	copied = 0;
919 
920 	err = -EPIPE;
921 	if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN))
922 		goto out_err;
923 
924 	while (size > 0) {
925 		struct sk_buff *skb = tcp_write_queue_tail(sk);
926 		int copy, i;
927 		bool can_coalesce;
928 
929 		if (!tcp_send_head(sk) || (copy = size_goal - skb->len) <= 0) {
930 new_segment:
931 			if (!sk_stream_memory_free(sk))
932 				goto wait_for_sndbuf;
933 
934 			skb = sk_stream_alloc_skb(sk, 0, sk->sk_allocation,
935 						  skb_queue_empty(&sk->sk_write_queue));
936 			if (!skb)
937 				goto wait_for_memory;
938 
939 			skb_entail(sk, skb);
940 			copy = size_goal;
941 		}
942 
943 		if (copy > size)
944 			copy = size;
945 
946 		i = skb_shinfo(skb)->nr_frags;
947 		can_coalesce = skb_can_coalesce(skb, i, page, offset);
948 		if (!can_coalesce && i >= sysctl_max_skb_frags) {
949 			tcp_mark_push(tp, skb);
950 			goto new_segment;
951 		}
952 		if (!sk_wmem_schedule(sk, copy))
953 			goto wait_for_memory;
954 
955 		if (can_coalesce) {
956 			skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy);
957 		} else {
958 			get_page(page);
959 			skb_fill_page_desc(skb, i, page, offset, copy);
960 		}
961 		skb_shinfo(skb)->tx_flags |= SKBTX_SHARED_FRAG;
962 
963 		skb->len += copy;
964 		skb->data_len += copy;
965 		skb->truesize += copy;
966 		sk->sk_wmem_queued += copy;
967 		sk_mem_charge(sk, copy);
968 		skb->ip_summed = CHECKSUM_PARTIAL;
969 		tp->write_seq += copy;
970 		TCP_SKB_CB(skb)->end_seq += copy;
971 		tcp_skb_pcount_set(skb, 0);
972 
973 		if (!copied)
974 			TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_PSH;
975 
976 		copied += copy;
977 		offset += copy;
978 		size -= copy;
979 		if (!size) {
980 			tcp_tx_timestamp(sk, skb);
981 			goto out;
982 		}
983 
984 		if (skb->len < size_goal || (flags & MSG_OOB))
985 			continue;
986 
987 		if (forced_push(tp)) {
988 			tcp_mark_push(tp, skb);
989 			__tcp_push_pending_frames(sk, mss_now, TCP_NAGLE_PUSH);
990 		} else if (skb == tcp_send_head(sk))
991 			tcp_push_one(sk, mss_now);
992 		continue;
993 
994 wait_for_sndbuf:
995 		set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
996 wait_for_memory:
997 		tcp_push(sk, flags & ~MSG_MORE, mss_now,
998 			 TCP_NAGLE_PUSH, size_goal);
999 
1000 		err = sk_stream_wait_memory(sk, &timeo);
1001 		if (err != 0)
1002 			goto do_error;
1003 
1004 		mss_now = tcp_send_mss(sk, &size_goal, flags);
1005 	}
1006 
1007 out:
1008 	if (copied && !(flags & MSG_SENDPAGE_NOTLAST))
1009 		tcp_push(sk, flags, mss_now, tp->nonagle, size_goal);
1010 	return copied;
1011 
1012 do_error:
1013 	if (copied)
1014 		goto out;
1015 out_err:
1016 	/* make sure we wake any epoll edge trigger waiter */
1017 	if (unlikely(skb_queue_len(&sk->sk_write_queue) == 0 && err == -EAGAIN))
1018 		sk->sk_write_space(sk);
1019 	return sk_stream_error(sk, flags, err);
1020 }
1021 
tcp_sendpage(struct sock * sk,struct page * page,int offset,size_t size,int flags)1022 int tcp_sendpage(struct sock *sk, struct page *page, int offset,
1023 		 size_t size, int flags)
1024 {
1025 	ssize_t res;
1026 
1027 	if (!(sk->sk_route_caps & NETIF_F_SG) ||
1028 	    !(sk->sk_route_caps & NETIF_F_ALL_CSUM))
1029 		return sock_no_sendpage(sk->sk_socket, page, offset, size,
1030 					flags);
1031 
1032 	lock_sock(sk);
1033 	res = do_tcp_sendpages(sk, page, offset, size, flags);
1034 	release_sock(sk);
1035 	return res;
1036 }
1037 EXPORT_SYMBOL(tcp_sendpage);
1038 
select_size(const struct sock * sk,bool sg)1039 static inline int select_size(const struct sock *sk, bool sg)
1040 {
1041 	const struct tcp_sock *tp = tcp_sk(sk);
1042 	int tmp = tp->mss_cache;
1043 
1044 	if (sg) {
1045 		if (sk_can_gso(sk)) {
1046 			/* Small frames wont use a full page:
1047 			 * Payload will immediately follow tcp header.
1048 			 */
1049 			tmp = SKB_WITH_OVERHEAD(2048 - MAX_TCP_HEADER);
1050 		} else {
1051 			int pgbreak = SKB_MAX_HEAD(MAX_TCP_HEADER);
1052 
1053 			if (tmp >= pgbreak &&
1054 			    tmp <= pgbreak + (MAX_SKB_FRAGS - 1) * PAGE_SIZE)
1055 				tmp = pgbreak;
1056 		}
1057 	}
1058 
1059 	return tmp;
1060 }
1061 
tcp_free_fastopen_req(struct tcp_sock * tp)1062 void tcp_free_fastopen_req(struct tcp_sock *tp)
1063 {
1064 	if (tp->fastopen_req) {
1065 		kfree(tp->fastopen_req);
1066 		tp->fastopen_req = NULL;
1067 	}
1068 }
1069 
tcp_sendmsg_fastopen(struct sock * sk,struct msghdr * msg,int * copied,size_t size)1070 static int tcp_sendmsg_fastopen(struct sock *sk, struct msghdr *msg,
1071 				int *copied, size_t size)
1072 {
1073 	struct tcp_sock *tp = tcp_sk(sk);
1074 	struct sockaddr *uaddr = msg->msg_name;
1075 	int err, flags;
1076 
1077 	if (!(sysctl_tcp_fastopen & TFO_CLIENT_ENABLE) ||
1078 	    (uaddr && msg->msg_namelen >= sizeof(uaddr->sa_family) &&
1079 	     uaddr->sa_family == AF_UNSPEC))
1080 		return -EOPNOTSUPP;
1081 	if (tp->fastopen_req)
1082 		return -EALREADY; /* Another Fast Open is in progress */
1083 
1084 	tp->fastopen_req = kzalloc(sizeof(struct tcp_fastopen_request),
1085 				   sk->sk_allocation);
1086 	if (unlikely(!tp->fastopen_req))
1087 		return -ENOBUFS;
1088 	tp->fastopen_req->data = msg;
1089 	tp->fastopen_req->size = size;
1090 
1091 	flags = (msg->msg_flags & MSG_DONTWAIT) ? O_NONBLOCK : 0;
1092 	err = __inet_stream_connect(sk->sk_socket, uaddr,
1093 				    msg->msg_namelen, flags);
1094 	*copied = tp->fastopen_req->copied;
1095 	tcp_free_fastopen_req(tp);
1096 	return err;
1097 }
1098 
tcp_sendmsg(struct sock * sk,struct msghdr * msg,size_t size)1099 int tcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)
1100 {
1101 	struct tcp_sock *tp = tcp_sk(sk);
1102 	struct sk_buff *skb;
1103 	int flags, err, copied = 0;
1104 	int mss_now = 0, size_goal, copied_syn = 0;
1105 	bool sg;
1106 	long timeo;
1107 
1108 	lock_sock(sk);
1109 
1110 	flags = msg->msg_flags;
1111 	if ((flags & MSG_FASTOPEN) && !tp->repair) {
1112 		err = tcp_sendmsg_fastopen(sk, msg, &copied_syn, size);
1113 		if (err == -EINPROGRESS && copied_syn > 0)
1114 			goto out;
1115 		else if (err)
1116 			goto out_err;
1117 	}
1118 
1119 	timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT);
1120 
1121 	/* Wait for a connection to finish. One exception is TCP Fast Open
1122 	 * (passive side) where data is allowed to be sent before a connection
1123 	 * is fully established.
1124 	 */
1125 	if (((1 << sk->sk_state) & ~(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)) &&
1126 	    !tcp_passive_fastopen(sk)) {
1127 		err = sk_stream_wait_connect(sk, &timeo);
1128 		if (err != 0)
1129 			goto do_error;
1130 	}
1131 
1132 	if (unlikely(tp->repair)) {
1133 		if (tp->repair_queue == TCP_RECV_QUEUE) {
1134 			copied = tcp_send_rcvq(sk, msg, size);
1135 			goto out_nopush;
1136 		}
1137 
1138 		err = -EINVAL;
1139 		if (tp->repair_queue == TCP_NO_QUEUE)
1140 			goto out_err;
1141 
1142 		/* 'common' sending to sendq */
1143 	}
1144 
1145 	/* This should be in poll */
1146 	sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk);
1147 
1148 	mss_now = tcp_send_mss(sk, &size_goal, flags);
1149 
1150 	/* Ok commence sending. */
1151 	copied = 0;
1152 
1153 	err = -EPIPE;
1154 	if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN))
1155 		goto out_err;
1156 
1157 	sg = !!(sk->sk_route_caps & NETIF_F_SG);
1158 
1159 	while (msg_data_left(msg)) {
1160 		int copy = 0;
1161 		int max = size_goal;
1162 
1163 		skb = tcp_write_queue_tail(sk);
1164 		if (tcp_send_head(sk)) {
1165 			if (skb->ip_summed == CHECKSUM_NONE)
1166 				max = mss_now;
1167 			copy = max - skb->len;
1168 		}
1169 
1170 		if (copy <= 0) {
1171 new_segment:
1172 			/* Allocate new segment. If the interface is SG,
1173 			 * allocate skb fitting to single page.
1174 			 */
1175 			if (!sk_stream_memory_free(sk))
1176 				goto wait_for_sndbuf;
1177 
1178 			skb = sk_stream_alloc_skb(sk,
1179 						  select_size(sk, sg),
1180 						  sk->sk_allocation,
1181 						  skb_queue_empty(&sk->sk_write_queue));
1182 			if (!skb)
1183 				goto wait_for_memory;
1184 
1185 			/*
1186 			 * Check whether we can use HW checksum.
1187 			 */
1188 			if (sk->sk_route_caps & NETIF_F_ALL_CSUM)
1189 				skb->ip_summed = CHECKSUM_PARTIAL;
1190 
1191 			skb_entail(sk, skb);
1192 			copy = size_goal;
1193 			max = size_goal;
1194 
1195 			/* All packets are restored as if they have
1196 			 * already been sent. skb_mstamp isn't set to
1197 			 * avoid wrong rtt estimation.
1198 			 */
1199 			if (tp->repair)
1200 				TCP_SKB_CB(skb)->sacked |= TCPCB_REPAIRED;
1201 		}
1202 
1203 		/* Try to append data to the end of skb. */
1204 		if (copy > msg_data_left(msg))
1205 			copy = msg_data_left(msg);
1206 
1207 		/* Where to copy to? */
1208 		if (skb_availroom(skb) > 0) {
1209 			/* We have some space in skb head. Superb! */
1210 			copy = min_t(int, copy, skb_availroom(skb));
1211 			err = skb_add_data_nocache(sk, skb, &msg->msg_iter, copy);
1212 			if (err)
1213 				goto do_fault;
1214 		} else {
1215 			bool merge = true;
1216 			int i = skb_shinfo(skb)->nr_frags;
1217 			struct page_frag *pfrag = sk_page_frag(sk);
1218 
1219 			if (!sk_page_frag_refill(sk, pfrag))
1220 				goto wait_for_memory;
1221 
1222 			if (!skb_can_coalesce(skb, i, pfrag->page,
1223 					      pfrag->offset)) {
1224 				if (i >= sysctl_max_skb_frags || !sg) {
1225 					tcp_mark_push(tp, skb);
1226 					goto new_segment;
1227 				}
1228 				merge = false;
1229 			}
1230 
1231 			copy = min_t(int, copy, pfrag->size - pfrag->offset);
1232 
1233 			if (!sk_wmem_schedule(sk, copy))
1234 				goto wait_for_memory;
1235 
1236 			err = skb_copy_to_page_nocache(sk, &msg->msg_iter, skb,
1237 						       pfrag->page,
1238 						       pfrag->offset,
1239 						       copy);
1240 			if (err)
1241 				goto do_error;
1242 
1243 			/* Update the skb. */
1244 			if (merge) {
1245 				skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy);
1246 			} else {
1247 				skb_fill_page_desc(skb, i, pfrag->page,
1248 						   pfrag->offset, copy);
1249 				get_page(pfrag->page);
1250 			}
1251 			pfrag->offset += copy;
1252 		}
1253 
1254 		if (!copied)
1255 			TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_PSH;
1256 
1257 		tp->write_seq += copy;
1258 		TCP_SKB_CB(skb)->end_seq += copy;
1259 		tcp_skb_pcount_set(skb, 0);
1260 
1261 		copied += copy;
1262 		if (!msg_data_left(msg)) {
1263 			tcp_tx_timestamp(sk, skb);
1264 			goto out;
1265 		}
1266 
1267 		if (skb->len < max || (flags & MSG_OOB) || unlikely(tp->repair))
1268 			continue;
1269 
1270 		if (forced_push(tp)) {
1271 			tcp_mark_push(tp, skb);
1272 			__tcp_push_pending_frames(sk, mss_now, TCP_NAGLE_PUSH);
1273 		} else if (skb == tcp_send_head(sk))
1274 			tcp_push_one(sk, mss_now);
1275 		continue;
1276 
1277 wait_for_sndbuf:
1278 		set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
1279 wait_for_memory:
1280 		if (copied)
1281 			tcp_push(sk, flags & ~MSG_MORE, mss_now,
1282 				 TCP_NAGLE_PUSH, size_goal);
1283 
1284 		err = sk_stream_wait_memory(sk, &timeo);
1285 		if (err != 0)
1286 			goto do_error;
1287 
1288 		mss_now = tcp_send_mss(sk, &size_goal, flags);
1289 	}
1290 
1291 out:
1292 	if (copied)
1293 		tcp_push(sk, flags, mss_now, tp->nonagle, size_goal);
1294 out_nopush:
1295 	release_sock(sk);
1296 	return copied + copied_syn;
1297 
1298 do_fault:
1299 	if (!skb->len) {
1300 		tcp_unlink_write_queue(skb, sk);
1301 		/* It is the one place in all of TCP, except connection
1302 		 * reset, where we can be unlinking the send_head.
1303 		 */
1304 		tcp_check_send_head(sk, skb);
1305 		sk_wmem_free_skb(sk, skb);
1306 	}
1307 
1308 do_error:
1309 	if (copied + copied_syn)
1310 		goto out;
1311 out_err:
1312 	err = sk_stream_error(sk, flags, err);
1313 	/* make sure we wake any epoll edge trigger waiter */
1314 	if (unlikely(skb_queue_len(&sk->sk_write_queue) == 0 && err == -EAGAIN))
1315 		sk->sk_write_space(sk);
1316 	release_sock(sk);
1317 	return err;
1318 }
1319 EXPORT_SYMBOL(tcp_sendmsg);
1320 
1321 /*
1322  *	Handle reading urgent data. BSD has very simple semantics for
1323  *	this, no blocking and very strange errors 8)
1324  */
1325 
tcp_recv_urg(struct sock * sk,struct msghdr * msg,int len,int flags)1326 static int tcp_recv_urg(struct sock *sk, struct msghdr *msg, int len, int flags)
1327 {
1328 	struct tcp_sock *tp = tcp_sk(sk);
1329 
1330 	/* No URG data to read. */
1331 	if (sock_flag(sk, SOCK_URGINLINE) || !tp->urg_data ||
1332 	    tp->urg_data == TCP_URG_READ)
1333 		return -EINVAL;	/* Yes this is right ! */
1334 
1335 	if (sk->sk_state == TCP_CLOSE && !sock_flag(sk, SOCK_DONE))
1336 		return -ENOTCONN;
1337 
1338 	if (tp->urg_data & TCP_URG_VALID) {
1339 		int err = 0;
1340 		char c = tp->urg_data;
1341 
1342 		if (!(flags & MSG_PEEK))
1343 			tp->urg_data = TCP_URG_READ;
1344 
1345 		/* Read urgent data. */
1346 		msg->msg_flags |= MSG_OOB;
1347 
1348 		if (len > 0) {
1349 			if (!(flags & MSG_TRUNC))
1350 				err = memcpy_to_msg(msg, &c, 1);
1351 			len = 1;
1352 		} else
1353 			msg->msg_flags |= MSG_TRUNC;
1354 
1355 		return err ? -EFAULT : len;
1356 	}
1357 
1358 	if (sk->sk_state == TCP_CLOSE || (sk->sk_shutdown & RCV_SHUTDOWN))
1359 		return 0;
1360 
1361 	/* Fixed the recv(..., MSG_OOB) behaviour.  BSD docs and
1362 	 * the available implementations agree in this case:
1363 	 * this call should never block, independent of the
1364 	 * blocking state of the socket.
1365 	 * Mike <pall@rz.uni-karlsruhe.de>
1366 	 */
1367 	return -EAGAIN;
1368 }
1369 
tcp_peek_sndq(struct sock * sk,struct msghdr * msg,int len)1370 static int tcp_peek_sndq(struct sock *sk, struct msghdr *msg, int len)
1371 {
1372 	struct sk_buff *skb;
1373 	int copied = 0, err = 0;
1374 
1375 	/* XXX -- need to support SO_PEEK_OFF */
1376 
1377 	skb_queue_walk(&sk->sk_write_queue, skb) {
1378 		err = skb_copy_datagram_msg(skb, 0, msg, skb->len);
1379 		if (err)
1380 			break;
1381 
1382 		copied += skb->len;
1383 	}
1384 
1385 	return err ?: copied;
1386 }
1387 
1388 /* Clean up the receive buffer for full frames taken by the user,
1389  * then send an ACK if necessary.  COPIED is the number of bytes
1390  * tcp_recvmsg has given to the user so far, it speeds up the
1391  * calculation of whether or not we must ACK for the sake of
1392  * a window update.
1393  */
tcp_cleanup_rbuf(struct sock * sk,int copied)1394 static void tcp_cleanup_rbuf(struct sock *sk, int copied)
1395 {
1396 	struct tcp_sock *tp = tcp_sk(sk);
1397 	bool time_to_ack = false;
1398 
1399 	struct sk_buff *skb = skb_peek(&sk->sk_receive_queue);
1400 
1401 	WARN(skb && !before(tp->copied_seq, TCP_SKB_CB(skb)->end_seq),
1402 	     "cleanup rbuf bug: copied %X seq %X rcvnxt %X\n",
1403 	     tp->copied_seq, TCP_SKB_CB(skb)->end_seq, tp->rcv_nxt);
1404 
1405 	if (inet_csk_ack_scheduled(sk)) {
1406 		const struct inet_connection_sock *icsk = inet_csk(sk);
1407 		   /* Delayed ACKs frequently hit locked sockets during bulk
1408 		    * receive. */
1409 		if (icsk->icsk_ack.blocked ||
1410 		    /* Once-per-two-segments ACK was not sent by tcp_input.c */
1411 		    tp->rcv_nxt - tp->rcv_wup > icsk->icsk_ack.rcv_mss ||
1412 		    /*
1413 		     * If this read emptied read buffer, we send ACK, if
1414 		     * connection is not bidirectional, user drained
1415 		     * receive buffer and there was a small segment
1416 		     * in queue.
1417 		     */
1418 		    (copied > 0 &&
1419 		     ((icsk->icsk_ack.pending & ICSK_ACK_PUSHED2) ||
1420 		      ((icsk->icsk_ack.pending & ICSK_ACK_PUSHED) &&
1421 		       !icsk->icsk_ack.pingpong)) &&
1422 		      !atomic_read(&sk->sk_rmem_alloc)))
1423 			time_to_ack = true;
1424 	}
1425 
1426 	/* We send an ACK if we can now advertise a non-zero window
1427 	 * which has been raised "significantly".
1428 	 *
1429 	 * Even if window raised up to infinity, do not send window open ACK
1430 	 * in states, where we will not receive more. It is useless.
1431 	 */
1432 	if (copied > 0 && !time_to_ack && !(sk->sk_shutdown & RCV_SHUTDOWN)) {
1433 		__u32 rcv_window_now = tcp_receive_window(tp);
1434 
1435 		/* Optimize, __tcp_select_window() is not cheap. */
1436 		if (2*rcv_window_now <= tp->window_clamp) {
1437 			__u32 new_window = __tcp_select_window(sk);
1438 
1439 			/* Send ACK now, if this read freed lots of space
1440 			 * in our buffer. Certainly, new_window is new window.
1441 			 * We can advertise it now, if it is not less than current one.
1442 			 * "Lots" means "at least twice" here.
1443 			 */
1444 			if (new_window && new_window >= 2 * rcv_window_now)
1445 				time_to_ack = true;
1446 		}
1447 	}
1448 	if (time_to_ack)
1449 		tcp_send_ack(sk);
1450 }
1451 
tcp_prequeue_process(struct sock * sk)1452 static void tcp_prequeue_process(struct sock *sk)
1453 {
1454 	struct sk_buff *skb;
1455 	struct tcp_sock *tp = tcp_sk(sk);
1456 
1457 	NET_INC_STATS_USER(sock_net(sk), LINUX_MIB_TCPPREQUEUED);
1458 
1459 	/* RX process wants to run with disabled BHs, though it is not
1460 	 * necessary */
1461 	local_bh_disable();
1462 	while ((skb = __skb_dequeue(&tp->ucopy.prequeue)) != NULL)
1463 		sk_backlog_rcv(sk, skb);
1464 	local_bh_enable();
1465 
1466 	/* Clear memory counter. */
1467 	tp->ucopy.memory = 0;
1468 }
1469 
tcp_recv_skb(struct sock * sk,u32 seq,u32 * off)1470 static struct sk_buff *tcp_recv_skb(struct sock *sk, u32 seq, u32 *off)
1471 {
1472 	struct sk_buff *skb;
1473 	u32 offset;
1474 
1475 	while ((skb = skb_peek(&sk->sk_receive_queue)) != NULL) {
1476 		offset = seq - TCP_SKB_CB(skb)->seq;
1477 		if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)
1478 			offset--;
1479 		if (offset < skb->len || (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)) {
1480 			*off = offset;
1481 			return skb;
1482 		}
1483 		/* This looks weird, but this can happen if TCP collapsing
1484 		 * splitted a fat GRO packet, while we released socket lock
1485 		 * in skb_splice_bits()
1486 		 */
1487 		sk_eat_skb(sk, skb);
1488 	}
1489 	return NULL;
1490 }
1491 
1492 /*
1493  * This routine provides an alternative to tcp_recvmsg() for routines
1494  * that would like to handle copying from skbuffs directly in 'sendfile'
1495  * fashion.
1496  * Note:
1497  *	- It is assumed that the socket was locked by the caller.
1498  *	- The routine does not block.
1499  *	- At present, there is no support for reading OOB data
1500  *	  or for 'peeking' the socket using this routine
1501  *	  (although both would be easy to implement).
1502  */
tcp_read_sock(struct sock * sk,read_descriptor_t * desc,sk_read_actor_t recv_actor)1503 int tcp_read_sock(struct sock *sk, read_descriptor_t *desc,
1504 		  sk_read_actor_t recv_actor)
1505 {
1506 	struct sk_buff *skb;
1507 	struct tcp_sock *tp = tcp_sk(sk);
1508 	u32 seq = tp->copied_seq;
1509 	u32 offset;
1510 	int copied = 0;
1511 
1512 	if (sk->sk_state == TCP_LISTEN)
1513 		return -ENOTCONN;
1514 	while ((skb = tcp_recv_skb(sk, seq, &offset)) != NULL) {
1515 		if (offset < skb->len) {
1516 			int used;
1517 			size_t len;
1518 
1519 			len = skb->len - offset;
1520 			/* Stop reading if we hit a patch of urgent data */
1521 			if (tp->urg_data) {
1522 				u32 urg_offset = tp->urg_seq - seq;
1523 				if (urg_offset < len)
1524 					len = urg_offset;
1525 				if (!len)
1526 					break;
1527 			}
1528 			used = recv_actor(desc, skb, offset, len);
1529 			if (used <= 0) {
1530 				if (!copied)
1531 					copied = used;
1532 				break;
1533 			} else if (used <= len) {
1534 				seq += used;
1535 				copied += used;
1536 				offset += used;
1537 			}
1538 			/* If recv_actor drops the lock (e.g. TCP splice
1539 			 * receive) the skb pointer might be invalid when
1540 			 * getting here: tcp_collapse might have deleted it
1541 			 * while aggregating skbs from the socket queue.
1542 			 */
1543 			skb = tcp_recv_skb(sk, seq - 1, &offset);
1544 			if (!skb)
1545 				break;
1546 			/* TCP coalescing might have appended data to the skb.
1547 			 * Try to splice more frags
1548 			 */
1549 			if (offset + 1 != skb->len)
1550 				continue;
1551 		}
1552 		if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) {
1553 			sk_eat_skb(sk, skb);
1554 			++seq;
1555 			break;
1556 		}
1557 		sk_eat_skb(sk, skb);
1558 		if (!desc->count)
1559 			break;
1560 		tp->copied_seq = seq;
1561 	}
1562 	tp->copied_seq = seq;
1563 
1564 	tcp_rcv_space_adjust(sk);
1565 
1566 	/* Clean up data we have read: This will do ACK frames. */
1567 	if (copied > 0) {
1568 		tcp_recv_skb(sk, seq, &offset);
1569 		tcp_cleanup_rbuf(sk, copied);
1570 	}
1571 	return copied;
1572 }
1573 EXPORT_SYMBOL(tcp_read_sock);
1574 
1575 /*
1576  *	This routine copies from a sock struct into the user buffer.
1577  *
1578  *	Technical note: in 2.3 we work on _locked_ socket, so that
1579  *	tricks with *seq access order and skb->users are not required.
1580  *	Probably, code can be easily improved even more.
1581  */
1582 
tcp_recvmsg(struct sock * sk,struct msghdr * msg,size_t len,int nonblock,int flags,int * addr_len)1583 int tcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int nonblock,
1584 		int flags, int *addr_len)
1585 {
1586 	struct tcp_sock *tp = tcp_sk(sk);
1587 	int copied = 0;
1588 	u32 peek_seq;
1589 	u32 *seq;
1590 	unsigned long used;
1591 	int err;
1592 	int target;		/* Read at least this many bytes */
1593 	long timeo;
1594 	struct task_struct *user_recv = NULL;
1595 	struct sk_buff *skb, *last;
1596 	u32 urg_hole = 0;
1597 
1598 	if (unlikely(flags & MSG_ERRQUEUE))
1599 		return inet_recv_error(sk, msg, len, addr_len);
1600 
1601 	if (sk_can_busy_loop(sk) && skb_queue_empty(&sk->sk_receive_queue) &&
1602 	    (sk->sk_state == TCP_ESTABLISHED))
1603 		sk_busy_loop(sk, nonblock);
1604 
1605 	lock_sock(sk);
1606 
1607 	err = -ENOTCONN;
1608 	if (sk->sk_state == TCP_LISTEN)
1609 		goto out;
1610 
1611 	timeo = sock_rcvtimeo(sk, nonblock);
1612 
1613 	/* Urgent data needs to be handled specially. */
1614 	if (flags & MSG_OOB)
1615 		goto recv_urg;
1616 
1617 	if (unlikely(tp->repair)) {
1618 		err = -EPERM;
1619 		if (!(flags & MSG_PEEK))
1620 			goto out;
1621 
1622 		if (tp->repair_queue == TCP_SEND_QUEUE)
1623 			goto recv_sndq;
1624 
1625 		err = -EINVAL;
1626 		if (tp->repair_queue == TCP_NO_QUEUE)
1627 			goto out;
1628 
1629 		/* 'common' recv queue MSG_PEEK-ing */
1630 	}
1631 
1632 	seq = &tp->copied_seq;
1633 	if (flags & MSG_PEEK) {
1634 		peek_seq = tp->copied_seq;
1635 		seq = &peek_seq;
1636 	}
1637 
1638 	target = sock_rcvlowat(sk, flags & MSG_WAITALL, len);
1639 
1640 	do {
1641 		u32 offset;
1642 
1643 		/* Are we at urgent data? Stop if we have read anything or have SIGURG pending. */
1644 		if (tp->urg_data && tp->urg_seq == *seq) {
1645 			if (copied)
1646 				break;
1647 			if (signal_pending(current)) {
1648 				copied = timeo ? sock_intr_errno(timeo) : -EAGAIN;
1649 				break;
1650 			}
1651 		}
1652 
1653 		/* Next get a buffer. */
1654 
1655 		last = skb_peek_tail(&sk->sk_receive_queue);
1656 		skb_queue_walk(&sk->sk_receive_queue, skb) {
1657 			last = skb;
1658 			/* Now that we have two receive queues this
1659 			 * shouldn't happen.
1660 			 */
1661 			if (WARN(before(*seq, TCP_SKB_CB(skb)->seq),
1662 				 "TCP recvmsg seq # bug: copied %X, seq %X, rcvnxt %X, fl %X\n",
1663 				 *seq, TCP_SKB_CB(skb)->seq, tp->rcv_nxt,
1664 				 flags))
1665 				break;
1666 
1667 			offset = *seq - TCP_SKB_CB(skb)->seq;
1668 			if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)
1669 				offset--;
1670 			if (offset < skb->len)
1671 				goto found_ok_skb;
1672 			if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)
1673 				goto found_fin_ok;
1674 			WARN(!(flags & MSG_PEEK),
1675 			     "TCP recvmsg seq # bug 2: copied %X, seq %X, rcvnxt %X, fl %X\n",
1676 			     *seq, TCP_SKB_CB(skb)->seq, tp->rcv_nxt, flags);
1677 		}
1678 
1679 		/* Well, if we have backlog, try to process it now yet. */
1680 
1681 		if (copied >= target && !sk->sk_backlog.tail)
1682 			break;
1683 
1684 		if (copied) {
1685 			if (sk->sk_err ||
1686 			    sk->sk_state == TCP_CLOSE ||
1687 			    (sk->sk_shutdown & RCV_SHUTDOWN) ||
1688 			    !timeo ||
1689 			    signal_pending(current))
1690 				break;
1691 		} else {
1692 			if (sock_flag(sk, SOCK_DONE))
1693 				break;
1694 
1695 			if (sk->sk_err) {
1696 				copied = sock_error(sk);
1697 				break;
1698 			}
1699 
1700 			if (sk->sk_shutdown & RCV_SHUTDOWN)
1701 				break;
1702 
1703 			if (sk->sk_state == TCP_CLOSE) {
1704 				if (!sock_flag(sk, SOCK_DONE)) {
1705 					/* This occurs when user tries to read
1706 					 * from never connected socket.
1707 					 */
1708 					copied = -ENOTCONN;
1709 					break;
1710 				}
1711 				break;
1712 			}
1713 
1714 			if (!timeo) {
1715 				copied = -EAGAIN;
1716 				break;
1717 			}
1718 
1719 			if (signal_pending(current)) {
1720 				copied = sock_intr_errno(timeo);
1721 				break;
1722 			}
1723 		}
1724 
1725 		tcp_cleanup_rbuf(sk, copied);
1726 
1727 		if (!sysctl_tcp_low_latency && tp->ucopy.task == user_recv) {
1728 			/* Install new reader */
1729 			if (!user_recv && !(flags & (MSG_TRUNC | MSG_PEEK))) {
1730 				user_recv = current;
1731 				tp->ucopy.task = user_recv;
1732 				tp->ucopy.msg = msg;
1733 			}
1734 
1735 			tp->ucopy.len = len;
1736 
1737 			WARN_ON(tp->copied_seq != tp->rcv_nxt &&
1738 				!(flags & (MSG_PEEK | MSG_TRUNC)));
1739 
1740 			/* Ugly... If prequeue is not empty, we have to
1741 			 * process it before releasing socket, otherwise
1742 			 * order will be broken at second iteration.
1743 			 * More elegant solution is required!!!
1744 			 *
1745 			 * Look: we have the following (pseudo)queues:
1746 			 *
1747 			 * 1. packets in flight
1748 			 * 2. backlog
1749 			 * 3. prequeue
1750 			 * 4. receive_queue
1751 			 *
1752 			 * Each queue can be processed only if the next ones
1753 			 * are empty. At this point we have empty receive_queue.
1754 			 * But prequeue _can_ be not empty after 2nd iteration,
1755 			 * when we jumped to start of loop because backlog
1756 			 * processing added something to receive_queue.
1757 			 * We cannot release_sock(), because backlog contains
1758 			 * packets arrived _after_ prequeued ones.
1759 			 *
1760 			 * Shortly, algorithm is clear --- to process all
1761 			 * the queues in order. We could make it more directly,
1762 			 * requeueing packets from backlog to prequeue, if
1763 			 * is not empty. It is more elegant, but eats cycles,
1764 			 * unfortunately.
1765 			 */
1766 			if (!skb_queue_empty(&tp->ucopy.prequeue))
1767 				goto do_prequeue;
1768 
1769 			/* __ Set realtime policy in scheduler __ */
1770 		}
1771 
1772 		if (copied >= target) {
1773 			/* Do not sleep, just process backlog. */
1774 			release_sock(sk);
1775 			lock_sock(sk);
1776 		} else {
1777 			sk_wait_data(sk, &timeo, last);
1778 		}
1779 
1780 		if (user_recv) {
1781 			int chunk;
1782 
1783 			/* __ Restore normal policy in scheduler __ */
1784 
1785 			chunk = len - tp->ucopy.len;
1786 			if (chunk != 0) {
1787 				NET_ADD_STATS_USER(sock_net(sk), LINUX_MIB_TCPDIRECTCOPYFROMBACKLOG, chunk);
1788 				len -= chunk;
1789 				copied += chunk;
1790 			}
1791 
1792 			if (tp->rcv_nxt == tp->copied_seq &&
1793 			    !skb_queue_empty(&tp->ucopy.prequeue)) {
1794 do_prequeue:
1795 				tcp_prequeue_process(sk);
1796 
1797 				chunk = len - tp->ucopy.len;
1798 				if (chunk != 0) {
1799 					NET_ADD_STATS_USER(sock_net(sk), LINUX_MIB_TCPDIRECTCOPYFROMPREQUEUE, chunk);
1800 					len -= chunk;
1801 					copied += chunk;
1802 				}
1803 			}
1804 		}
1805 		if ((flags & MSG_PEEK) &&
1806 		    (peek_seq - copied - urg_hole != tp->copied_seq)) {
1807 			net_dbg_ratelimited("TCP(%s:%d): Application bug, race in MSG_PEEK\n",
1808 					    current->comm,
1809 					    task_pid_nr(current));
1810 			peek_seq = tp->copied_seq;
1811 		}
1812 		continue;
1813 
1814 	found_ok_skb:
1815 		/* Ok so how much can we use? */
1816 		used = skb->len - offset;
1817 		if (len < used)
1818 			used = len;
1819 
1820 		/* Do we have urgent data here? */
1821 		if (tp->urg_data) {
1822 			u32 urg_offset = tp->urg_seq - *seq;
1823 			if (urg_offset < used) {
1824 				if (!urg_offset) {
1825 					if (!sock_flag(sk, SOCK_URGINLINE)) {
1826 						++*seq;
1827 						urg_hole++;
1828 						offset++;
1829 						used--;
1830 						if (!used)
1831 							goto skip_copy;
1832 					}
1833 				} else
1834 					used = urg_offset;
1835 			}
1836 		}
1837 
1838 		if (!(flags & MSG_TRUNC)) {
1839 			err = skb_copy_datagram_msg(skb, offset, msg, used);
1840 			if (err) {
1841 				/* Exception. Bailout! */
1842 				if (!copied)
1843 					copied = -EFAULT;
1844 				break;
1845 			}
1846 		}
1847 
1848 		*seq += used;
1849 		copied += used;
1850 		len -= used;
1851 
1852 		tcp_rcv_space_adjust(sk);
1853 
1854 skip_copy:
1855 		if (tp->urg_data && after(tp->copied_seq, tp->urg_seq)) {
1856 			tp->urg_data = 0;
1857 			tcp_fast_path_check(sk);
1858 		}
1859 		if (used + offset < skb->len)
1860 			continue;
1861 
1862 		if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)
1863 			goto found_fin_ok;
1864 		if (!(flags & MSG_PEEK))
1865 			sk_eat_skb(sk, skb);
1866 		continue;
1867 
1868 	found_fin_ok:
1869 		/* Process the FIN. */
1870 		++*seq;
1871 		if (!(flags & MSG_PEEK))
1872 			sk_eat_skb(sk, skb);
1873 		break;
1874 	} while (len > 0);
1875 
1876 	if (user_recv) {
1877 		if (!skb_queue_empty(&tp->ucopy.prequeue)) {
1878 			int chunk;
1879 
1880 			tp->ucopy.len = copied > 0 ? len : 0;
1881 
1882 			tcp_prequeue_process(sk);
1883 
1884 			if (copied > 0 && (chunk = len - tp->ucopy.len) != 0) {
1885 				NET_ADD_STATS_USER(sock_net(sk), LINUX_MIB_TCPDIRECTCOPYFROMPREQUEUE, chunk);
1886 				len -= chunk;
1887 				copied += chunk;
1888 			}
1889 		}
1890 
1891 		tp->ucopy.task = NULL;
1892 		tp->ucopy.len = 0;
1893 	}
1894 
1895 	/* According to UNIX98, msg_name/msg_namelen are ignored
1896 	 * on connected socket. I was just happy when found this 8) --ANK
1897 	 */
1898 
1899 	/* Clean up data we have read: This will do ACK frames. */
1900 	tcp_cleanup_rbuf(sk, copied);
1901 
1902 	release_sock(sk);
1903 	return copied;
1904 
1905 out:
1906 	release_sock(sk);
1907 	return err;
1908 
1909 recv_urg:
1910 	err = tcp_recv_urg(sk, msg, len, flags);
1911 	goto out;
1912 
1913 recv_sndq:
1914 	err = tcp_peek_sndq(sk, msg, len);
1915 	goto out;
1916 }
1917 EXPORT_SYMBOL(tcp_recvmsg);
1918 
tcp_set_state(struct sock * sk,int state)1919 void tcp_set_state(struct sock *sk, int state)
1920 {
1921 	int oldstate = sk->sk_state;
1922 
1923 	switch (state) {
1924 	case TCP_ESTABLISHED:
1925 		if (oldstate != TCP_ESTABLISHED)
1926 			TCP_INC_STATS(sock_net(sk), TCP_MIB_CURRESTAB);
1927 		break;
1928 
1929 	case TCP_CLOSE:
1930 		if (oldstate == TCP_CLOSE_WAIT || oldstate == TCP_ESTABLISHED)
1931 			TCP_INC_STATS(sock_net(sk), TCP_MIB_ESTABRESETS);
1932 
1933 		sk->sk_prot->unhash(sk);
1934 		if (inet_csk(sk)->icsk_bind_hash &&
1935 		    !(sk->sk_userlocks & SOCK_BINDPORT_LOCK))
1936 			inet_put_port(sk);
1937 		/* fall through */
1938 	default:
1939 		if (oldstate == TCP_ESTABLISHED)
1940 			TCP_DEC_STATS(sock_net(sk), TCP_MIB_CURRESTAB);
1941 	}
1942 
1943 	/* Change state AFTER socket is unhashed to avoid closed
1944 	 * socket sitting in hash tables.
1945 	 */
1946 	sk_state_store(sk, state);
1947 
1948 #ifdef STATE_TRACE
1949 	SOCK_DEBUG(sk, "TCP sk=%p, State %s -> %s\n", sk, statename[oldstate], statename[state]);
1950 #endif
1951 }
1952 EXPORT_SYMBOL_GPL(tcp_set_state);
1953 
1954 /*
1955  *	State processing on a close. This implements the state shift for
1956  *	sending our FIN frame. Note that we only send a FIN for some
1957  *	states. A shutdown() may have already sent the FIN, or we may be
1958  *	closed.
1959  */
1960 
1961 static const unsigned char new_state[16] = {
1962   /* current state:        new state:      action:	*/
1963   [0 /* (Invalid) */]	= TCP_CLOSE,
1964   [TCP_ESTABLISHED]	= TCP_FIN_WAIT1 | TCP_ACTION_FIN,
1965   [TCP_SYN_SENT]	= TCP_CLOSE,
1966   [TCP_SYN_RECV]	= TCP_FIN_WAIT1 | TCP_ACTION_FIN,
1967   [TCP_FIN_WAIT1]	= TCP_FIN_WAIT1,
1968   [TCP_FIN_WAIT2]	= TCP_FIN_WAIT2,
1969   [TCP_TIME_WAIT]	= TCP_CLOSE,
1970   [TCP_CLOSE]		= TCP_CLOSE,
1971   [TCP_CLOSE_WAIT]	= TCP_LAST_ACK  | TCP_ACTION_FIN,
1972   [TCP_LAST_ACK]	= TCP_LAST_ACK,
1973   [TCP_LISTEN]		= TCP_CLOSE,
1974   [TCP_CLOSING]		= TCP_CLOSING,
1975   [TCP_NEW_SYN_RECV]	= TCP_CLOSE,	/* should not happen ! */
1976 };
1977 
tcp_close_state(struct sock * sk)1978 static int tcp_close_state(struct sock *sk)
1979 {
1980 	int next = (int)new_state[sk->sk_state];
1981 	int ns = next & TCP_STATE_MASK;
1982 
1983 	tcp_set_state(sk, ns);
1984 
1985 	return next & TCP_ACTION_FIN;
1986 }
1987 
1988 /*
1989  *	Shutdown the sending side of a connection. Much like close except
1990  *	that we don't receive shut down or sock_set_flag(sk, SOCK_DEAD).
1991  */
1992 
tcp_shutdown(struct sock * sk,int how)1993 void tcp_shutdown(struct sock *sk, int how)
1994 {
1995 	/*	We need to grab some memory, and put together a FIN,
1996 	 *	and then put it into the queue to be sent.
1997 	 *		Tim MacKenzie(tym@dibbler.cs.monash.edu.au) 4 Dec '92.
1998 	 */
1999 	if (!(how & SEND_SHUTDOWN))
2000 		return;
2001 
2002 	/* If we've already sent a FIN, or it's a closed state, skip this. */
2003 	if ((1 << sk->sk_state) &
2004 	    (TCPF_ESTABLISHED | TCPF_SYN_SENT |
2005 	     TCPF_SYN_RECV | TCPF_CLOSE_WAIT)) {
2006 		/* Clear out any half completed packets.  FIN if needed. */
2007 		if (tcp_close_state(sk))
2008 			tcp_send_fin(sk);
2009 	}
2010 }
2011 EXPORT_SYMBOL(tcp_shutdown);
2012 
tcp_check_oom(struct sock * sk,int shift)2013 bool tcp_check_oom(struct sock *sk, int shift)
2014 {
2015 	bool too_many_orphans, out_of_socket_memory;
2016 
2017 	too_many_orphans = tcp_too_many_orphans(sk, shift);
2018 	out_of_socket_memory = tcp_out_of_memory(sk);
2019 
2020 	if (too_many_orphans)
2021 		net_info_ratelimited("too many orphaned sockets\n");
2022 	if (out_of_socket_memory)
2023 		net_info_ratelimited("out of memory -- consider tuning tcp_mem\n");
2024 	return too_many_orphans || out_of_socket_memory;
2025 }
2026 
tcp_close(struct sock * sk,long timeout)2027 void tcp_close(struct sock *sk, long timeout)
2028 {
2029 	struct sk_buff *skb;
2030 	int data_was_unread = 0;
2031 	int state;
2032 
2033 	lock_sock(sk);
2034 	sk->sk_shutdown = SHUTDOWN_MASK;
2035 
2036 	if (sk->sk_state == TCP_LISTEN) {
2037 		tcp_set_state(sk, TCP_CLOSE);
2038 
2039 		/* Special case. */
2040 		inet_csk_listen_stop(sk);
2041 
2042 		goto adjudge_to_death;
2043 	}
2044 
2045 	/*  We need to flush the recv. buffs.  We do this only on the
2046 	 *  descriptor close, not protocol-sourced closes, because the
2047 	 *  reader process may not have drained the data yet!
2048 	 */
2049 	while ((skb = __skb_dequeue(&sk->sk_receive_queue)) != NULL) {
2050 		u32 len = TCP_SKB_CB(skb)->end_seq - TCP_SKB_CB(skb)->seq;
2051 
2052 		if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)
2053 			len--;
2054 		data_was_unread += len;
2055 		__kfree_skb(skb);
2056 	}
2057 
2058 	sk_mem_reclaim(sk);
2059 
2060 	/* If socket has been already reset (e.g. in tcp_reset()) - kill it. */
2061 	if (sk->sk_state == TCP_CLOSE)
2062 		goto adjudge_to_death;
2063 
2064 	/* As outlined in RFC 2525, section 2.17, we send a RST here because
2065 	 * data was lost. To witness the awful effects of the old behavior of
2066 	 * always doing a FIN, run an older 2.1.x kernel or 2.0.x, start a bulk
2067 	 * GET in an FTP client, suspend the process, wait for the client to
2068 	 * advertise a zero window, then kill -9 the FTP client, wheee...
2069 	 * Note: timeout is always zero in such a case.
2070 	 */
2071 	if (unlikely(tcp_sk(sk)->repair)) {
2072 		sk->sk_prot->disconnect(sk, 0);
2073 	} else if (data_was_unread) {
2074 		/* Unread data was tossed, zap the connection. */
2075 		NET_INC_STATS_USER(sock_net(sk), LINUX_MIB_TCPABORTONCLOSE);
2076 		tcp_set_state(sk, TCP_CLOSE);
2077 		tcp_send_active_reset(sk, sk->sk_allocation);
2078 	} else if (sock_flag(sk, SOCK_LINGER) && !sk->sk_lingertime) {
2079 		/* Check zero linger _after_ checking for unread data. */
2080 		sk->sk_prot->disconnect(sk, 0);
2081 		NET_INC_STATS_USER(sock_net(sk), LINUX_MIB_TCPABORTONDATA);
2082 	} else if (tcp_close_state(sk)) {
2083 		/* We FIN if the application ate all the data before
2084 		 * zapping the connection.
2085 		 */
2086 
2087 		/* RED-PEN. Formally speaking, we have broken TCP state
2088 		 * machine. State transitions:
2089 		 *
2090 		 * TCP_ESTABLISHED -> TCP_FIN_WAIT1
2091 		 * TCP_SYN_RECV	-> TCP_FIN_WAIT1 (forget it, it's impossible)
2092 		 * TCP_CLOSE_WAIT -> TCP_LAST_ACK
2093 		 *
2094 		 * are legal only when FIN has been sent (i.e. in window),
2095 		 * rather than queued out of window. Purists blame.
2096 		 *
2097 		 * F.e. "RFC state" is ESTABLISHED,
2098 		 * if Linux state is FIN-WAIT-1, but FIN is still not sent.
2099 		 *
2100 		 * The visible declinations are that sometimes
2101 		 * we enter time-wait state, when it is not required really
2102 		 * (harmless), do not send active resets, when they are
2103 		 * required by specs (TCP_ESTABLISHED, TCP_CLOSE_WAIT, when
2104 		 * they look as CLOSING or LAST_ACK for Linux)
2105 		 * Probably, I missed some more holelets.
2106 		 * 						--ANK
2107 		 * XXX (TFO) - To start off we don't support SYN+ACK+FIN
2108 		 * in a single packet! (May consider it later but will
2109 		 * probably need API support or TCP_CORK SYN-ACK until
2110 		 * data is written and socket is closed.)
2111 		 */
2112 		tcp_send_fin(sk);
2113 	}
2114 
2115 	sk_stream_wait_close(sk, timeout);
2116 
2117 adjudge_to_death:
2118 	state = sk->sk_state;
2119 	sock_hold(sk);
2120 	sock_orphan(sk);
2121 
2122 	/* It is the last release_sock in its life. It will remove backlog. */
2123 	release_sock(sk);
2124 
2125 
2126 	/* Now socket is owned by kernel and we acquire BH lock
2127 	   to finish close. No need to check for user refs.
2128 	 */
2129 	local_bh_disable();
2130 	bh_lock_sock(sk);
2131 	WARN_ON(sock_owned_by_user(sk));
2132 
2133 	percpu_counter_inc(sk->sk_prot->orphan_count);
2134 
2135 	/* Have we already been destroyed by a softirq or backlog? */
2136 	if (state != TCP_CLOSE && sk->sk_state == TCP_CLOSE)
2137 		goto out;
2138 
2139 	/*	This is a (useful) BSD violating of the RFC. There is a
2140 	 *	problem with TCP as specified in that the other end could
2141 	 *	keep a socket open forever with no application left this end.
2142 	 *	We use a 1 minute timeout (about the same as BSD) then kill
2143 	 *	our end. If they send after that then tough - BUT: long enough
2144 	 *	that we won't make the old 4*rto = almost no time - whoops
2145 	 *	reset mistake.
2146 	 *
2147 	 *	Nope, it was not mistake. It is really desired behaviour
2148 	 *	f.e. on http servers, when such sockets are useless, but
2149 	 *	consume significant resources. Let's do it with special
2150 	 *	linger2	option.					--ANK
2151 	 */
2152 
2153 	if (sk->sk_state == TCP_FIN_WAIT2) {
2154 		struct tcp_sock *tp = tcp_sk(sk);
2155 		if (tp->linger2 < 0) {
2156 			tcp_set_state(sk, TCP_CLOSE);
2157 			tcp_send_active_reset(sk, GFP_ATOMIC);
2158 			NET_INC_STATS_BH(sock_net(sk),
2159 					LINUX_MIB_TCPABORTONLINGER);
2160 		} else {
2161 			const int tmo = tcp_fin_time(sk);
2162 
2163 			if (tmo > TCP_TIMEWAIT_LEN) {
2164 				inet_csk_reset_keepalive_timer(sk,
2165 						tmo - TCP_TIMEWAIT_LEN);
2166 			} else {
2167 				tcp_time_wait(sk, TCP_FIN_WAIT2, tmo);
2168 				goto out;
2169 			}
2170 		}
2171 	}
2172 	if (sk->sk_state != TCP_CLOSE) {
2173 		sk_mem_reclaim(sk);
2174 		if (tcp_check_oom(sk, 0)) {
2175 			tcp_set_state(sk, TCP_CLOSE);
2176 			tcp_send_active_reset(sk, GFP_ATOMIC);
2177 			NET_INC_STATS_BH(sock_net(sk),
2178 					LINUX_MIB_TCPABORTONMEMORY);
2179 		} else if (!check_net(sock_net(sk))) {
2180 			/* Not possible to send reset; just close */
2181 			tcp_set_state(sk, TCP_CLOSE);
2182 		}
2183 	}
2184 
2185 	if (sk->sk_state == TCP_CLOSE) {
2186 		struct request_sock *req = tcp_sk(sk)->fastopen_rsk;
2187 		/* We could get here with a non-NULL req if the socket is
2188 		 * aborted (e.g., closed with unread data) before 3WHS
2189 		 * finishes.
2190 		 */
2191 		if (req)
2192 			reqsk_fastopen_remove(sk, req, false);
2193 		inet_csk_destroy_sock(sk);
2194 	}
2195 	/* Otherwise, socket is reprieved until protocol close. */
2196 
2197 out:
2198 	bh_unlock_sock(sk);
2199 	local_bh_enable();
2200 	sock_put(sk);
2201 }
2202 EXPORT_SYMBOL(tcp_close);
2203 
2204 /* These states need RST on ABORT according to RFC793 */
2205 
tcp_need_reset(int state)2206 static inline bool tcp_need_reset(int state)
2207 {
2208 	return (1 << state) &
2209 	       (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT | TCPF_FIN_WAIT1 |
2210 		TCPF_FIN_WAIT2 | TCPF_SYN_RECV);
2211 }
2212 
tcp_disconnect(struct sock * sk,int flags)2213 int tcp_disconnect(struct sock *sk, int flags)
2214 {
2215 	struct inet_sock *inet = inet_sk(sk);
2216 	struct inet_connection_sock *icsk = inet_csk(sk);
2217 	struct tcp_sock *tp = tcp_sk(sk);
2218 	int err = 0;
2219 	int old_state = sk->sk_state;
2220 
2221 	if (old_state != TCP_CLOSE)
2222 		tcp_set_state(sk, TCP_CLOSE);
2223 
2224 	/* ABORT function of RFC793 */
2225 	if (old_state == TCP_LISTEN) {
2226 		inet_csk_listen_stop(sk);
2227 	} else if (unlikely(tp->repair)) {
2228 		sk->sk_err = ECONNABORTED;
2229 	} else if (tcp_need_reset(old_state) ||
2230 		   (tp->snd_nxt != tp->write_seq &&
2231 		    (1 << old_state) & (TCPF_CLOSING | TCPF_LAST_ACK))) {
2232 		/* The last check adjusts for discrepancy of Linux wrt. RFC
2233 		 * states
2234 		 */
2235 		tcp_send_active_reset(sk, gfp_any());
2236 		sk->sk_err = ECONNRESET;
2237 	} else if (old_state == TCP_SYN_SENT)
2238 		sk->sk_err = ECONNRESET;
2239 
2240 	tcp_clear_xmit_timers(sk);
2241 	__skb_queue_purge(&sk->sk_receive_queue);
2242 	tcp_write_queue_purge(sk);
2243 	skb_rbtree_purge(&tp->out_of_order_queue);
2244 
2245 	inet->inet_dport = 0;
2246 
2247 	if (!(sk->sk_userlocks & SOCK_BINDADDR_LOCK))
2248 		inet_reset_saddr(sk);
2249 
2250 	sk->sk_shutdown = 0;
2251 	sock_reset_flag(sk, SOCK_DONE);
2252 	tp->srtt_us = 0;
2253 	tp->write_seq += tp->max_window + 2;
2254 	if (tp->write_seq == 0)
2255 		tp->write_seq = 1;
2256 	tp->snd_cwnd = 2;
2257 	icsk->icsk_probes_out = 0;
2258 	tp->packets_out = 0;
2259 	tp->snd_ssthresh = TCP_INFINITE_SSTHRESH;
2260 	tp->snd_cwnd_cnt = 0;
2261 	tp->window_clamp = 0;
2262 	if (icsk->icsk_ca_ops->release)
2263 		icsk->icsk_ca_ops->release(sk);
2264 	memset(icsk->icsk_ca_priv, 0, sizeof(icsk->icsk_ca_priv));
2265 	tcp_set_ca_state(sk, TCP_CA_Open);
2266 	tcp_clear_retrans(tp);
2267 	tp->total_retrans = 0;
2268 	inet_csk_delack_init(sk);
2269 	/* Initialize rcv_mss to TCP_MIN_MSS to avoid division by 0
2270 	 * issue in __tcp_select_window()
2271 	 */
2272 	icsk->icsk_ack.rcv_mss = TCP_MIN_MSS;
2273 	tcp_init_send_head(sk);
2274 	memset(&tp->rx_opt, 0, sizeof(tp->rx_opt));
2275 	__sk_dst_reset(sk);
2276 	dst_release(sk->sk_rx_dst);
2277 	sk->sk_rx_dst = NULL;
2278 	tcp_saved_syn_free(tp);
2279 	tp->segs_in = 0;
2280 	tp->segs_out = 0;
2281 	tp->bytes_acked = 0;
2282 	tp->bytes_received = 0;
2283 
2284 	WARN_ON(inet->inet_num && !icsk->icsk_bind_hash);
2285 
2286 	if (sk->sk_frag.page) {
2287 		put_page(sk->sk_frag.page);
2288 		sk->sk_frag.page = NULL;
2289 		sk->sk_frag.offset = 0;
2290 	}
2291 
2292 	sk->sk_error_report(sk);
2293 	return err;
2294 }
2295 EXPORT_SYMBOL(tcp_disconnect);
2296 
tcp_can_repair_sock(const struct sock * sk)2297 static inline bool tcp_can_repair_sock(const struct sock *sk)
2298 {
2299 	return ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN) &&
2300 		((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_ESTABLISHED));
2301 }
2302 
tcp_repair_options_est(struct tcp_sock * tp,struct tcp_repair_opt __user * optbuf,unsigned int len)2303 static int tcp_repair_options_est(struct tcp_sock *tp,
2304 		struct tcp_repair_opt __user *optbuf, unsigned int len)
2305 {
2306 	struct tcp_repair_opt opt;
2307 
2308 	while (len >= sizeof(opt)) {
2309 		if (copy_from_user(&opt, optbuf, sizeof(opt)))
2310 			return -EFAULT;
2311 
2312 		optbuf++;
2313 		len -= sizeof(opt);
2314 
2315 		switch (opt.opt_code) {
2316 		case TCPOPT_MSS:
2317 			tp->rx_opt.mss_clamp = opt.opt_val;
2318 			break;
2319 		case TCPOPT_WINDOW:
2320 			{
2321 				u16 snd_wscale = opt.opt_val & 0xFFFF;
2322 				u16 rcv_wscale = opt.opt_val >> 16;
2323 
2324 				if (snd_wscale > 14 || rcv_wscale > 14)
2325 					return -EFBIG;
2326 
2327 				tp->rx_opt.snd_wscale = snd_wscale;
2328 				tp->rx_opt.rcv_wscale = rcv_wscale;
2329 				tp->rx_opt.wscale_ok = 1;
2330 			}
2331 			break;
2332 		case TCPOPT_SACK_PERM:
2333 			if (opt.opt_val != 0)
2334 				return -EINVAL;
2335 
2336 			tp->rx_opt.sack_ok |= TCP_SACK_SEEN;
2337 			if (sysctl_tcp_fack)
2338 				tcp_enable_fack(tp);
2339 			break;
2340 		case TCPOPT_TIMESTAMP:
2341 			if (opt.opt_val != 0)
2342 				return -EINVAL;
2343 
2344 			tp->rx_opt.tstamp_ok = 1;
2345 			break;
2346 		}
2347 	}
2348 
2349 	return 0;
2350 }
2351 
2352 /*
2353  *	Socket option code for TCP.
2354  */
do_tcp_setsockopt(struct sock * sk,int level,int optname,char __user * optval,unsigned int optlen)2355 static int do_tcp_setsockopt(struct sock *sk, int level,
2356 		int optname, char __user *optval, unsigned int optlen)
2357 {
2358 	struct tcp_sock *tp = tcp_sk(sk);
2359 	struct inet_connection_sock *icsk = inet_csk(sk);
2360 	int val;
2361 	int err = 0;
2362 
2363 	/* These are data/string values, all the others are ints */
2364 	switch (optname) {
2365 	case TCP_CONGESTION: {
2366 		char name[TCP_CA_NAME_MAX];
2367 
2368 		if (optlen < 1)
2369 			return -EINVAL;
2370 
2371 		val = strncpy_from_user(name, optval,
2372 					min_t(long, TCP_CA_NAME_MAX-1, optlen));
2373 		if (val < 0)
2374 			return -EFAULT;
2375 		name[val] = 0;
2376 
2377 		lock_sock(sk);
2378 		err = tcp_set_congestion_control(sk, name);
2379 		release_sock(sk);
2380 		return err;
2381 	}
2382 	default:
2383 		/* fallthru */
2384 		break;
2385 	}
2386 
2387 	if (optlen < sizeof(int))
2388 		return -EINVAL;
2389 
2390 	if (get_user(val, (int __user *)optval))
2391 		return -EFAULT;
2392 
2393 	lock_sock(sk);
2394 
2395 	switch (optname) {
2396 	case TCP_MAXSEG:
2397 		/* Values greater than interface MTU won't take effect. However
2398 		 * at the point when this call is done we typically don't yet
2399 		 * know which interface is going to be used */
2400 		if (val < TCP_MIN_MSS || val > MAX_TCP_WINDOW) {
2401 			err = -EINVAL;
2402 			break;
2403 		}
2404 		tp->rx_opt.user_mss = val;
2405 		break;
2406 
2407 	case TCP_NODELAY:
2408 		if (val) {
2409 			/* TCP_NODELAY is weaker than TCP_CORK, so that
2410 			 * this option on corked socket is remembered, but
2411 			 * it is not activated until cork is cleared.
2412 			 *
2413 			 * However, when TCP_NODELAY is set we make
2414 			 * an explicit push, which overrides even TCP_CORK
2415 			 * for currently queued segments.
2416 			 */
2417 			tp->nonagle |= TCP_NAGLE_OFF|TCP_NAGLE_PUSH;
2418 			tcp_push_pending_frames(sk);
2419 		} else {
2420 			tp->nonagle &= ~TCP_NAGLE_OFF;
2421 		}
2422 		break;
2423 
2424 	case TCP_THIN_LINEAR_TIMEOUTS:
2425 		if (val < 0 || val > 1)
2426 			err = -EINVAL;
2427 		else
2428 			tp->thin_lto = val;
2429 		break;
2430 
2431 	case TCP_THIN_DUPACK:
2432 		if (val < 0 || val > 1)
2433 			err = -EINVAL;
2434 		else {
2435 			tp->thin_dupack = val;
2436 			if (tp->thin_dupack)
2437 				tcp_disable_early_retrans(tp);
2438 		}
2439 		break;
2440 
2441 	case TCP_REPAIR:
2442 		if (!tcp_can_repair_sock(sk))
2443 			err = -EPERM;
2444 		else if (val == 1) {
2445 			tp->repair = 1;
2446 			sk->sk_reuse = SK_FORCE_REUSE;
2447 			tp->repair_queue = TCP_NO_QUEUE;
2448 		} else if (val == 0) {
2449 			tp->repair = 0;
2450 			sk->sk_reuse = SK_NO_REUSE;
2451 			tcp_send_window_probe(sk);
2452 		} else
2453 			err = -EINVAL;
2454 
2455 		break;
2456 
2457 	case TCP_REPAIR_QUEUE:
2458 		if (!tp->repair)
2459 			err = -EPERM;
2460 		else if ((unsigned int)val < TCP_QUEUES_NR)
2461 			tp->repair_queue = val;
2462 		else
2463 			err = -EINVAL;
2464 		break;
2465 
2466 	case TCP_QUEUE_SEQ:
2467 		if (sk->sk_state != TCP_CLOSE)
2468 			err = -EPERM;
2469 		else if (tp->repair_queue == TCP_SEND_QUEUE)
2470 			tp->write_seq = val;
2471 		else if (tp->repair_queue == TCP_RECV_QUEUE)
2472 			tp->rcv_nxt = val;
2473 		else
2474 			err = -EINVAL;
2475 		break;
2476 
2477 	case TCP_REPAIR_OPTIONS:
2478 		if (!tp->repair)
2479 			err = -EINVAL;
2480 		else if (sk->sk_state == TCP_ESTABLISHED)
2481 			err = tcp_repair_options_est(tp,
2482 					(struct tcp_repair_opt __user *)optval,
2483 					optlen);
2484 		else
2485 			err = -EPERM;
2486 		break;
2487 
2488 	case TCP_CORK:
2489 		/* When set indicates to always queue non-full frames.
2490 		 * Later the user clears this option and we transmit
2491 		 * any pending partial frames in the queue.  This is
2492 		 * meant to be used alongside sendfile() to get properly
2493 		 * filled frames when the user (for example) must write
2494 		 * out headers with a write() call first and then use
2495 		 * sendfile to send out the data parts.
2496 		 *
2497 		 * TCP_CORK can be set together with TCP_NODELAY and it is
2498 		 * stronger than TCP_NODELAY.
2499 		 */
2500 		if (val) {
2501 			tp->nonagle |= TCP_NAGLE_CORK;
2502 		} else {
2503 			tp->nonagle &= ~TCP_NAGLE_CORK;
2504 			if (tp->nonagle&TCP_NAGLE_OFF)
2505 				tp->nonagle |= TCP_NAGLE_PUSH;
2506 			tcp_push_pending_frames(sk);
2507 		}
2508 		break;
2509 
2510 	case TCP_KEEPIDLE:
2511 		if (val < 1 || val > MAX_TCP_KEEPIDLE)
2512 			err = -EINVAL;
2513 		else {
2514 			tp->keepalive_time = val * HZ;
2515 			if (sock_flag(sk, SOCK_KEEPOPEN) &&
2516 			    !((1 << sk->sk_state) &
2517 			      (TCPF_CLOSE | TCPF_LISTEN))) {
2518 				u32 elapsed = keepalive_time_elapsed(tp);
2519 				if (tp->keepalive_time > elapsed)
2520 					elapsed = tp->keepalive_time - elapsed;
2521 				else
2522 					elapsed = 0;
2523 				inet_csk_reset_keepalive_timer(sk, elapsed);
2524 			}
2525 		}
2526 		break;
2527 	case TCP_KEEPINTVL:
2528 		if (val < 1 || val > MAX_TCP_KEEPINTVL)
2529 			err = -EINVAL;
2530 		else
2531 			tp->keepalive_intvl = val * HZ;
2532 		break;
2533 	case TCP_KEEPCNT:
2534 		if (val < 1 || val > MAX_TCP_KEEPCNT)
2535 			err = -EINVAL;
2536 		else
2537 			tp->keepalive_probes = val;
2538 		break;
2539 	case TCP_SYNCNT:
2540 		if (val < 1 || val > MAX_TCP_SYNCNT)
2541 			err = -EINVAL;
2542 		else
2543 			icsk->icsk_syn_retries = val;
2544 		break;
2545 
2546 	case TCP_SAVE_SYN:
2547 		if (val < 0 || val > 1)
2548 			err = -EINVAL;
2549 		else
2550 			tp->save_syn = val;
2551 		break;
2552 
2553 	case TCP_LINGER2:
2554 		if (val < 0)
2555 			tp->linger2 = -1;
2556 		else if (val > sysctl_tcp_fin_timeout / HZ)
2557 			tp->linger2 = 0;
2558 		else
2559 			tp->linger2 = val * HZ;
2560 		break;
2561 
2562 	case TCP_DEFER_ACCEPT:
2563 		/* Translate value in seconds to number of retransmits */
2564 		icsk->icsk_accept_queue.rskq_defer_accept =
2565 			secs_to_retrans(val, TCP_TIMEOUT_INIT / HZ,
2566 					TCP_RTO_MAX / HZ);
2567 		break;
2568 
2569 	case TCP_WINDOW_CLAMP:
2570 		if (!val) {
2571 			if (sk->sk_state != TCP_CLOSE) {
2572 				err = -EINVAL;
2573 				break;
2574 			}
2575 			tp->window_clamp = 0;
2576 		} else
2577 			tp->window_clamp = val < SOCK_MIN_RCVBUF / 2 ?
2578 						SOCK_MIN_RCVBUF / 2 : val;
2579 		break;
2580 
2581 	case TCP_QUICKACK:
2582 		if (!val) {
2583 			icsk->icsk_ack.pingpong = 1;
2584 		} else {
2585 			icsk->icsk_ack.pingpong = 0;
2586 			if ((1 << sk->sk_state) &
2587 			    (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT) &&
2588 			    inet_csk_ack_scheduled(sk)) {
2589 				icsk->icsk_ack.pending |= ICSK_ACK_PUSHED;
2590 				tcp_cleanup_rbuf(sk, 1);
2591 				if (!(val & 1))
2592 					icsk->icsk_ack.pingpong = 1;
2593 			}
2594 		}
2595 		break;
2596 
2597 #ifdef CONFIG_TCP_MD5SIG
2598 	case TCP_MD5SIG:
2599 		err = tp->af_specific->md5_parse(sk, optval, optlen);
2600 		break;
2601 #endif
2602 	case TCP_USER_TIMEOUT:
2603 		/* Cap the max time in ms TCP will retry or probe the window
2604 		 * before giving up and aborting (ETIMEDOUT) a connection.
2605 		 */
2606 		if (val < 0)
2607 			err = -EINVAL;
2608 		else
2609 			icsk->icsk_user_timeout = msecs_to_jiffies(val);
2610 		break;
2611 
2612 	case TCP_FASTOPEN:
2613 		if (val >= 0 && ((1 << sk->sk_state) & (TCPF_CLOSE |
2614 		    TCPF_LISTEN))) {
2615 			tcp_fastopen_init_key_once(true);
2616 
2617 			fastopen_queue_tune(sk, val);
2618 		} else {
2619 			err = -EINVAL;
2620 		}
2621 		break;
2622 	case TCP_TIMESTAMP:
2623 		if (!tp->repair)
2624 			err = -EPERM;
2625 		else
2626 			tp->tsoffset = val - tcp_time_stamp;
2627 		break;
2628 	case TCP_NOTSENT_LOWAT:
2629 		tp->notsent_lowat = val;
2630 		sk->sk_write_space(sk);
2631 		break;
2632 	default:
2633 		err = -ENOPROTOOPT;
2634 		break;
2635 	}
2636 
2637 	release_sock(sk);
2638 	return err;
2639 }
2640 
tcp_setsockopt(struct sock * sk,int level,int optname,char __user * optval,unsigned int optlen)2641 int tcp_setsockopt(struct sock *sk, int level, int optname, char __user *optval,
2642 		   unsigned int optlen)
2643 {
2644 	const struct inet_connection_sock *icsk = inet_csk(sk);
2645 
2646 	if (level != SOL_TCP)
2647 		return icsk->icsk_af_ops->setsockopt(sk, level, optname,
2648 						     optval, optlen);
2649 	return do_tcp_setsockopt(sk, level, optname, optval, optlen);
2650 }
2651 EXPORT_SYMBOL(tcp_setsockopt);
2652 
2653 #ifdef CONFIG_COMPAT
compat_tcp_setsockopt(struct sock * sk,int level,int optname,char __user * optval,unsigned int optlen)2654 int compat_tcp_setsockopt(struct sock *sk, int level, int optname,
2655 			  char __user *optval, unsigned int optlen)
2656 {
2657 	if (level != SOL_TCP)
2658 		return inet_csk_compat_setsockopt(sk, level, optname,
2659 						  optval, optlen);
2660 	return do_tcp_setsockopt(sk, level, optname, optval, optlen);
2661 }
2662 EXPORT_SYMBOL(compat_tcp_setsockopt);
2663 #endif
2664 
2665 /* Return information about state of tcp endpoint in API format. */
tcp_get_info(struct sock * sk,struct tcp_info * info)2666 void tcp_get_info(struct sock *sk, struct tcp_info *info)
2667 {
2668 	const struct tcp_sock *tp = tcp_sk(sk); /* iff sk_type == SOCK_STREAM */
2669 	const struct inet_connection_sock *icsk = inet_csk(sk);
2670 	u32 now = tcp_time_stamp;
2671 	unsigned int start;
2672 	u64 rate64;
2673 	u32 rate;
2674 
2675 	memset(info, 0, sizeof(*info));
2676 	if (sk->sk_type != SOCK_STREAM)
2677 		return;
2678 
2679 	info->tcpi_state = sk_state_load(sk);
2680 
2681 	info->tcpi_ca_state = icsk->icsk_ca_state;
2682 	info->tcpi_retransmits = icsk->icsk_retransmits;
2683 	info->tcpi_probes = icsk->icsk_probes_out;
2684 	info->tcpi_backoff = icsk->icsk_backoff;
2685 
2686 	if (tp->rx_opt.tstamp_ok)
2687 		info->tcpi_options |= TCPI_OPT_TIMESTAMPS;
2688 	if (tcp_is_sack(tp))
2689 		info->tcpi_options |= TCPI_OPT_SACK;
2690 	if (tp->rx_opt.wscale_ok) {
2691 		info->tcpi_options |= TCPI_OPT_WSCALE;
2692 		info->tcpi_snd_wscale = tp->rx_opt.snd_wscale;
2693 		info->tcpi_rcv_wscale = tp->rx_opt.rcv_wscale;
2694 	}
2695 
2696 	if (tp->ecn_flags & TCP_ECN_OK)
2697 		info->tcpi_options |= TCPI_OPT_ECN;
2698 	if (tp->ecn_flags & TCP_ECN_SEEN)
2699 		info->tcpi_options |= TCPI_OPT_ECN_SEEN;
2700 	if (tp->syn_data_acked)
2701 		info->tcpi_options |= TCPI_OPT_SYN_DATA;
2702 
2703 	info->tcpi_rto = jiffies_to_usecs(icsk->icsk_rto);
2704 	info->tcpi_ato = jiffies_to_usecs(icsk->icsk_ack.ato);
2705 	info->tcpi_snd_mss = tp->mss_cache;
2706 	info->tcpi_rcv_mss = icsk->icsk_ack.rcv_mss;
2707 
2708 	if (info->tcpi_state == TCP_LISTEN) {
2709 		info->tcpi_unacked = sk->sk_ack_backlog;
2710 		info->tcpi_sacked = sk->sk_max_ack_backlog;
2711 	} else {
2712 		info->tcpi_unacked = tp->packets_out;
2713 		info->tcpi_sacked = tp->sacked_out;
2714 	}
2715 	info->tcpi_lost = tp->lost_out;
2716 	info->tcpi_retrans = tp->retrans_out;
2717 	info->tcpi_fackets = tp->fackets_out;
2718 
2719 	info->tcpi_last_data_sent = jiffies_to_msecs(now - tp->lsndtime);
2720 	info->tcpi_last_data_recv = jiffies_to_msecs(now - icsk->icsk_ack.lrcvtime);
2721 	info->tcpi_last_ack_recv = jiffies_to_msecs(now - tp->rcv_tstamp);
2722 
2723 	info->tcpi_pmtu = icsk->icsk_pmtu_cookie;
2724 	info->tcpi_rcv_ssthresh = tp->rcv_ssthresh;
2725 	info->tcpi_rtt = tp->srtt_us >> 3;
2726 	info->tcpi_rttvar = tp->mdev_us >> 2;
2727 	info->tcpi_snd_ssthresh = tp->snd_ssthresh;
2728 	info->tcpi_snd_cwnd = tp->snd_cwnd;
2729 	info->tcpi_advmss = tp->advmss;
2730 	info->tcpi_reordering = tp->reordering;
2731 
2732 	info->tcpi_rcv_rtt = jiffies_to_usecs(tp->rcv_rtt_est.rtt)>>3;
2733 	info->tcpi_rcv_space = tp->rcvq_space.space;
2734 
2735 	info->tcpi_total_retrans = tp->total_retrans;
2736 
2737 	rate = READ_ONCE(sk->sk_pacing_rate);
2738 	rate64 = rate != ~0U ? rate : ~0ULL;
2739 	put_unaligned(rate64, &info->tcpi_pacing_rate);
2740 
2741 	rate = READ_ONCE(sk->sk_max_pacing_rate);
2742 	rate64 = rate != ~0U ? rate : ~0ULL;
2743 	put_unaligned(rate64, &info->tcpi_max_pacing_rate);
2744 
2745 	do {
2746 		start = u64_stats_fetch_begin_irq(&tp->syncp);
2747 		put_unaligned(tp->bytes_acked, &info->tcpi_bytes_acked);
2748 		put_unaligned(tp->bytes_received, &info->tcpi_bytes_received);
2749 	} while (u64_stats_fetch_retry_irq(&tp->syncp, start));
2750 	info->tcpi_segs_out = tp->segs_out;
2751 	info->tcpi_segs_in = tp->segs_in;
2752 }
2753 EXPORT_SYMBOL_GPL(tcp_get_info);
2754 
do_tcp_getsockopt(struct sock * sk,int level,int optname,char __user * optval,int __user * optlen)2755 static int do_tcp_getsockopt(struct sock *sk, int level,
2756 		int optname, char __user *optval, int __user *optlen)
2757 {
2758 	struct inet_connection_sock *icsk = inet_csk(sk);
2759 	struct tcp_sock *tp = tcp_sk(sk);
2760 	int val, len;
2761 
2762 	if (get_user(len, optlen))
2763 		return -EFAULT;
2764 
2765 	len = min_t(unsigned int, len, sizeof(int));
2766 
2767 	if (len < 0)
2768 		return -EINVAL;
2769 
2770 	switch (optname) {
2771 	case TCP_MAXSEG:
2772 		val = tp->mss_cache;
2773 		if (!val && ((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN)))
2774 			val = tp->rx_opt.user_mss;
2775 		if (tp->repair)
2776 			val = tp->rx_opt.mss_clamp;
2777 		break;
2778 	case TCP_NODELAY:
2779 		val = !!(tp->nonagle&TCP_NAGLE_OFF);
2780 		break;
2781 	case TCP_CORK:
2782 		val = !!(tp->nonagle&TCP_NAGLE_CORK);
2783 		break;
2784 	case TCP_KEEPIDLE:
2785 		val = keepalive_time_when(tp) / HZ;
2786 		break;
2787 	case TCP_KEEPINTVL:
2788 		val = keepalive_intvl_when(tp) / HZ;
2789 		break;
2790 	case TCP_KEEPCNT:
2791 		val = keepalive_probes(tp);
2792 		break;
2793 	case TCP_SYNCNT:
2794 		val = icsk->icsk_syn_retries ? : sysctl_tcp_syn_retries;
2795 		break;
2796 	case TCP_LINGER2:
2797 		val = tp->linger2;
2798 		if (val >= 0)
2799 			val = (val ? : sysctl_tcp_fin_timeout) / HZ;
2800 		break;
2801 	case TCP_DEFER_ACCEPT:
2802 		val = retrans_to_secs(icsk->icsk_accept_queue.rskq_defer_accept,
2803 				      TCP_TIMEOUT_INIT / HZ, TCP_RTO_MAX / HZ);
2804 		break;
2805 	case TCP_WINDOW_CLAMP:
2806 		val = tp->window_clamp;
2807 		break;
2808 	case TCP_INFO: {
2809 		struct tcp_info info;
2810 
2811 		if (get_user(len, optlen))
2812 			return -EFAULT;
2813 
2814 		tcp_get_info(sk, &info);
2815 
2816 		len = min_t(unsigned int, len, sizeof(info));
2817 		if (put_user(len, optlen))
2818 			return -EFAULT;
2819 		if (copy_to_user(optval, &info, len))
2820 			return -EFAULT;
2821 		return 0;
2822 	}
2823 	case TCP_CC_INFO: {
2824 		const struct tcp_congestion_ops *ca_ops;
2825 		union tcp_cc_info info;
2826 		size_t sz = 0;
2827 		int attr;
2828 
2829 		if (get_user(len, optlen))
2830 			return -EFAULT;
2831 
2832 		ca_ops = icsk->icsk_ca_ops;
2833 		if (ca_ops && ca_ops->get_info)
2834 			sz = ca_ops->get_info(sk, ~0U, &attr, &info);
2835 
2836 		len = min_t(unsigned int, len, sz);
2837 		if (put_user(len, optlen))
2838 			return -EFAULT;
2839 		if (copy_to_user(optval, &info, len))
2840 			return -EFAULT;
2841 		return 0;
2842 	}
2843 	case TCP_QUICKACK:
2844 		val = !icsk->icsk_ack.pingpong;
2845 		break;
2846 
2847 	case TCP_CONGESTION:
2848 		if (get_user(len, optlen))
2849 			return -EFAULT;
2850 		len = min_t(unsigned int, len, TCP_CA_NAME_MAX);
2851 		if (put_user(len, optlen))
2852 			return -EFAULT;
2853 		if (copy_to_user(optval, icsk->icsk_ca_ops->name, len))
2854 			return -EFAULT;
2855 		return 0;
2856 
2857 	case TCP_THIN_LINEAR_TIMEOUTS:
2858 		val = tp->thin_lto;
2859 		break;
2860 	case TCP_THIN_DUPACK:
2861 		val = tp->thin_dupack;
2862 		break;
2863 
2864 	case TCP_REPAIR:
2865 		val = tp->repair;
2866 		break;
2867 
2868 	case TCP_REPAIR_QUEUE:
2869 		if (tp->repair)
2870 			val = tp->repair_queue;
2871 		else
2872 			return -EINVAL;
2873 		break;
2874 
2875 	case TCP_QUEUE_SEQ:
2876 		if (tp->repair_queue == TCP_SEND_QUEUE)
2877 			val = tp->write_seq;
2878 		else if (tp->repair_queue == TCP_RECV_QUEUE)
2879 			val = tp->rcv_nxt;
2880 		else
2881 			return -EINVAL;
2882 		break;
2883 
2884 	case TCP_USER_TIMEOUT:
2885 		val = jiffies_to_msecs(icsk->icsk_user_timeout);
2886 		break;
2887 
2888 	case TCP_FASTOPEN:
2889 		val = icsk->icsk_accept_queue.fastopenq.max_qlen;
2890 		break;
2891 
2892 	case TCP_TIMESTAMP:
2893 		val = tcp_time_stamp + tp->tsoffset;
2894 		break;
2895 	case TCP_NOTSENT_LOWAT:
2896 		val = tp->notsent_lowat;
2897 		break;
2898 	case TCP_SAVE_SYN:
2899 		val = tp->save_syn;
2900 		break;
2901 	case TCP_SAVED_SYN: {
2902 		if (get_user(len, optlen))
2903 			return -EFAULT;
2904 
2905 		lock_sock(sk);
2906 		if (tp->saved_syn) {
2907 			if (len < tp->saved_syn[0]) {
2908 				if (put_user(tp->saved_syn[0], optlen)) {
2909 					release_sock(sk);
2910 					return -EFAULT;
2911 				}
2912 				release_sock(sk);
2913 				return -EINVAL;
2914 			}
2915 			len = tp->saved_syn[0];
2916 			if (put_user(len, optlen)) {
2917 				release_sock(sk);
2918 				return -EFAULT;
2919 			}
2920 			if (copy_to_user(optval, tp->saved_syn + 1, len)) {
2921 				release_sock(sk);
2922 				return -EFAULT;
2923 			}
2924 			tcp_saved_syn_free(tp);
2925 			release_sock(sk);
2926 		} else {
2927 			release_sock(sk);
2928 			len = 0;
2929 			if (put_user(len, optlen))
2930 				return -EFAULT;
2931 		}
2932 		return 0;
2933 	}
2934 	default:
2935 		return -ENOPROTOOPT;
2936 	}
2937 
2938 	if (put_user(len, optlen))
2939 		return -EFAULT;
2940 	if (copy_to_user(optval, &val, len))
2941 		return -EFAULT;
2942 	return 0;
2943 }
2944 
tcp_getsockopt(struct sock * sk,int level,int optname,char __user * optval,int __user * optlen)2945 int tcp_getsockopt(struct sock *sk, int level, int optname, char __user *optval,
2946 		   int __user *optlen)
2947 {
2948 	struct inet_connection_sock *icsk = inet_csk(sk);
2949 
2950 	if (level != SOL_TCP)
2951 		return icsk->icsk_af_ops->getsockopt(sk, level, optname,
2952 						     optval, optlen);
2953 	return do_tcp_getsockopt(sk, level, optname, optval, optlen);
2954 }
2955 EXPORT_SYMBOL(tcp_getsockopt);
2956 
2957 #ifdef CONFIG_COMPAT
compat_tcp_getsockopt(struct sock * sk,int level,int optname,char __user * optval,int __user * optlen)2958 int compat_tcp_getsockopt(struct sock *sk, int level, int optname,
2959 			  char __user *optval, int __user *optlen)
2960 {
2961 	if (level != SOL_TCP)
2962 		return inet_csk_compat_getsockopt(sk, level, optname,
2963 						  optval, optlen);
2964 	return do_tcp_getsockopt(sk, level, optname, optval, optlen);
2965 }
2966 EXPORT_SYMBOL(compat_tcp_getsockopt);
2967 #endif
2968 
2969 #ifdef CONFIG_TCP_MD5SIG
2970 static DEFINE_PER_CPU(struct tcp_md5sig_pool, tcp_md5sig_pool);
2971 static DEFINE_MUTEX(tcp_md5sig_mutex);
2972 static bool tcp_md5sig_pool_populated = false;
2973 
__tcp_alloc_md5sig_pool(void)2974 static void __tcp_alloc_md5sig_pool(void)
2975 {
2976 	int cpu;
2977 
2978 	for_each_possible_cpu(cpu) {
2979 		if (!per_cpu(tcp_md5sig_pool, cpu).md5_desc.tfm) {
2980 			struct crypto_hash *hash;
2981 
2982 			hash = crypto_alloc_hash("md5", 0, CRYPTO_ALG_ASYNC);
2983 			if (IS_ERR_OR_NULL(hash))
2984 				return;
2985 			per_cpu(tcp_md5sig_pool, cpu).md5_desc.tfm = hash;
2986 		}
2987 	}
2988 	/* before setting tcp_md5sig_pool_populated, we must commit all writes
2989 	 * to memory. See smp_rmb() in tcp_get_md5sig_pool()
2990 	 */
2991 	smp_wmb();
2992 	tcp_md5sig_pool_populated = true;
2993 }
2994 
tcp_alloc_md5sig_pool(void)2995 bool tcp_alloc_md5sig_pool(void)
2996 {
2997 	if (unlikely(!tcp_md5sig_pool_populated)) {
2998 		mutex_lock(&tcp_md5sig_mutex);
2999 
3000 		if (!tcp_md5sig_pool_populated)
3001 			__tcp_alloc_md5sig_pool();
3002 
3003 		mutex_unlock(&tcp_md5sig_mutex);
3004 	}
3005 	return tcp_md5sig_pool_populated;
3006 }
3007 EXPORT_SYMBOL(tcp_alloc_md5sig_pool);
3008 
3009 
3010 /**
3011  *	tcp_get_md5sig_pool - get md5sig_pool for this user
3012  *
3013  *	We use percpu structure, so if we succeed, we exit with preemption
3014  *	and BH disabled, to make sure another thread or softirq handling
3015  *	wont try to get same context.
3016  */
tcp_get_md5sig_pool(void)3017 struct tcp_md5sig_pool *tcp_get_md5sig_pool(void)
3018 {
3019 	local_bh_disable();
3020 
3021 	if (tcp_md5sig_pool_populated) {
3022 		/* coupled with smp_wmb() in __tcp_alloc_md5sig_pool() */
3023 		smp_rmb();
3024 		return this_cpu_ptr(&tcp_md5sig_pool);
3025 	}
3026 	local_bh_enable();
3027 	return NULL;
3028 }
3029 EXPORT_SYMBOL(tcp_get_md5sig_pool);
3030 
tcp_md5_hash_header(struct tcp_md5sig_pool * hp,const struct tcphdr * th)3031 int tcp_md5_hash_header(struct tcp_md5sig_pool *hp,
3032 			const struct tcphdr *th)
3033 {
3034 	struct scatterlist sg;
3035 	struct tcphdr hdr;
3036 	int err;
3037 
3038 	/* We are not allowed to change tcphdr, make a local copy */
3039 	memcpy(&hdr, th, sizeof(hdr));
3040 	hdr.check = 0;
3041 
3042 	/* options aren't included in the hash */
3043 	sg_init_one(&sg, &hdr, sizeof(hdr));
3044 	err = crypto_hash_update(&hp->md5_desc, &sg, sizeof(hdr));
3045 	return err;
3046 }
3047 EXPORT_SYMBOL(tcp_md5_hash_header);
3048 
tcp_md5_hash_skb_data(struct tcp_md5sig_pool * hp,const struct sk_buff * skb,unsigned int header_len)3049 int tcp_md5_hash_skb_data(struct tcp_md5sig_pool *hp,
3050 			  const struct sk_buff *skb, unsigned int header_len)
3051 {
3052 	struct scatterlist sg;
3053 	const struct tcphdr *tp = tcp_hdr(skb);
3054 	struct hash_desc *desc = &hp->md5_desc;
3055 	unsigned int i;
3056 	const unsigned int head_data_len = skb_headlen(skb) > header_len ?
3057 					   skb_headlen(skb) - header_len : 0;
3058 	const struct skb_shared_info *shi = skb_shinfo(skb);
3059 	struct sk_buff *frag_iter;
3060 
3061 	sg_init_table(&sg, 1);
3062 
3063 	sg_set_buf(&sg, ((u8 *) tp) + header_len, head_data_len);
3064 	if (crypto_hash_update(desc, &sg, head_data_len))
3065 		return 1;
3066 
3067 	for (i = 0; i < shi->nr_frags; ++i) {
3068 		const struct skb_frag_struct *f = &shi->frags[i];
3069 		unsigned int offset = f->page_offset;
3070 		struct page *page = skb_frag_page(f) + (offset >> PAGE_SHIFT);
3071 
3072 		sg_set_page(&sg, page, skb_frag_size(f),
3073 			    offset_in_page(offset));
3074 		if (crypto_hash_update(desc, &sg, skb_frag_size(f)))
3075 			return 1;
3076 	}
3077 
3078 	skb_walk_frags(skb, frag_iter)
3079 		if (tcp_md5_hash_skb_data(hp, frag_iter, 0))
3080 			return 1;
3081 
3082 	return 0;
3083 }
3084 EXPORT_SYMBOL(tcp_md5_hash_skb_data);
3085 
tcp_md5_hash_key(struct tcp_md5sig_pool * hp,const struct tcp_md5sig_key * key)3086 int tcp_md5_hash_key(struct tcp_md5sig_pool *hp, const struct tcp_md5sig_key *key)
3087 {
3088 	u8 keylen = READ_ONCE(key->keylen); /* paired with WRITE_ONCE() in tcp_md5_do_add */
3089 	struct scatterlist sg;
3090 
3091 	sg_init_one(&sg, key->key, keylen);
3092 
3093 	/* tcp_md5_do_add() might change key->key under us */
3094 	return crypto_hash_update(&hp->md5_desc, &sg, key->keylen);
3095 }
3096 EXPORT_SYMBOL(tcp_md5_hash_key);
3097 
3098 #endif
3099 
tcp_done(struct sock * sk)3100 void tcp_done(struct sock *sk)
3101 {
3102 	struct request_sock *req = tcp_sk(sk)->fastopen_rsk;
3103 
3104 	if (sk->sk_state == TCP_SYN_SENT || sk->sk_state == TCP_SYN_RECV)
3105 		TCP_INC_STATS_BH(sock_net(sk), TCP_MIB_ATTEMPTFAILS);
3106 
3107 	tcp_set_state(sk, TCP_CLOSE);
3108 	tcp_clear_xmit_timers(sk);
3109 	if (req)
3110 		reqsk_fastopen_remove(sk, req, false);
3111 
3112 	sk->sk_shutdown = SHUTDOWN_MASK;
3113 
3114 	if (!sock_flag(sk, SOCK_DEAD))
3115 		sk->sk_state_change(sk);
3116 	else
3117 		inet_csk_destroy_sock(sk);
3118 }
3119 EXPORT_SYMBOL_GPL(tcp_done);
3120 
tcp_abort(struct sock * sk,int err)3121 int tcp_abort(struct sock *sk, int err)
3122 {
3123 	if (!sk_fullsock(sk)) {
3124 		if (sk->sk_state == TCP_NEW_SYN_RECV) {
3125 			struct request_sock *req = inet_reqsk(sk);
3126 
3127 			local_bh_disable();
3128 			inet_csk_reqsk_queue_drop_and_put(req->rsk_listener,
3129 							  req);
3130 			local_bh_enable();
3131 			return 0;
3132 		}
3133 		sock_gen_put(sk);
3134 		return -EOPNOTSUPP;
3135 	}
3136 
3137 	/* Don't race with userspace socket closes such as tcp_close. */
3138 	lock_sock(sk);
3139 
3140 	if (sk->sk_state == TCP_LISTEN) {
3141 		tcp_set_state(sk, TCP_CLOSE);
3142 		inet_csk_listen_stop(sk);
3143 	}
3144 
3145 	/* Don't race with BH socket closes such as inet_csk_listen_stop. */
3146 	local_bh_disable();
3147 	bh_lock_sock(sk);
3148 
3149 	if (!sock_flag(sk, SOCK_DEAD)) {
3150 		sk->sk_err = err;
3151 		/* This barrier is coupled with smp_rmb() in tcp_poll() */
3152 		smp_wmb();
3153 		sk->sk_error_report(sk);
3154 		if (tcp_need_reset(sk->sk_state))
3155 			tcp_send_active_reset(sk, GFP_ATOMIC);
3156 		tcp_done(sk);
3157 	}
3158 
3159 	bh_unlock_sock(sk);
3160 	local_bh_enable();
3161 	release_sock(sk);
3162 	sock_put(sk);
3163 	return 0;
3164 }
3165 EXPORT_SYMBOL_GPL(tcp_abort);
3166 
3167 extern struct tcp_congestion_ops tcp_reno;
3168 
3169 static __initdata unsigned long thash_entries;
set_thash_entries(char * str)3170 static int __init set_thash_entries(char *str)
3171 {
3172 	ssize_t ret;
3173 
3174 	if (!str)
3175 		return 0;
3176 
3177 	ret = kstrtoul(str, 0, &thash_entries);
3178 	if (ret)
3179 		return 0;
3180 
3181 	return 1;
3182 }
3183 __setup("thash_entries=", set_thash_entries);
3184 
tcp_init_mem(void)3185 static void __init tcp_init_mem(void)
3186 {
3187 	unsigned long limit = nr_free_buffer_pages() / 16;
3188 
3189 	limit = max(limit, 128UL);
3190 	sysctl_tcp_mem[0] = limit / 4 * 3;		/* 4.68 % */
3191 	sysctl_tcp_mem[1] = limit;			/* 6.25 % */
3192 	sysctl_tcp_mem[2] = sysctl_tcp_mem[0] * 2;	/* 9.37 % */
3193 }
3194 
tcp_init(void)3195 void __init tcp_init(void)
3196 {
3197 	unsigned long limit;
3198 	int max_rshare, max_wshare, cnt;
3199 	unsigned int i;
3200 
3201 	BUILD_BUG_ON(TCP_MIN_SND_MSS <= MAX_TCP_OPTION_SPACE);
3202 	sock_skb_cb_check_size(sizeof(struct tcp_skb_cb));
3203 
3204 	percpu_counter_init(&tcp_sockets_allocated, 0, GFP_KERNEL);
3205 	percpu_counter_init(&tcp_orphan_count, 0, GFP_KERNEL);
3206 	tcp_hashinfo.bind_bucket_cachep =
3207 		kmem_cache_create("tcp_bind_bucket",
3208 				  sizeof(struct inet_bind_bucket), 0,
3209 				  SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
3210 
3211 	/* Size and allocate the main established and bind bucket
3212 	 * hash tables.
3213 	 *
3214 	 * The methodology is similar to that of the buffer cache.
3215 	 */
3216 	tcp_hashinfo.ehash =
3217 		alloc_large_system_hash("TCP established",
3218 					sizeof(struct inet_ehash_bucket),
3219 					thash_entries,
3220 					17, /* one slot per 128 KB of memory */
3221 					0,
3222 					NULL,
3223 					&tcp_hashinfo.ehash_mask,
3224 					0,
3225 					thash_entries ? 0 : 512 * 1024);
3226 	for (i = 0; i <= tcp_hashinfo.ehash_mask; i++)
3227 		INIT_HLIST_NULLS_HEAD(&tcp_hashinfo.ehash[i].chain, i);
3228 
3229 	if (inet_ehash_locks_alloc(&tcp_hashinfo))
3230 		panic("TCP: failed to alloc ehash_locks");
3231 	tcp_hashinfo.bhash =
3232 		alloc_large_system_hash("TCP bind",
3233 					sizeof(struct inet_bind_hashbucket),
3234 					tcp_hashinfo.ehash_mask + 1,
3235 					17, /* one slot per 128 KB of memory */
3236 					0,
3237 					&tcp_hashinfo.bhash_size,
3238 					NULL,
3239 					0,
3240 					64 * 1024);
3241 	tcp_hashinfo.bhash_size = 1U << tcp_hashinfo.bhash_size;
3242 	for (i = 0; i < tcp_hashinfo.bhash_size; i++) {
3243 		spin_lock_init(&tcp_hashinfo.bhash[i].lock);
3244 		INIT_HLIST_HEAD(&tcp_hashinfo.bhash[i].chain);
3245 	}
3246 
3247 
3248 	cnt = tcp_hashinfo.ehash_mask + 1;
3249 
3250 	tcp_death_row.sysctl_max_tw_buckets = cnt / 2;
3251 	sysctl_tcp_max_orphans = cnt / 2;
3252 	sysctl_max_syn_backlog = max(128, cnt / 256);
3253 
3254 	tcp_init_mem();
3255 	/* Set per-socket limits to no more than 1/128 the pressure threshold */
3256 	limit = nr_free_buffer_pages() << (PAGE_SHIFT - 7);
3257 	max_wshare = min(4UL*1024*1024, limit);
3258 	max_rshare = min(6UL*1024*1024, limit);
3259 
3260 	sysctl_tcp_wmem[0] = SK_MEM_QUANTUM;
3261 	sysctl_tcp_wmem[1] = 16*1024;
3262 	sysctl_tcp_wmem[2] = max(64*1024, max_wshare);
3263 
3264 	sysctl_tcp_rmem[0] = SK_MEM_QUANTUM;
3265 	sysctl_tcp_rmem[1] = 87380;
3266 	sysctl_tcp_rmem[2] = max(87380, max_rshare);
3267 
3268 	pr_info("Hash tables configured (established %u bind %u)\n",
3269 		tcp_hashinfo.ehash_mask + 1, tcp_hashinfo.bhash_size);
3270 
3271 	tcp_metrics_init();
3272 	BUG_ON(tcp_register_congestion_control(&tcp_reno) != 0);
3273 	tcp_tasklet_init();
3274 }
3275