1 /*
2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
5 *
6 * Implementation of the Transmission Control Protocol(TCP).
7 *
8 * Authors: Ross Biro
9 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
10 * Mark Evans, <evansmp@uhura.aston.ac.uk>
11 * Corey Minyard <wf-rch!minyard@relay.EU.net>
12 * Florian La Roche, <flla@stud.uni-sb.de>
13 * Charles Hedrick, <hedrick@klinzhai.rutgers.edu>
14 * Linus Torvalds, <torvalds@cs.helsinki.fi>
15 * Alan Cox, <gw4pts@gw4pts.ampr.org>
16 * Matthew Dillon, <dillon@apollo.west.oic.com>
17 * Arnt Gulbrandsen, <agulbra@nvg.unit.no>
18 * Jorge Cwik, <jorge@laser.satlink.net>
19 *
20 * Fixes:
21 * Alan Cox : Numerous verify_area() calls
22 * Alan Cox : Set the ACK bit on a reset
23 * Alan Cox : Stopped it crashing if it closed while
24 * sk->inuse=1 and was trying to connect
25 * (tcp_err()).
26 * Alan Cox : All icmp error handling was broken
27 * pointers passed where wrong and the
28 * socket was looked up backwards. Nobody
29 * tested any icmp error code obviously.
30 * Alan Cox : tcp_err() now handled properly. It
31 * wakes people on errors. poll
32 * behaves and the icmp error race
33 * has gone by moving it into sock.c
34 * Alan Cox : tcp_send_reset() fixed to work for
35 * everything not just packets for
36 * unknown sockets.
37 * Alan Cox : tcp option processing.
38 * Alan Cox : Reset tweaked (still not 100%) [Had
39 * syn rule wrong]
40 * Herp Rosmanith : More reset fixes
41 * Alan Cox : No longer acks invalid rst frames.
42 * Acking any kind of RST is right out.
43 * Alan Cox : Sets an ignore me flag on an rst
44 * receive otherwise odd bits of prattle
45 * escape still
46 * Alan Cox : Fixed another acking RST frame bug.
47 * Should stop LAN workplace lockups.
48 * Alan Cox : Some tidyups using the new skb list
49 * facilities
50 * Alan Cox : sk->keepopen now seems to work
51 * Alan Cox : Pulls options out correctly on accepts
52 * Alan Cox : Fixed assorted sk->rqueue->next errors
53 * Alan Cox : PSH doesn't end a TCP read. Switched a
54 * bit to skb ops.
55 * Alan Cox : Tidied tcp_data to avoid a potential
56 * nasty.
57 * Alan Cox : Added some better commenting, as the
58 * tcp is hard to follow
59 * Alan Cox : Removed incorrect check for 20 * psh
60 * Michael O'Reilly : ack < copied bug fix.
61 * Johannes Stille : Misc tcp fixes (not all in yet).
62 * Alan Cox : FIN with no memory -> CRASH
63 * Alan Cox : Added socket option proto entries.
64 * Also added awareness of them to accept.
65 * Alan Cox : Added TCP options (SOL_TCP)
66 * Alan Cox : Switched wakeup calls to callbacks,
67 * so the kernel can layer network
68 * sockets.
69 * Alan Cox : Use ip_tos/ip_ttl settings.
70 * Alan Cox : Handle FIN (more) properly (we hope).
71 * Alan Cox : RST frames sent on unsynchronised
72 * state ack error.
73 * Alan Cox : Put in missing check for SYN bit.
74 * Alan Cox : Added tcp_select_window() aka NET2E
75 * window non shrink trick.
76 * Alan Cox : Added a couple of small NET2E timer
77 * fixes
78 * Charles Hedrick : TCP fixes
79 * Toomas Tamm : TCP window fixes
80 * Alan Cox : Small URG fix to rlogin ^C ack fight
81 * Charles Hedrick : Rewrote most of it to actually work
82 * Linus : Rewrote tcp_read() and URG handling
83 * completely
84 * Gerhard Koerting: Fixed some missing timer handling
85 * Matthew Dillon : Reworked TCP machine states as per RFC
86 * Gerhard Koerting: PC/TCP workarounds
87 * Adam Caldwell : Assorted timer/timing errors
88 * Matthew Dillon : Fixed another RST bug
89 * Alan Cox : Move to kernel side addressing changes.
90 * Alan Cox : Beginning work on TCP fastpathing
91 * (not yet usable)
92 * Arnt Gulbrandsen: Turbocharged tcp_check() routine.
93 * Alan Cox : TCP fast path debugging
94 * Alan Cox : Window clamping
95 * Michael Riepe : Bug in tcp_check()
96 * Matt Dillon : More TCP improvements and RST bug fixes
97 * Matt Dillon : Yet more small nasties remove from the
98 * TCP code (Be very nice to this man if
99 * tcp finally works 100%) 8)
100 * Alan Cox : BSD accept semantics.
101 * Alan Cox : Reset on closedown bug.
102 * Peter De Schrijver : ENOTCONN check missing in tcp_sendto().
103 * Michael Pall : Handle poll() after URG properly in
104 * all cases.
105 * Michael Pall : Undo the last fix in tcp_read_urg()
106 * (multi URG PUSH broke rlogin).
107 * Michael Pall : Fix the multi URG PUSH problem in
108 * tcp_readable(), poll() after URG
109 * works now.
110 * Michael Pall : recv(...,MSG_OOB) never blocks in the
111 * BSD api.
112 * Alan Cox : Changed the semantics of sk->socket to
113 * fix a race and a signal problem with
114 * accept() and async I/O.
115 * Alan Cox : Relaxed the rules on tcp_sendto().
116 * Yury Shevchuk : Really fixed accept() blocking problem.
117 * Craig I. Hagan : Allow for BSD compatible TIME_WAIT for
118 * clients/servers which listen in on
119 * fixed ports.
120 * Alan Cox : Cleaned the above up and shrank it to
121 * a sensible code size.
122 * Alan Cox : Self connect lockup fix.
123 * Alan Cox : No connect to multicast.
124 * Ross Biro : Close unaccepted children on master
125 * socket close.
126 * Alan Cox : Reset tracing code.
127 * Alan Cox : Spurious resets on shutdown.
128 * Alan Cox : Giant 15 minute/60 second timer error
129 * Alan Cox : Small whoops in polling before an
130 * accept.
131 * Alan Cox : Kept the state trace facility since
132 * it's handy for debugging.
133 * Alan Cox : More reset handler fixes.
134 * Alan Cox : Started rewriting the code based on
135 * the RFC's for other useful protocol
136 * references see: Comer, KA9Q NOS, and
137 * for a reference on the difference
138 * between specifications and how BSD
139 * works see the 4.4lite source.
140 * A.N.Kuznetsov : Don't time wait on completion of tidy
141 * close.
142 * Linus Torvalds : Fin/Shutdown & copied_seq changes.
143 * Linus Torvalds : Fixed BSD port reuse to work first syn
144 * Alan Cox : Reimplemented timers as per the RFC
145 * and using multiple timers for sanity.
146 * Alan Cox : Small bug fixes, and a lot of new
147 * comments.
148 * Alan Cox : Fixed dual reader crash by locking
149 * the buffers (much like datagram.c)
150 * Alan Cox : Fixed stuck sockets in probe. A probe
151 * now gets fed up of retrying without
152 * (even a no space) answer.
153 * Alan Cox : Extracted closing code better
154 * Alan Cox : Fixed the closing state machine to
155 * resemble the RFC.
156 * Alan Cox : More 'per spec' fixes.
157 * Jorge Cwik : Even faster checksumming.
158 * Alan Cox : tcp_data() doesn't ack illegal PSH
159 * only frames. At least one pc tcp stack
160 * generates them.
161 * Alan Cox : Cache last socket.
162 * Alan Cox : Per route irtt.
163 * Matt Day : poll()->select() match BSD precisely on error
164 * Alan Cox : New buffers
165 * Marc Tamsky : Various sk->prot->retransmits and
166 * sk->retransmits misupdating fixed.
167 * Fixed tcp_write_timeout: stuck close,
168 * and TCP syn retries gets used now.
169 * Mark Yarvis : In tcp_read_wakeup(), don't send an
170 * ack if state is TCP_CLOSED.
171 * Alan Cox : Look up device on a retransmit - routes may
172 * change. Doesn't yet cope with MSS shrink right
173 * but it's a start!
174 * Marc Tamsky : Closing in closing fixes.
175 * Mike Shaver : RFC1122 verifications.
176 * Alan Cox : rcv_saddr errors.
177 * Alan Cox : Block double connect().
178 * Alan Cox : Small hooks for enSKIP.
179 * Alexey Kuznetsov: Path MTU discovery.
180 * Alan Cox : Support soft errors.
181 * Alan Cox : Fix MTU discovery pathological case
182 * when the remote claims no mtu!
183 * Marc Tamsky : TCP_CLOSE fix.
184 * Colin (G3TNE) : Send a reset on syn ack replies in
185 * window but wrong (fixes NT lpd problems)
186 * Pedro Roque : Better TCP window handling, delayed ack.
187 * Joerg Reuter : No modification of locked buffers in
188 * tcp_do_retransmit()
189 * Eric Schenk : Changed receiver side silly window
190 * avoidance algorithm to BSD style
191 * algorithm. This doubles throughput
192 * against machines running Solaris,
193 * and seems to result in general
194 * improvement.
195 * Stefan Magdalinski : adjusted tcp_readable() to fix FIONREAD
196 * Willy Konynenberg : Transparent proxying support.
197 * Mike McLagan : Routing by source
198 * Keith Owens : Do proper merging with partial SKB's in
199 * tcp_do_sendmsg to avoid burstiness.
200 * Eric Schenk : Fix fast close down bug with
201 * shutdown() followed by close().
202 * Andi Kleen : Make poll agree with SIGIO
203 * Salvatore Sanfilippo : Support SO_LINGER with linger == 1 and
204 * lingertime == 0 (RFC 793 ABORT Call)
205 * Hirokazu Takahashi : Use copy_from_user() instead of
206 * csum_and_copy_from_user() if possible.
207 *
208 * This program is free software; you can redistribute it and/or
209 * modify it under the terms of the GNU General Public License
210 * as published by the Free Software Foundation; either version
211 * 2 of the License, or(at your option) any later version.
212 *
213 * Description of States:
214 *
215 * TCP_SYN_SENT sent a connection request, waiting for ack
216 *
217 * TCP_SYN_RECV received a connection request, sent ack,
218 * waiting for final ack in three-way handshake.
219 *
220 * TCP_ESTABLISHED connection established
221 *
222 * TCP_FIN_WAIT1 our side has shutdown, waiting to complete
223 * transmission of remaining buffered data
224 *
225 * TCP_FIN_WAIT2 all buffered data sent, waiting for remote
226 * to shutdown
227 *
228 * TCP_CLOSING both sides have shutdown but we still have
229 * data we have to finish sending
230 *
231 * TCP_TIME_WAIT timeout to catch resent junk before entering
232 * closed, can only be entered from FIN_WAIT2
233 * or CLOSING. Required because the other end
234 * may not have gotten our last ACK causing it
235 * to retransmit the data packet (which we ignore)
236 *
237 * TCP_CLOSE_WAIT remote side has shutdown and is waiting for
238 * us to finish writing our data and to shutdown
239 * (we have to close() to move on to LAST_ACK)
240 *
241 * TCP_LAST_ACK out side has shutdown after remote has
242 * shutdown. There may still be data in our
243 * buffer that we have to finish sending
244 *
245 * TCP_CLOSE socket is finished
246 */
247
248 #define pr_fmt(fmt) "TCP: " fmt
249
250 #include <linux/kernel.h>
251 #include <linux/module.h>
252 #include <linux/types.h>
253 #include <linux/fcntl.h>
254 #include <linux/poll.h>
255 #include <linux/inet_diag.h>
256 #include <linux/init.h>
257 #include <linux/fs.h>
258 #include <linux/skbuff.h>
259 #include <linux/scatterlist.h>
260 #include <linux/splice.h>
261 #include <linux/net.h>
262 #include <linux/socket.h>
263 #include <linux/random.h>
264 #include <linux/bootmem.h>
265 #include <linux/highmem.h>
266 #include <linux/swap.h>
267 #include <linux/cache.h>
268 #include <linux/err.h>
269 #include <linux/crypto.h>
270 #include <linux/time.h>
271 #include <linux/slab.h>
272
273 #include <net/icmp.h>
274 #include <net/inet_common.h>
275 #include <net/tcp.h>
276 #include <net/xfrm.h>
277 #include <net/ip.h>
278 #include <net/sock.h>
279
280 #include <asm/uaccess.h>
281 #include <asm/ioctls.h>
282 #include <asm/unaligned.h>
283 #include <net/busy_poll.h>
284
285 int sysctl_tcp_fin_timeout __read_mostly = TCP_FIN_TIMEOUT;
286
287 int sysctl_tcp_min_tso_segs __read_mostly = 2;
288
289 int sysctl_tcp_autocorking __read_mostly = 1;
290
291 struct percpu_counter tcp_orphan_count;
292 EXPORT_SYMBOL_GPL(tcp_orphan_count);
293
294 long sysctl_tcp_mem[3] __read_mostly;
295 int sysctl_tcp_wmem[3] __read_mostly;
296 int sysctl_tcp_rmem[3] __read_mostly;
297
298 EXPORT_SYMBOL(sysctl_tcp_mem);
299 EXPORT_SYMBOL(sysctl_tcp_rmem);
300 EXPORT_SYMBOL(sysctl_tcp_wmem);
301
302 atomic_long_t tcp_memory_allocated; /* Current allocated memory. */
303 EXPORT_SYMBOL(tcp_memory_allocated);
304
305 /*
306 * Current number of TCP sockets.
307 */
308 struct percpu_counter tcp_sockets_allocated;
309 EXPORT_SYMBOL(tcp_sockets_allocated);
310
311 /*
312 * TCP splice context
313 */
314 struct tcp_splice_state {
315 struct pipe_inode_info *pipe;
316 size_t len;
317 unsigned int flags;
318 };
319
320 /*
321 * Pressure flag: try to collapse.
322 * Technical note: it is used by multiple contexts non atomically.
323 * All the __sk_mem_schedule() is of this nature: accounting
324 * is strict, actions are advisory and have some latency.
325 */
326 int tcp_memory_pressure __read_mostly;
327 EXPORT_SYMBOL(tcp_memory_pressure);
328
tcp_enter_memory_pressure(struct sock * sk)329 void tcp_enter_memory_pressure(struct sock *sk)
330 {
331 if (!tcp_memory_pressure) {
332 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMEMORYPRESSURES);
333 tcp_memory_pressure = 1;
334 }
335 }
336 EXPORT_SYMBOL(tcp_enter_memory_pressure);
337
338 /* Convert seconds to retransmits based on initial and max timeout */
secs_to_retrans(int seconds,int timeout,int rto_max)339 static u8 secs_to_retrans(int seconds, int timeout, int rto_max)
340 {
341 u8 res = 0;
342
343 if (seconds > 0) {
344 int period = timeout;
345
346 res = 1;
347 while (seconds > period && res < 255) {
348 res++;
349 timeout <<= 1;
350 if (timeout > rto_max)
351 timeout = rto_max;
352 period += timeout;
353 }
354 }
355 return res;
356 }
357
358 /* Convert retransmits to seconds based on initial and max timeout */
retrans_to_secs(u8 retrans,int timeout,int rto_max)359 static int retrans_to_secs(u8 retrans, int timeout, int rto_max)
360 {
361 int period = 0;
362
363 if (retrans > 0) {
364 period = timeout;
365 while (--retrans) {
366 timeout <<= 1;
367 if (timeout > rto_max)
368 timeout = rto_max;
369 period += timeout;
370 }
371 }
372 return period;
373 }
374
375 /* Address-family independent initialization for a tcp_sock.
376 *
377 * NOTE: A lot of things set to zero explicitly by call to
378 * sk_alloc() so need not be done here.
379 */
tcp_init_sock(struct sock * sk)380 void tcp_init_sock(struct sock *sk)
381 {
382 struct inet_connection_sock *icsk = inet_csk(sk);
383 struct tcp_sock *tp = tcp_sk(sk);
384
385 tp->out_of_order_queue = RB_ROOT;
386 tcp_init_xmit_timers(sk);
387 tcp_prequeue_init(tp);
388 INIT_LIST_HEAD(&tp->tsq_node);
389
390 icsk->icsk_rto = TCP_TIMEOUT_INIT;
391 tp->mdev_us = jiffies_to_usecs(TCP_TIMEOUT_INIT);
392 tp->rtt_min[0].rtt = ~0U;
393
394 /* So many TCP implementations out there (incorrectly) count the
395 * initial SYN frame in their delayed-ACK and congestion control
396 * algorithms that we must have the following bandaid to talk
397 * efficiently to them. -DaveM
398 */
399 tp->snd_cwnd = TCP_INIT_CWND;
400
401 /* See draft-stevens-tcpca-spec-01 for discussion of the
402 * initialization of these values.
403 */
404 tp->snd_ssthresh = TCP_INFINITE_SSTHRESH;
405 tp->snd_cwnd_clamp = ~0;
406 tp->mss_cache = TCP_MSS_DEFAULT;
407 u64_stats_init(&tp->syncp);
408
409 tp->reordering = sysctl_tcp_reordering;
410 tcp_enable_early_retrans(tp);
411 tcp_assign_congestion_control(sk);
412
413 tp->tsoffset = 0;
414
415 sk->sk_state = TCP_CLOSE;
416
417 sk->sk_write_space = sk_stream_write_space;
418 sock_set_flag(sk, SOCK_USE_WRITE_QUEUE);
419
420 icsk->icsk_sync_mss = tcp_sync_mss;
421
422 sk->sk_sndbuf = sysctl_tcp_wmem[1];
423 sk->sk_rcvbuf = sysctl_tcp_rmem[1];
424
425 local_bh_disable();
426 sock_update_memcg(sk);
427 sk_sockets_allocated_inc(sk);
428 local_bh_enable();
429 }
430 EXPORT_SYMBOL(tcp_init_sock);
431
tcp_tx_timestamp(struct sock * sk,struct sk_buff * skb)432 static void tcp_tx_timestamp(struct sock *sk, struct sk_buff *skb)
433 {
434 if (sk->sk_tsflags) {
435 struct skb_shared_info *shinfo = skb_shinfo(skb);
436
437 sock_tx_timestamp(sk, &shinfo->tx_flags);
438 if (shinfo->tx_flags & SKBTX_ANY_TSTAMP)
439 shinfo->tskey = TCP_SKB_CB(skb)->seq + skb->len - 1;
440 }
441 }
442
443 /*
444 * Wait for a TCP event.
445 *
446 * Note that we don't need to lock the socket, as the upper poll layers
447 * take care of normal races (between the test and the event) and we don't
448 * go look at any of the socket buffers directly.
449 */
tcp_poll(struct file * file,struct socket * sock,poll_table * wait)450 unsigned int tcp_poll(struct file *file, struct socket *sock, poll_table *wait)
451 {
452 unsigned int mask;
453 struct sock *sk = sock->sk;
454 const struct tcp_sock *tp = tcp_sk(sk);
455 int state;
456
457 sock_rps_record_flow(sk);
458
459 sock_poll_wait(file, sk_sleep(sk), wait);
460
461 state = sk_state_load(sk);
462 if (state == TCP_LISTEN)
463 return inet_csk_listen_poll(sk);
464
465 /* Socket is not locked. We are protected from async events
466 * by poll logic and correct handling of state changes
467 * made by other threads is impossible in any case.
468 */
469
470 mask = 0;
471
472 /*
473 * POLLHUP is certainly not done right. But poll() doesn't
474 * have a notion of HUP in just one direction, and for a
475 * socket the read side is more interesting.
476 *
477 * Some poll() documentation says that POLLHUP is incompatible
478 * with the POLLOUT/POLLWR flags, so somebody should check this
479 * all. But careful, it tends to be safer to return too many
480 * bits than too few, and you can easily break real applications
481 * if you don't tell them that something has hung up!
482 *
483 * Check-me.
484 *
485 * Check number 1. POLLHUP is _UNMASKABLE_ event (see UNIX98 and
486 * our fs/select.c). It means that after we received EOF,
487 * poll always returns immediately, making impossible poll() on write()
488 * in state CLOSE_WAIT. One solution is evident --- to set POLLHUP
489 * if and only if shutdown has been made in both directions.
490 * Actually, it is interesting to look how Solaris and DUX
491 * solve this dilemma. I would prefer, if POLLHUP were maskable,
492 * then we could set it on SND_SHUTDOWN. BTW examples given
493 * in Stevens' books assume exactly this behaviour, it explains
494 * why POLLHUP is incompatible with POLLOUT. --ANK
495 *
496 * NOTE. Check for TCP_CLOSE is added. The goal is to prevent
497 * blocking on fresh not-connected or disconnected socket. --ANK
498 */
499 if (sk->sk_shutdown == SHUTDOWN_MASK || state == TCP_CLOSE)
500 mask |= POLLHUP;
501 if (sk->sk_shutdown & RCV_SHUTDOWN)
502 mask |= POLLIN | POLLRDNORM | POLLRDHUP;
503
504 /* Connected or passive Fast Open socket? */
505 if (state != TCP_SYN_SENT &&
506 (state != TCP_SYN_RECV || tp->fastopen_rsk)) {
507 int target = sock_rcvlowat(sk, 0, INT_MAX);
508
509 if (tp->urg_seq == tp->copied_seq &&
510 !sock_flag(sk, SOCK_URGINLINE) &&
511 tp->urg_data)
512 target++;
513
514 if (tp->rcv_nxt - tp->copied_seq >= target)
515 mask |= POLLIN | POLLRDNORM;
516
517 if (!(sk->sk_shutdown & SEND_SHUTDOWN)) {
518 if (sk_stream_is_writeable(sk)) {
519 mask |= POLLOUT | POLLWRNORM;
520 } else { /* send SIGIO later */
521 sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk);
522 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
523
524 /* Race breaker. If space is freed after
525 * wspace test but before the flags are set,
526 * IO signal will be lost. Memory barrier
527 * pairs with the input side.
528 */
529 smp_mb__after_atomic();
530 if (sk_stream_is_writeable(sk))
531 mask |= POLLOUT | POLLWRNORM;
532 }
533 } else
534 mask |= POLLOUT | POLLWRNORM;
535
536 if (tp->urg_data & TCP_URG_VALID)
537 mask |= POLLPRI;
538 }
539 /* This barrier is coupled with smp_wmb() in tcp_reset() */
540 smp_rmb();
541 if (sk->sk_err || !skb_queue_empty(&sk->sk_error_queue))
542 mask |= POLLERR;
543
544 return mask;
545 }
546 EXPORT_SYMBOL(tcp_poll);
547
tcp_ioctl(struct sock * sk,int cmd,unsigned long arg)548 int tcp_ioctl(struct sock *sk, int cmd, unsigned long arg)
549 {
550 struct tcp_sock *tp = tcp_sk(sk);
551 int answ;
552 bool slow;
553
554 switch (cmd) {
555 case SIOCINQ:
556 if (sk->sk_state == TCP_LISTEN)
557 return -EINVAL;
558
559 slow = lock_sock_fast(sk);
560 if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV))
561 answ = 0;
562 else if (sock_flag(sk, SOCK_URGINLINE) ||
563 !tp->urg_data ||
564 before(tp->urg_seq, tp->copied_seq) ||
565 !before(tp->urg_seq, tp->rcv_nxt)) {
566
567 answ = tp->rcv_nxt - tp->copied_seq;
568
569 /* Subtract 1, if FIN was received */
570 if (answ && sock_flag(sk, SOCK_DONE))
571 answ--;
572 } else
573 answ = tp->urg_seq - tp->copied_seq;
574 unlock_sock_fast(sk, slow);
575 break;
576 case SIOCATMARK:
577 answ = tp->urg_data && tp->urg_seq == tp->copied_seq;
578 break;
579 case SIOCOUTQ:
580 if (sk->sk_state == TCP_LISTEN)
581 return -EINVAL;
582
583 if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV))
584 answ = 0;
585 else
586 answ = tp->write_seq - tp->snd_una;
587 break;
588 case SIOCOUTQNSD:
589 if (sk->sk_state == TCP_LISTEN)
590 return -EINVAL;
591
592 if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV))
593 answ = 0;
594 else
595 answ = tp->write_seq - tp->snd_nxt;
596 break;
597 default:
598 return -ENOIOCTLCMD;
599 }
600
601 return put_user(answ, (int __user *)arg);
602 }
603 EXPORT_SYMBOL(tcp_ioctl);
604
tcp_mark_push(struct tcp_sock * tp,struct sk_buff * skb)605 static inline void tcp_mark_push(struct tcp_sock *tp, struct sk_buff *skb)
606 {
607 TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_PSH;
608 tp->pushed_seq = tp->write_seq;
609 }
610
forced_push(const struct tcp_sock * tp)611 static inline bool forced_push(const struct tcp_sock *tp)
612 {
613 return after(tp->write_seq, tp->pushed_seq + (tp->max_window >> 1));
614 }
615
skb_entail(struct sock * sk,struct sk_buff * skb)616 static void skb_entail(struct sock *sk, struct sk_buff *skb)
617 {
618 struct tcp_sock *tp = tcp_sk(sk);
619 struct tcp_skb_cb *tcb = TCP_SKB_CB(skb);
620
621 skb->csum = 0;
622 tcb->seq = tcb->end_seq = tp->write_seq;
623 tcb->tcp_flags = TCPHDR_ACK;
624 tcb->sacked = 0;
625 __skb_header_release(skb);
626 tcp_add_write_queue_tail(sk, skb);
627 sk->sk_wmem_queued += skb->truesize;
628 sk_mem_charge(sk, skb->truesize);
629 if (tp->nonagle & TCP_NAGLE_PUSH)
630 tp->nonagle &= ~TCP_NAGLE_PUSH;
631
632 tcp_slow_start_after_idle_check(sk);
633 }
634
tcp_mark_urg(struct tcp_sock * tp,int flags)635 static inline void tcp_mark_urg(struct tcp_sock *tp, int flags)
636 {
637 if (flags & MSG_OOB)
638 tp->snd_up = tp->write_seq;
639 }
640
641 /* If a not yet filled skb is pushed, do not send it if
642 * we have data packets in Qdisc or NIC queues :
643 * Because TX completion will happen shortly, it gives a chance
644 * to coalesce future sendmsg() payload into this skb, without
645 * need for a timer, and with no latency trade off.
646 * As packets containing data payload have a bigger truesize
647 * than pure acks (dataless) packets, the last checks prevent
648 * autocorking if we only have an ACK in Qdisc/NIC queues,
649 * or if TX completion was delayed after we processed ACK packet.
650 */
tcp_should_autocork(struct sock * sk,struct sk_buff * skb,int size_goal)651 static bool tcp_should_autocork(struct sock *sk, struct sk_buff *skb,
652 int size_goal)
653 {
654 return skb->len < size_goal &&
655 sysctl_tcp_autocorking &&
656 skb != tcp_write_queue_head(sk) &&
657 atomic_read(&sk->sk_wmem_alloc) > skb->truesize;
658 }
659
tcp_push(struct sock * sk,int flags,int mss_now,int nonagle,int size_goal)660 static void tcp_push(struct sock *sk, int flags, int mss_now,
661 int nonagle, int size_goal)
662 {
663 struct tcp_sock *tp = tcp_sk(sk);
664 struct sk_buff *skb;
665
666 if (!tcp_send_head(sk))
667 return;
668
669 skb = tcp_write_queue_tail(sk);
670 if (!(flags & MSG_MORE) || forced_push(tp))
671 tcp_mark_push(tp, skb);
672
673 tcp_mark_urg(tp, flags);
674
675 if (tcp_should_autocork(sk, skb, size_goal)) {
676
677 /* avoid atomic op if TSQ_THROTTLED bit is already set */
678 if (!test_bit(TSQ_THROTTLED, &tp->tsq_flags)) {
679 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPAUTOCORKING);
680 set_bit(TSQ_THROTTLED, &tp->tsq_flags);
681 }
682 /* It is possible TX completion already happened
683 * before we set TSQ_THROTTLED.
684 */
685 if (atomic_read(&sk->sk_wmem_alloc) > skb->truesize)
686 return;
687 }
688
689 if (flags & MSG_MORE)
690 nonagle = TCP_NAGLE_CORK;
691
692 __tcp_push_pending_frames(sk, mss_now, nonagle);
693 }
694
tcp_splice_data_recv(read_descriptor_t * rd_desc,struct sk_buff * skb,unsigned int offset,size_t len)695 static int tcp_splice_data_recv(read_descriptor_t *rd_desc, struct sk_buff *skb,
696 unsigned int offset, size_t len)
697 {
698 struct tcp_splice_state *tss = rd_desc->arg.data;
699 int ret;
700
701 ret = skb_splice_bits(skb, skb->sk, offset, tss->pipe,
702 min(rd_desc->count, len), tss->flags,
703 skb_socket_splice);
704 if (ret > 0)
705 rd_desc->count -= ret;
706 return ret;
707 }
708
__tcp_splice_read(struct sock * sk,struct tcp_splice_state * tss)709 static int __tcp_splice_read(struct sock *sk, struct tcp_splice_state *tss)
710 {
711 /* Store TCP splice context information in read_descriptor_t. */
712 read_descriptor_t rd_desc = {
713 .arg.data = tss,
714 .count = tss->len,
715 };
716
717 return tcp_read_sock(sk, &rd_desc, tcp_splice_data_recv);
718 }
719
720 /**
721 * tcp_splice_read - splice data from TCP socket to a pipe
722 * @sock: socket to splice from
723 * @ppos: position (not valid)
724 * @pipe: pipe to splice to
725 * @len: number of bytes to splice
726 * @flags: splice modifier flags
727 *
728 * Description:
729 * Will read pages from given socket and fill them into a pipe.
730 *
731 **/
tcp_splice_read(struct socket * sock,loff_t * ppos,struct pipe_inode_info * pipe,size_t len,unsigned int flags)732 ssize_t tcp_splice_read(struct socket *sock, loff_t *ppos,
733 struct pipe_inode_info *pipe, size_t len,
734 unsigned int flags)
735 {
736 struct sock *sk = sock->sk;
737 struct tcp_splice_state tss = {
738 .pipe = pipe,
739 .len = len,
740 .flags = flags,
741 };
742 long timeo;
743 ssize_t spliced;
744 int ret;
745
746 sock_rps_record_flow(sk);
747 /*
748 * We can't seek on a socket input
749 */
750 if (unlikely(*ppos))
751 return -ESPIPE;
752
753 ret = spliced = 0;
754
755 lock_sock(sk);
756
757 timeo = sock_rcvtimeo(sk, sock->file->f_flags & O_NONBLOCK);
758 while (tss.len) {
759 ret = __tcp_splice_read(sk, &tss);
760 if (ret < 0)
761 break;
762 else if (!ret) {
763 if (spliced)
764 break;
765 if (sock_flag(sk, SOCK_DONE))
766 break;
767 if (sk->sk_err) {
768 ret = sock_error(sk);
769 break;
770 }
771 if (sk->sk_shutdown & RCV_SHUTDOWN)
772 break;
773 if (sk->sk_state == TCP_CLOSE) {
774 /*
775 * This occurs when user tries to read
776 * from never connected socket.
777 */
778 if (!sock_flag(sk, SOCK_DONE))
779 ret = -ENOTCONN;
780 break;
781 }
782 if (!timeo) {
783 ret = -EAGAIN;
784 break;
785 }
786 /* if __tcp_splice_read() got nothing while we have
787 * an skb in receive queue, we do not want to loop.
788 * This might happen with URG data.
789 */
790 if (!skb_queue_empty(&sk->sk_receive_queue))
791 break;
792 sk_wait_data(sk, &timeo, NULL);
793 if (signal_pending(current)) {
794 ret = sock_intr_errno(timeo);
795 break;
796 }
797 continue;
798 }
799 tss.len -= ret;
800 spliced += ret;
801
802 if (!timeo)
803 break;
804 release_sock(sk);
805 lock_sock(sk);
806
807 if (sk->sk_err || sk->sk_state == TCP_CLOSE ||
808 (sk->sk_shutdown & RCV_SHUTDOWN) ||
809 signal_pending(current))
810 break;
811 }
812
813 release_sock(sk);
814
815 if (spliced)
816 return spliced;
817
818 return ret;
819 }
820 EXPORT_SYMBOL(tcp_splice_read);
821
sk_stream_alloc_skb(struct sock * sk,int size,gfp_t gfp,bool force_schedule)822 struct sk_buff *sk_stream_alloc_skb(struct sock *sk, int size, gfp_t gfp,
823 bool force_schedule)
824 {
825 struct sk_buff *skb;
826
827 /* The TCP header must be at least 32-bit aligned. */
828 size = ALIGN(size, 4);
829
830 if (unlikely(tcp_under_memory_pressure(sk)))
831 sk_mem_reclaim_partial(sk);
832
833 skb = alloc_skb_fclone(size + sk->sk_prot->max_header, gfp);
834 if (likely(skb)) {
835 bool mem_scheduled;
836
837 if (force_schedule) {
838 mem_scheduled = true;
839 sk_forced_mem_schedule(sk, skb->truesize);
840 } else {
841 mem_scheduled = sk_wmem_schedule(sk, skb->truesize);
842 }
843 if (likely(mem_scheduled)) {
844 skb_reserve(skb, sk->sk_prot->max_header);
845 /*
846 * Make sure that we have exactly size bytes
847 * available to the caller, no more, no less.
848 */
849 skb->reserved_tailroom = skb->end - skb->tail - size;
850 return skb;
851 }
852 __kfree_skb(skb);
853 } else {
854 sk->sk_prot->enter_memory_pressure(sk);
855 sk_stream_moderate_sndbuf(sk);
856 }
857 return NULL;
858 }
859
tcp_xmit_size_goal(struct sock * sk,u32 mss_now,int large_allowed)860 static unsigned int tcp_xmit_size_goal(struct sock *sk, u32 mss_now,
861 int large_allowed)
862 {
863 struct tcp_sock *tp = tcp_sk(sk);
864 u32 new_size_goal, size_goal;
865
866 if (!large_allowed || !sk_can_gso(sk))
867 return mss_now;
868
869 /* Note : tcp_tso_autosize() will eventually split this later */
870 new_size_goal = sk->sk_gso_max_size - 1 - MAX_TCP_HEADER;
871 new_size_goal = tcp_bound_to_half_wnd(tp, new_size_goal);
872
873 /* We try hard to avoid divides here */
874 size_goal = tp->gso_segs * mss_now;
875 if (unlikely(new_size_goal < size_goal ||
876 new_size_goal >= size_goal + mss_now)) {
877 tp->gso_segs = min_t(u16, new_size_goal / mss_now,
878 sk->sk_gso_max_segs);
879 size_goal = tp->gso_segs * mss_now;
880 }
881
882 return max(size_goal, mss_now);
883 }
884
tcp_send_mss(struct sock * sk,int * size_goal,int flags)885 static int tcp_send_mss(struct sock *sk, int *size_goal, int flags)
886 {
887 int mss_now;
888
889 mss_now = tcp_current_mss(sk);
890 *size_goal = tcp_xmit_size_goal(sk, mss_now, !(flags & MSG_OOB));
891
892 return mss_now;
893 }
894
do_tcp_sendpages(struct sock * sk,struct page * page,int offset,size_t size,int flags)895 static ssize_t do_tcp_sendpages(struct sock *sk, struct page *page, int offset,
896 size_t size, int flags)
897 {
898 struct tcp_sock *tp = tcp_sk(sk);
899 int mss_now, size_goal;
900 int err;
901 ssize_t copied;
902 long timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT);
903
904 /* Wait for a connection to finish. One exception is TCP Fast Open
905 * (passive side) where data is allowed to be sent before a connection
906 * is fully established.
907 */
908 if (((1 << sk->sk_state) & ~(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)) &&
909 !tcp_passive_fastopen(sk)) {
910 err = sk_stream_wait_connect(sk, &timeo);
911 if (err != 0)
912 goto out_err;
913 }
914
915 sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk);
916
917 mss_now = tcp_send_mss(sk, &size_goal, flags);
918 copied = 0;
919
920 err = -EPIPE;
921 if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN))
922 goto out_err;
923
924 while (size > 0) {
925 struct sk_buff *skb = tcp_write_queue_tail(sk);
926 int copy, i;
927 bool can_coalesce;
928
929 if (!tcp_send_head(sk) || (copy = size_goal - skb->len) <= 0) {
930 new_segment:
931 if (!sk_stream_memory_free(sk))
932 goto wait_for_sndbuf;
933
934 skb = sk_stream_alloc_skb(sk, 0, sk->sk_allocation,
935 skb_queue_empty(&sk->sk_write_queue));
936 if (!skb)
937 goto wait_for_memory;
938
939 skb_entail(sk, skb);
940 copy = size_goal;
941 }
942
943 if (copy > size)
944 copy = size;
945
946 i = skb_shinfo(skb)->nr_frags;
947 can_coalesce = skb_can_coalesce(skb, i, page, offset);
948 if (!can_coalesce && i >= sysctl_max_skb_frags) {
949 tcp_mark_push(tp, skb);
950 goto new_segment;
951 }
952 if (!sk_wmem_schedule(sk, copy))
953 goto wait_for_memory;
954
955 if (can_coalesce) {
956 skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy);
957 } else {
958 get_page(page);
959 skb_fill_page_desc(skb, i, page, offset, copy);
960 }
961 skb_shinfo(skb)->tx_flags |= SKBTX_SHARED_FRAG;
962
963 skb->len += copy;
964 skb->data_len += copy;
965 skb->truesize += copy;
966 sk->sk_wmem_queued += copy;
967 sk_mem_charge(sk, copy);
968 skb->ip_summed = CHECKSUM_PARTIAL;
969 tp->write_seq += copy;
970 TCP_SKB_CB(skb)->end_seq += copy;
971 tcp_skb_pcount_set(skb, 0);
972
973 if (!copied)
974 TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_PSH;
975
976 copied += copy;
977 offset += copy;
978 size -= copy;
979 if (!size) {
980 tcp_tx_timestamp(sk, skb);
981 goto out;
982 }
983
984 if (skb->len < size_goal || (flags & MSG_OOB))
985 continue;
986
987 if (forced_push(tp)) {
988 tcp_mark_push(tp, skb);
989 __tcp_push_pending_frames(sk, mss_now, TCP_NAGLE_PUSH);
990 } else if (skb == tcp_send_head(sk))
991 tcp_push_one(sk, mss_now);
992 continue;
993
994 wait_for_sndbuf:
995 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
996 wait_for_memory:
997 tcp_push(sk, flags & ~MSG_MORE, mss_now,
998 TCP_NAGLE_PUSH, size_goal);
999
1000 err = sk_stream_wait_memory(sk, &timeo);
1001 if (err != 0)
1002 goto do_error;
1003
1004 mss_now = tcp_send_mss(sk, &size_goal, flags);
1005 }
1006
1007 out:
1008 if (copied && !(flags & MSG_SENDPAGE_NOTLAST))
1009 tcp_push(sk, flags, mss_now, tp->nonagle, size_goal);
1010 return copied;
1011
1012 do_error:
1013 if (copied)
1014 goto out;
1015 out_err:
1016 /* make sure we wake any epoll edge trigger waiter */
1017 if (unlikely(skb_queue_len(&sk->sk_write_queue) == 0 && err == -EAGAIN))
1018 sk->sk_write_space(sk);
1019 return sk_stream_error(sk, flags, err);
1020 }
1021
tcp_sendpage(struct sock * sk,struct page * page,int offset,size_t size,int flags)1022 int tcp_sendpage(struct sock *sk, struct page *page, int offset,
1023 size_t size, int flags)
1024 {
1025 ssize_t res;
1026
1027 if (!(sk->sk_route_caps & NETIF_F_SG) ||
1028 !(sk->sk_route_caps & NETIF_F_ALL_CSUM))
1029 return sock_no_sendpage(sk->sk_socket, page, offset, size,
1030 flags);
1031
1032 lock_sock(sk);
1033 res = do_tcp_sendpages(sk, page, offset, size, flags);
1034 release_sock(sk);
1035 return res;
1036 }
1037 EXPORT_SYMBOL(tcp_sendpage);
1038
select_size(const struct sock * sk,bool sg)1039 static inline int select_size(const struct sock *sk, bool sg)
1040 {
1041 const struct tcp_sock *tp = tcp_sk(sk);
1042 int tmp = tp->mss_cache;
1043
1044 if (sg) {
1045 if (sk_can_gso(sk)) {
1046 /* Small frames wont use a full page:
1047 * Payload will immediately follow tcp header.
1048 */
1049 tmp = SKB_WITH_OVERHEAD(2048 - MAX_TCP_HEADER);
1050 } else {
1051 int pgbreak = SKB_MAX_HEAD(MAX_TCP_HEADER);
1052
1053 if (tmp >= pgbreak &&
1054 tmp <= pgbreak + (MAX_SKB_FRAGS - 1) * PAGE_SIZE)
1055 tmp = pgbreak;
1056 }
1057 }
1058
1059 return tmp;
1060 }
1061
tcp_free_fastopen_req(struct tcp_sock * tp)1062 void tcp_free_fastopen_req(struct tcp_sock *tp)
1063 {
1064 if (tp->fastopen_req) {
1065 kfree(tp->fastopen_req);
1066 tp->fastopen_req = NULL;
1067 }
1068 }
1069
tcp_sendmsg_fastopen(struct sock * sk,struct msghdr * msg,int * copied,size_t size)1070 static int tcp_sendmsg_fastopen(struct sock *sk, struct msghdr *msg,
1071 int *copied, size_t size)
1072 {
1073 struct tcp_sock *tp = tcp_sk(sk);
1074 struct sockaddr *uaddr = msg->msg_name;
1075 int err, flags;
1076
1077 if (!(sysctl_tcp_fastopen & TFO_CLIENT_ENABLE) ||
1078 (uaddr && msg->msg_namelen >= sizeof(uaddr->sa_family) &&
1079 uaddr->sa_family == AF_UNSPEC))
1080 return -EOPNOTSUPP;
1081 if (tp->fastopen_req)
1082 return -EALREADY; /* Another Fast Open is in progress */
1083
1084 tp->fastopen_req = kzalloc(sizeof(struct tcp_fastopen_request),
1085 sk->sk_allocation);
1086 if (unlikely(!tp->fastopen_req))
1087 return -ENOBUFS;
1088 tp->fastopen_req->data = msg;
1089 tp->fastopen_req->size = size;
1090
1091 flags = (msg->msg_flags & MSG_DONTWAIT) ? O_NONBLOCK : 0;
1092 err = __inet_stream_connect(sk->sk_socket, uaddr,
1093 msg->msg_namelen, flags);
1094 *copied = tp->fastopen_req->copied;
1095 tcp_free_fastopen_req(tp);
1096 return err;
1097 }
1098
tcp_sendmsg(struct sock * sk,struct msghdr * msg,size_t size)1099 int tcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)
1100 {
1101 struct tcp_sock *tp = tcp_sk(sk);
1102 struct sk_buff *skb;
1103 int flags, err, copied = 0;
1104 int mss_now = 0, size_goal, copied_syn = 0;
1105 bool sg;
1106 long timeo;
1107
1108 lock_sock(sk);
1109
1110 flags = msg->msg_flags;
1111 if ((flags & MSG_FASTOPEN) && !tp->repair) {
1112 err = tcp_sendmsg_fastopen(sk, msg, &copied_syn, size);
1113 if (err == -EINPROGRESS && copied_syn > 0)
1114 goto out;
1115 else if (err)
1116 goto out_err;
1117 }
1118
1119 timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT);
1120
1121 /* Wait for a connection to finish. One exception is TCP Fast Open
1122 * (passive side) where data is allowed to be sent before a connection
1123 * is fully established.
1124 */
1125 if (((1 << sk->sk_state) & ~(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)) &&
1126 !tcp_passive_fastopen(sk)) {
1127 err = sk_stream_wait_connect(sk, &timeo);
1128 if (err != 0)
1129 goto do_error;
1130 }
1131
1132 if (unlikely(tp->repair)) {
1133 if (tp->repair_queue == TCP_RECV_QUEUE) {
1134 copied = tcp_send_rcvq(sk, msg, size);
1135 goto out_nopush;
1136 }
1137
1138 err = -EINVAL;
1139 if (tp->repair_queue == TCP_NO_QUEUE)
1140 goto out_err;
1141
1142 /* 'common' sending to sendq */
1143 }
1144
1145 /* This should be in poll */
1146 sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk);
1147
1148 mss_now = tcp_send_mss(sk, &size_goal, flags);
1149
1150 /* Ok commence sending. */
1151 copied = 0;
1152
1153 err = -EPIPE;
1154 if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN))
1155 goto out_err;
1156
1157 sg = !!(sk->sk_route_caps & NETIF_F_SG);
1158
1159 while (msg_data_left(msg)) {
1160 int copy = 0;
1161 int max = size_goal;
1162
1163 skb = tcp_write_queue_tail(sk);
1164 if (tcp_send_head(sk)) {
1165 if (skb->ip_summed == CHECKSUM_NONE)
1166 max = mss_now;
1167 copy = max - skb->len;
1168 }
1169
1170 if (copy <= 0) {
1171 new_segment:
1172 /* Allocate new segment. If the interface is SG,
1173 * allocate skb fitting to single page.
1174 */
1175 if (!sk_stream_memory_free(sk))
1176 goto wait_for_sndbuf;
1177
1178 skb = sk_stream_alloc_skb(sk,
1179 select_size(sk, sg),
1180 sk->sk_allocation,
1181 skb_queue_empty(&sk->sk_write_queue));
1182 if (!skb)
1183 goto wait_for_memory;
1184
1185 /*
1186 * Check whether we can use HW checksum.
1187 */
1188 if (sk->sk_route_caps & NETIF_F_ALL_CSUM)
1189 skb->ip_summed = CHECKSUM_PARTIAL;
1190
1191 skb_entail(sk, skb);
1192 copy = size_goal;
1193 max = size_goal;
1194
1195 /* All packets are restored as if they have
1196 * already been sent. skb_mstamp isn't set to
1197 * avoid wrong rtt estimation.
1198 */
1199 if (tp->repair)
1200 TCP_SKB_CB(skb)->sacked |= TCPCB_REPAIRED;
1201 }
1202
1203 /* Try to append data to the end of skb. */
1204 if (copy > msg_data_left(msg))
1205 copy = msg_data_left(msg);
1206
1207 /* Where to copy to? */
1208 if (skb_availroom(skb) > 0) {
1209 /* We have some space in skb head. Superb! */
1210 copy = min_t(int, copy, skb_availroom(skb));
1211 err = skb_add_data_nocache(sk, skb, &msg->msg_iter, copy);
1212 if (err)
1213 goto do_fault;
1214 } else {
1215 bool merge = true;
1216 int i = skb_shinfo(skb)->nr_frags;
1217 struct page_frag *pfrag = sk_page_frag(sk);
1218
1219 if (!sk_page_frag_refill(sk, pfrag))
1220 goto wait_for_memory;
1221
1222 if (!skb_can_coalesce(skb, i, pfrag->page,
1223 pfrag->offset)) {
1224 if (i >= sysctl_max_skb_frags || !sg) {
1225 tcp_mark_push(tp, skb);
1226 goto new_segment;
1227 }
1228 merge = false;
1229 }
1230
1231 copy = min_t(int, copy, pfrag->size - pfrag->offset);
1232
1233 if (!sk_wmem_schedule(sk, copy))
1234 goto wait_for_memory;
1235
1236 err = skb_copy_to_page_nocache(sk, &msg->msg_iter, skb,
1237 pfrag->page,
1238 pfrag->offset,
1239 copy);
1240 if (err)
1241 goto do_error;
1242
1243 /* Update the skb. */
1244 if (merge) {
1245 skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy);
1246 } else {
1247 skb_fill_page_desc(skb, i, pfrag->page,
1248 pfrag->offset, copy);
1249 get_page(pfrag->page);
1250 }
1251 pfrag->offset += copy;
1252 }
1253
1254 if (!copied)
1255 TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_PSH;
1256
1257 tp->write_seq += copy;
1258 TCP_SKB_CB(skb)->end_seq += copy;
1259 tcp_skb_pcount_set(skb, 0);
1260
1261 copied += copy;
1262 if (!msg_data_left(msg)) {
1263 tcp_tx_timestamp(sk, skb);
1264 goto out;
1265 }
1266
1267 if (skb->len < max || (flags & MSG_OOB) || unlikely(tp->repair))
1268 continue;
1269
1270 if (forced_push(tp)) {
1271 tcp_mark_push(tp, skb);
1272 __tcp_push_pending_frames(sk, mss_now, TCP_NAGLE_PUSH);
1273 } else if (skb == tcp_send_head(sk))
1274 tcp_push_one(sk, mss_now);
1275 continue;
1276
1277 wait_for_sndbuf:
1278 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
1279 wait_for_memory:
1280 if (copied)
1281 tcp_push(sk, flags & ~MSG_MORE, mss_now,
1282 TCP_NAGLE_PUSH, size_goal);
1283
1284 err = sk_stream_wait_memory(sk, &timeo);
1285 if (err != 0)
1286 goto do_error;
1287
1288 mss_now = tcp_send_mss(sk, &size_goal, flags);
1289 }
1290
1291 out:
1292 if (copied)
1293 tcp_push(sk, flags, mss_now, tp->nonagle, size_goal);
1294 out_nopush:
1295 release_sock(sk);
1296 return copied + copied_syn;
1297
1298 do_fault:
1299 if (!skb->len) {
1300 tcp_unlink_write_queue(skb, sk);
1301 /* It is the one place in all of TCP, except connection
1302 * reset, where we can be unlinking the send_head.
1303 */
1304 tcp_check_send_head(sk, skb);
1305 sk_wmem_free_skb(sk, skb);
1306 }
1307
1308 do_error:
1309 if (copied + copied_syn)
1310 goto out;
1311 out_err:
1312 err = sk_stream_error(sk, flags, err);
1313 /* make sure we wake any epoll edge trigger waiter */
1314 if (unlikely(skb_queue_len(&sk->sk_write_queue) == 0 && err == -EAGAIN))
1315 sk->sk_write_space(sk);
1316 release_sock(sk);
1317 return err;
1318 }
1319 EXPORT_SYMBOL(tcp_sendmsg);
1320
1321 /*
1322 * Handle reading urgent data. BSD has very simple semantics for
1323 * this, no blocking and very strange errors 8)
1324 */
1325
tcp_recv_urg(struct sock * sk,struct msghdr * msg,int len,int flags)1326 static int tcp_recv_urg(struct sock *sk, struct msghdr *msg, int len, int flags)
1327 {
1328 struct tcp_sock *tp = tcp_sk(sk);
1329
1330 /* No URG data to read. */
1331 if (sock_flag(sk, SOCK_URGINLINE) || !tp->urg_data ||
1332 tp->urg_data == TCP_URG_READ)
1333 return -EINVAL; /* Yes this is right ! */
1334
1335 if (sk->sk_state == TCP_CLOSE && !sock_flag(sk, SOCK_DONE))
1336 return -ENOTCONN;
1337
1338 if (tp->urg_data & TCP_URG_VALID) {
1339 int err = 0;
1340 char c = tp->urg_data;
1341
1342 if (!(flags & MSG_PEEK))
1343 tp->urg_data = TCP_URG_READ;
1344
1345 /* Read urgent data. */
1346 msg->msg_flags |= MSG_OOB;
1347
1348 if (len > 0) {
1349 if (!(flags & MSG_TRUNC))
1350 err = memcpy_to_msg(msg, &c, 1);
1351 len = 1;
1352 } else
1353 msg->msg_flags |= MSG_TRUNC;
1354
1355 return err ? -EFAULT : len;
1356 }
1357
1358 if (sk->sk_state == TCP_CLOSE || (sk->sk_shutdown & RCV_SHUTDOWN))
1359 return 0;
1360
1361 /* Fixed the recv(..., MSG_OOB) behaviour. BSD docs and
1362 * the available implementations agree in this case:
1363 * this call should never block, independent of the
1364 * blocking state of the socket.
1365 * Mike <pall@rz.uni-karlsruhe.de>
1366 */
1367 return -EAGAIN;
1368 }
1369
tcp_peek_sndq(struct sock * sk,struct msghdr * msg,int len)1370 static int tcp_peek_sndq(struct sock *sk, struct msghdr *msg, int len)
1371 {
1372 struct sk_buff *skb;
1373 int copied = 0, err = 0;
1374
1375 /* XXX -- need to support SO_PEEK_OFF */
1376
1377 skb_queue_walk(&sk->sk_write_queue, skb) {
1378 err = skb_copy_datagram_msg(skb, 0, msg, skb->len);
1379 if (err)
1380 break;
1381
1382 copied += skb->len;
1383 }
1384
1385 return err ?: copied;
1386 }
1387
1388 /* Clean up the receive buffer for full frames taken by the user,
1389 * then send an ACK if necessary. COPIED is the number of bytes
1390 * tcp_recvmsg has given to the user so far, it speeds up the
1391 * calculation of whether or not we must ACK for the sake of
1392 * a window update.
1393 */
tcp_cleanup_rbuf(struct sock * sk,int copied)1394 static void tcp_cleanup_rbuf(struct sock *sk, int copied)
1395 {
1396 struct tcp_sock *tp = tcp_sk(sk);
1397 bool time_to_ack = false;
1398
1399 struct sk_buff *skb = skb_peek(&sk->sk_receive_queue);
1400
1401 WARN(skb && !before(tp->copied_seq, TCP_SKB_CB(skb)->end_seq),
1402 "cleanup rbuf bug: copied %X seq %X rcvnxt %X\n",
1403 tp->copied_seq, TCP_SKB_CB(skb)->end_seq, tp->rcv_nxt);
1404
1405 if (inet_csk_ack_scheduled(sk)) {
1406 const struct inet_connection_sock *icsk = inet_csk(sk);
1407 /* Delayed ACKs frequently hit locked sockets during bulk
1408 * receive. */
1409 if (icsk->icsk_ack.blocked ||
1410 /* Once-per-two-segments ACK was not sent by tcp_input.c */
1411 tp->rcv_nxt - tp->rcv_wup > icsk->icsk_ack.rcv_mss ||
1412 /*
1413 * If this read emptied read buffer, we send ACK, if
1414 * connection is not bidirectional, user drained
1415 * receive buffer and there was a small segment
1416 * in queue.
1417 */
1418 (copied > 0 &&
1419 ((icsk->icsk_ack.pending & ICSK_ACK_PUSHED2) ||
1420 ((icsk->icsk_ack.pending & ICSK_ACK_PUSHED) &&
1421 !icsk->icsk_ack.pingpong)) &&
1422 !atomic_read(&sk->sk_rmem_alloc)))
1423 time_to_ack = true;
1424 }
1425
1426 /* We send an ACK if we can now advertise a non-zero window
1427 * which has been raised "significantly".
1428 *
1429 * Even if window raised up to infinity, do not send window open ACK
1430 * in states, where we will not receive more. It is useless.
1431 */
1432 if (copied > 0 && !time_to_ack && !(sk->sk_shutdown & RCV_SHUTDOWN)) {
1433 __u32 rcv_window_now = tcp_receive_window(tp);
1434
1435 /* Optimize, __tcp_select_window() is not cheap. */
1436 if (2*rcv_window_now <= tp->window_clamp) {
1437 __u32 new_window = __tcp_select_window(sk);
1438
1439 /* Send ACK now, if this read freed lots of space
1440 * in our buffer. Certainly, new_window is new window.
1441 * We can advertise it now, if it is not less than current one.
1442 * "Lots" means "at least twice" here.
1443 */
1444 if (new_window && new_window >= 2 * rcv_window_now)
1445 time_to_ack = true;
1446 }
1447 }
1448 if (time_to_ack)
1449 tcp_send_ack(sk);
1450 }
1451
tcp_prequeue_process(struct sock * sk)1452 static void tcp_prequeue_process(struct sock *sk)
1453 {
1454 struct sk_buff *skb;
1455 struct tcp_sock *tp = tcp_sk(sk);
1456
1457 NET_INC_STATS_USER(sock_net(sk), LINUX_MIB_TCPPREQUEUED);
1458
1459 /* RX process wants to run with disabled BHs, though it is not
1460 * necessary */
1461 local_bh_disable();
1462 while ((skb = __skb_dequeue(&tp->ucopy.prequeue)) != NULL)
1463 sk_backlog_rcv(sk, skb);
1464 local_bh_enable();
1465
1466 /* Clear memory counter. */
1467 tp->ucopy.memory = 0;
1468 }
1469
tcp_recv_skb(struct sock * sk,u32 seq,u32 * off)1470 static struct sk_buff *tcp_recv_skb(struct sock *sk, u32 seq, u32 *off)
1471 {
1472 struct sk_buff *skb;
1473 u32 offset;
1474
1475 while ((skb = skb_peek(&sk->sk_receive_queue)) != NULL) {
1476 offset = seq - TCP_SKB_CB(skb)->seq;
1477 if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)
1478 offset--;
1479 if (offset < skb->len || (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)) {
1480 *off = offset;
1481 return skb;
1482 }
1483 /* This looks weird, but this can happen if TCP collapsing
1484 * splitted a fat GRO packet, while we released socket lock
1485 * in skb_splice_bits()
1486 */
1487 sk_eat_skb(sk, skb);
1488 }
1489 return NULL;
1490 }
1491
1492 /*
1493 * This routine provides an alternative to tcp_recvmsg() for routines
1494 * that would like to handle copying from skbuffs directly in 'sendfile'
1495 * fashion.
1496 * Note:
1497 * - It is assumed that the socket was locked by the caller.
1498 * - The routine does not block.
1499 * - At present, there is no support for reading OOB data
1500 * or for 'peeking' the socket using this routine
1501 * (although both would be easy to implement).
1502 */
tcp_read_sock(struct sock * sk,read_descriptor_t * desc,sk_read_actor_t recv_actor)1503 int tcp_read_sock(struct sock *sk, read_descriptor_t *desc,
1504 sk_read_actor_t recv_actor)
1505 {
1506 struct sk_buff *skb;
1507 struct tcp_sock *tp = tcp_sk(sk);
1508 u32 seq = tp->copied_seq;
1509 u32 offset;
1510 int copied = 0;
1511
1512 if (sk->sk_state == TCP_LISTEN)
1513 return -ENOTCONN;
1514 while ((skb = tcp_recv_skb(sk, seq, &offset)) != NULL) {
1515 if (offset < skb->len) {
1516 int used;
1517 size_t len;
1518
1519 len = skb->len - offset;
1520 /* Stop reading if we hit a patch of urgent data */
1521 if (tp->urg_data) {
1522 u32 urg_offset = tp->urg_seq - seq;
1523 if (urg_offset < len)
1524 len = urg_offset;
1525 if (!len)
1526 break;
1527 }
1528 used = recv_actor(desc, skb, offset, len);
1529 if (used <= 0) {
1530 if (!copied)
1531 copied = used;
1532 break;
1533 } else if (used <= len) {
1534 seq += used;
1535 copied += used;
1536 offset += used;
1537 }
1538 /* If recv_actor drops the lock (e.g. TCP splice
1539 * receive) the skb pointer might be invalid when
1540 * getting here: tcp_collapse might have deleted it
1541 * while aggregating skbs from the socket queue.
1542 */
1543 skb = tcp_recv_skb(sk, seq - 1, &offset);
1544 if (!skb)
1545 break;
1546 /* TCP coalescing might have appended data to the skb.
1547 * Try to splice more frags
1548 */
1549 if (offset + 1 != skb->len)
1550 continue;
1551 }
1552 if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) {
1553 sk_eat_skb(sk, skb);
1554 ++seq;
1555 break;
1556 }
1557 sk_eat_skb(sk, skb);
1558 if (!desc->count)
1559 break;
1560 tp->copied_seq = seq;
1561 }
1562 tp->copied_seq = seq;
1563
1564 tcp_rcv_space_adjust(sk);
1565
1566 /* Clean up data we have read: This will do ACK frames. */
1567 if (copied > 0) {
1568 tcp_recv_skb(sk, seq, &offset);
1569 tcp_cleanup_rbuf(sk, copied);
1570 }
1571 return copied;
1572 }
1573 EXPORT_SYMBOL(tcp_read_sock);
1574
1575 /*
1576 * This routine copies from a sock struct into the user buffer.
1577 *
1578 * Technical note: in 2.3 we work on _locked_ socket, so that
1579 * tricks with *seq access order and skb->users are not required.
1580 * Probably, code can be easily improved even more.
1581 */
1582
tcp_recvmsg(struct sock * sk,struct msghdr * msg,size_t len,int nonblock,int flags,int * addr_len)1583 int tcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int nonblock,
1584 int flags, int *addr_len)
1585 {
1586 struct tcp_sock *tp = tcp_sk(sk);
1587 int copied = 0;
1588 u32 peek_seq;
1589 u32 *seq;
1590 unsigned long used;
1591 int err;
1592 int target; /* Read at least this many bytes */
1593 long timeo;
1594 struct task_struct *user_recv = NULL;
1595 struct sk_buff *skb, *last;
1596 u32 urg_hole = 0;
1597
1598 if (unlikely(flags & MSG_ERRQUEUE))
1599 return inet_recv_error(sk, msg, len, addr_len);
1600
1601 if (sk_can_busy_loop(sk) && skb_queue_empty(&sk->sk_receive_queue) &&
1602 (sk->sk_state == TCP_ESTABLISHED))
1603 sk_busy_loop(sk, nonblock);
1604
1605 lock_sock(sk);
1606
1607 err = -ENOTCONN;
1608 if (sk->sk_state == TCP_LISTEN)
1609 goto out;
1610
1611 timeo = sock_rcvtimeo(sk, nonblock);
1612
1613 /* Urgent data needs to be handled specially. */
1614 if (flags & MSG_OOB)
1615 goto recv_urg;
1616
1617 if (unlikely(tp->repair)) {
1618 err = -EPERM;
1619 if (!(flags & MSG_PEEK))
1620 goto out;
1621
1622 if (tp->repair_queue == TCP_SEND_QUEUE)
1623 goto recv_sndq;
1624
1625 err = -EINVAL;
1626 if (tp->repair_queue == TCP_NO_QUEUE)
1627 goto out;
1628
1629 /* 'common' recv queue MSG_PEEK-ing */
1630 }
1631
1632 seq = &tp->copied_seq;
1633 if (flags & MSG_PEEK) {
1634 peek_seq = tp->copied_seq;
1635 seq = &peek_seq;
1636 }
1637
1638 target = sock_rcvlowat(sk, flags & MSG_WAITALL, len);
1639
1640 do {
1641 u32 offset;
1642
1643 /* Are we at urgent data? Stop if we have read anything or have SIGURG pending. */
1644 if (tp->urg_data && tp->urg_seq == *seq) {
1645 if (copied)
1646 break;
1647 if (signal_pending(current)) {
1648 copied = timeo ? sock_intr_errno(timeo) : -EAGAIN;
1649 break;
1650 }
1651 }
1652
1653 /* Next get a buffer. */
1654
1655 last = skb_peek_tail(&sk->sk_receive_queue);
1656 skb_queue_walk(&sk->sk_receive_queue, skb) {
1657 last = skb;
1658 /* Now that we have two receive queues this
1659 * shouldn't happen.
1660 */
1661 if (WARN(before(*seq, TCP_SKB_CB(skb)->seq),
1662 "TCP recvmsg seq # bug: copied %X, seq %X, rcvnxt %X, fl %X\n",
1663 *seq, TCP_SKB_CB(skb)->seq, tp->rcv_nxt,
1664 flags))
1665 break;
1666
1667 offset = *seq - TCP_SKB_CB(skb)->seq;
1668 if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)
1669 offset--;
1670 if (offset < skb->len)
1671 goto found_ok_skb;
1672 if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)
1673 goto found_fin_ok;
1674 WARN(!(flags & MSG_PEEK),
1675 "TCP recvmsg seq # bug 2: copied %X, seq %X, rcvnxt %X, fl %X\n",
1676 *seq, TCP_SKB_CB(skb)->seq, tp->rcv_nxt, flags);
1677 }
1678
1679 /* Well, if we have backlog, try to process it now yet. */
1680
1681 if (copied >= target && !sk->sk_backlog.tail)
1682 break;
1683
1684 if (copied) {
1685 if (sk->sk_err ||
1686 sk->sk_state == TCP_CLOSE ||
1687 (sk->sk_shutdown & RCV_SHUTDOWN) ||
1688 !timeo ||
1689 signal_pending(current))
1690 break;
1691 } else {
1692 if (sock_flag(sk, SOCK_DONE))
1693 break;
1694
1695 if (sk->sk_err) {
1696 copied = sock_error(sk);
1697 break;
1698 }
1699
1700 if (sk->sk_shutdown & RCV_SHUTDOWN)
1701 break;
1702
1703 if (sk->sk_state == TCP_CLOSE) {
1704 if (!sock_flag(sk, SOCK_DONE)) {
1705 /* This occurs when user tries to read
1706 * from never connected socket.
1707 */
1708 copied = -ENOTCONN;
1709 break;
1710 }
1711 break;
1712 }
1713
1714 if (!timeo) {
1715 copied = -EAGAIN;
1716 break;
1717 }
1718
1719 if (signal_pending(current)) {
1720 copied = sock_intr_errno(timeo);
1721 break;
1722 }
1723 }
1724
1725 tcp_cleanup_rbuf(sk, copied);
1726
1727 if (!sysctl_tcp_low_latency && tp->ucopy.task == user_recv) {
1728 /* Install new reader */
1729 if (!user_recv && !(flags & (MSG_TRUNC | MSG_PEEK))) {
1730 user_recv = current;
1731 tp->ucopy.task = user_recv;
1732 tp->ucopy.msg = msg;
1733 }
1734
1735 tp->ucopy.len = len;
1736
1737 WARN_ON(tp->copied_seq != tp->rcv_nxt &&
1738 !(flags & (MSG_PEEK | MSG_TRUNC)));
1739
1740 /* Ugly... If prequeue is not empty, we have to
1741 * process it before releasing socket, otherwise
1742 * order will be broken at second iteration.
1743 * More elegant solution is required!!!
1744 *
1745 * Look: we have the following (pseudo)queues:
1746 *
1747 * 1. packets in flight
1748 * 2. backlog
1749 * 3. prequeue
1750 * 4. receive_queue
1751 *
1752 * Each queue can be processed only if the next ones
1753 * are empty. At this point we have empty receive_queue.
1754 * But prequeue _can_ be not empty after 2nd iteration,
1755 * when we jumped to start of loop because backlog
1756 * processing added something to receive_queue.
1757 * We cannot release_sock(), because backlog contains
1758 * packets arrived _after_ prequeued ones.
1759 *
1760 * Shortly, algorithm is clear --- to process all
1761 * the queues in order. We could make it more directly,
1762 * requeueing packets from backlog to prequeue, if
1763 * is not empty. It is more elegant, but eats cycles,
1764 * unfortunately.
1765 */
1766 if (!skb_queue_empty(&tp->ucopy.prequeue))
1767 goto do_prequeue;
1768
1769 /* __ Set realtime policy in scheduler __ */
1770 }
1771
1772 if (copied >= target) {
1773 /* Do not sleep, just process backlog. */
1774 release_sock(sk);
1775 lock_sock(sk);
1776 } else {
1777 sk_wait_data(sk, &timeo, last);
1778 }
1779
1780 if (user_recv) {
1781 int chunk;
1782
1783 /* __ Restore normal policy in scheduler __ */
1784
1785 chunk = len - tp->ucopy.len;
1786 if (chunk != 0) {
1787 NET_ADD_STATS_USER(sock_net(sk), LINUX_MIB_TCPDIRECTCOPYFROMBACKLOG, chunk);
1788 len -= chunk;
1789 copied += chunk;
1790 }
1791
1792 if (tp->rcv_nxt == tp->copied_seq &&
1793 !skb_queue_empty(&tp->ucopy.prequeue)) {
1794 do_prequeue:
1795 tcp_prequeue_process(sk);
1796
1797 chunk = len - tp->ucopy.len;
1798 if (chunk != 0) {
1799 NET_ADD_STATS_USER(sock_net(sk), LINUX_MIB_TCPDIRECTCOPYFROMPREQUEUE, chunk);
1800 len -= chunk;
1801 copied += chunk;
1802 }
1803 }
1804 }
1805 if ((flags & MSG_PEEK) &&
1806 (peek_seq - copied - urg_hole != tp->copied_seq)) {
1807 net_dbg_ratelimited("TCP(%s:%d): Application bug, race in MSG_PEEK\n",
1808 current->comm,
1809 task_pid_nr(current));
1810 peek_seq = tp->copied_seq;
1811 }
1812 continue;
1813
1814 found_ok_skb:
1815 /* Ok so how much can we use? */
1816 used = skb->len - offset;
1817 if (len < used)
1818 used = len;
1819
1820 /* Do we have urgent data here? */
1821 if (tp->urg_data) {
1822 u32 urg_offset = tp->urg_seq - *seq;
1823 if (urg_offset < used) {
1824 if (!urg_offset) {
1825 if (!sock_flag(sk, SOCK_URGINLINE)) {
1826 ++*seq;
1827 urg_hole++;
1828 offset++;
1829 used--;
1830 if (!used)
1831 goto skip_copy;
1832 }
1833 } else
1834 used = urg_offset;
1835 }
1836 }
1837
1838 if (!(flags & MSG_TRUNC)) {
1839 err = skb_copy_datagram_msg(skb, offset, msg, used);
1840 if (err) {
1841 /* Exception. Bailout! */
1842 if (!copied)
1843 copied = -EFAULT;
1844 break;
1845 }
1846 }
1847
1848 *seq += used;
1849 copied += used;
1850 len -= used;
1851
1852 tcp_rcv_space_adjust(sk);
1853
1854 skip_copy:
1855 if (tp->urg_data && after(tp->copied_seq, tp->urg_seq)) {
1856 tp->urg_data = 0;
1857 tcp_fast_path_check(sk);
1858 }
1859 if (used + offset < skb->len)
1860 continue;
1861
1862 if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)
1863 goto found_fin_ok;
1864 if (!(flags & MSG_PEEK))
1865 sk_eat_skb(sk, skb);
1866 continue;
1867
1868 found_fin_ok:
1869 /* Process the FIN. */
1870 ++*seq;
1871 if (!(flags & MSG_PEEK))
1872 sk_eat_skb(sk, skb);
1873 break;
1874 } while (len > 0);
1875
1876 if (user_recv) {
1877 if (!skb_queue_empty(&tp->ucopy.prequeue)) {
1878 int chunk;
1879
1880 tp->ucopy.len = copied > 0 ? len : 0;
1881
1882 tcp_prequeue_process(sk);
1883
1884 if (copied > 0 && (chunk = len - tp->ucopy.len) != 0) {
1885 NET_ADD_STATS_USER(sock_net(sk), LINUX_MIB_TCPDIRECTCOPYFROMPREQUEUE, chunk);
1886 len -= chunk;
1887 copied += chunk;
1888 }
1889 }
1890
1891 tp->ucopy.task = NULL;
1892 tp->ucopy.len = 0;
1893 }
1894
1895 /* According to UNIX98, msg_name/msg_namelen are ignored
1896 * on connected socket. I was just happy when found this 8) --ANK
1897 */
1898
1899 /* Clean up data we have read: This will do ACK frames. */
1900 tcp_cleanup_rbuf(sk, copied);
1901
1902 release_sock(sk);
1903 return copied;
1904
1905 out:
1906 release_sock(sk);
1907 return err;
1908
1909 recv_urg:
1910 err = tcp_recv_urg(sk, msg, len, flags);
1911 goto out;
1912
1913 recv_sndq:
1914 err = tcp_peek_sndq(sk, msg, len);
1915 goto out;
1916 }
1917 EXPORT_SYMBOL(tcp_recvmsg);
1918
tcp_set_state(struct sock * sk,int state)1919 void tcp_set_state(struct sock *sk, int state)
1920 {
1921 int oldstate = sk->sk_state;
1922
1923 switch (state) {
1924 case TCP_ESTABLISHED:
1925 if (oldstate != TCP_ESTABLISHED)
1926 TCP_INC_STATS(sock_net(sk), TCP_MIB_CURRESTAB);
1927 break;
1928
1929 case TCP_CLOSE:
1930 if (oldstate == TCP_CLOSE_WAIT || oldstate == TCP_ESTABLISHED)
1931 TCP_INC_STATS(sock_net(sk), TCP_MIB_ESTABRESETS);
1932
1933 sk->sk_prot->unhash(sk);
1934 if (inet_csk(sk)->icsk_bind_hash &&
1935 !(sk->sk_userlocks & SOCK_BINDPORT_LOCK))
1936 inet_put_port(sk);
1937 /* fall through */
1938 default:
1939 if (oldstate == TCP_ESTABLISHED)
1940 TCP_DEC_STATS(sock_net(sk), TCP_MIB_CURRESTAB);
1941 }
1942
1943 /* Change state AFTER socket is unhashed to avoid closed
1944 * socket sitting in hash tables.
1945 */
1946 sk_state_store(sk, state);
1947
1948 #ifdef STATE_TRACE
1949 SOCK_DEBUG(sk, "TCP sk=%p, State %s -> %s\n", sk, statename[oldstate], statename[state]);
1950 #endif
1951 }
1952 EXPORT_SYMBOL_GPL(tcp_set_state);
1953
1954 /*
1955 * State processing on a close. This implements the state shift for
1956 * sending our FIN frame. Note that we only send a FIN for some
1957 * states. A shutdown() may have already sent the FIN, or we may be
1958 * closed.
1959 */
1960
1961 static const unsigned char new_state[16] = {
1962 /* current state: new state: action: */
1963 [0 /* (Invalid) */] = TCP_CLOSE,
1964 [TCP_ESTABLISHED] = TCP_FIN_WAIT1 | TCP_ACTION_FIN,
1965 [TCP_SYN_SENT] = TCP_CLOSE,
1966 [TCP_SYN_RECV] = TCP_FIN_WAIT1 | TCP_ACTION_FIN,
1967 [TCP_FIN_WAIT1] = TCP_FIN_WAIT1,
1968 [TCP_FIN_WAIT2] = TCP_FIN_WAIT2,
1969 [TCP_TIME_WAIT] = TCP_CLOSE,
1970 [TCP_CLOSE] = TCP_CLOSE,
1971 [TCP_CLOSE_WAIT] = TCP_LAST_ACK | TCP_ACTION_FIN,
1972 [TCP_LAST_ACK] = TCP_LAST_ACK,
1973 [TCP_LISTEN] = TCP_CLOSE,
1974 [TCP_CLOSING] = TCP_CLOSING,
1975 [TCP_NEW_SYN_RECV] = TCP_CLOSE, /* should not happen ! */
1976 };
1977
tcp_close_state(struct sock * sk)1978 static int tcp_close_state(struct sock *sk)
1979 {
1980 int next = (int)new_state[sk->sk_state];
1981 int ns = next & TCP_STATE_MASK;
1982
1983 tcp_set_state(sk, ns);
1984
1985 return next & TCP_ACTION_FIN;
1986 }
1987
1988 /*
1989 * Shutdown the sending side of a connection. Much like close except
1990 * that we don't receive shut down or sock_set_flag(sk, SOCK_DEAD).
1991 */
1992
tcp_shutdown(struct sock * sk,int how)1993 void tcp_shutdown(struct sock *sk, int how)
1994 {
1995 /* We need to grab some memory, and put together a FIN,
1996 * and then put it into the queue to be sent.
1997 * Tim MacKenzie(tym@dibbler.cs.monash.edu.au) 4 Dec '92.
1998 */
1999 if (!(how & SEND_SHUTDOWN))
2000 return;
2001
2002 /* If we've already sent a FIN, or it's a closed state, skip this. */
2003 if ((1 << sk->sk_state) &
2004 (TCPF_ESTABLISHED | TCPF_SYN_SENT |
2005 TCPF_SYN_RECV | TCPF_CLOSE_WAIT)) {
2006 /* Clear out any half completed packets. FIN if needed. */
2007 if (tcp_close_state(sk))
2008 tcp_send_fin(sk);
2009 }
2010 }
2011 EXPORT_SYMBOL(tcp_shutdown);
2012
tcp_check_oom(struct sock * sk,int shift)2013 bool tcp_check_oom(struct sock *sk, int shift)
2014 {
2015 bool too_many_orphans, out_of_socket_memory;
2016
2017 too_many_orphans = tcp_too_many_orphans(sk, shift);
2018 out_of_socket_memory = tcp_out_of_memory(sk);
2019
2020 if (too_many_orphans)
2021 net_info_ratelimited("too many orphaned sockets\n");
2022 if (out_of_socket_memory)
2023 net_info_ratelimited("out of memory -- consider tuning tcp_mem\n");
2024 return too_many_orphans || out_of_socket_memory;
2025 }
2026
tcp_close(struct sock * sk,long timeout)2027 void tcp_close(struct sock *sk, long timeout)
2028 {
2029 struct sk_buff *skb;
2030 int data_was_unread = 0;
2031 int state;
2032
2033 lock_sock(sk);
2034 sk->sk_shutdown = SHUTDOWN_MASK;
2035
2036 if (sk->sk_state == TCP_LISTEN) {
2037 tcp_set_state(sk, TCP_CLOSE);
2038
2039 /* Special case. */
2040 inet_csk_listen_stop(sk);
2041
2042 goto adjudge_to_death;
2043 }
2044
2045 /* We need to flush the recv. buffs. We do this only on the
2046 * descriptor close, not protocol-sourced closes, because the
2047 * reader process may not have drained the data yet!
2048 */
2049 while ((skb = __skb_dequeue(&sk->sk_receive_queue)) != NULL) {
2050 u32 len = TCP_SKB_CB(skb)->end_seq - TCP_SKB_CB(skb)->seq;
2051
2052 if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)
2053 len--;
2054 data_was_unread += len;
2055 __kfree_skb(skb);
2056 }
2057
2058 sk_mem_reclaim(sk);
2059
2060 /* If socket has been already reset (e.g. in tcp_reset()) - kill it. */
2061 if (sk->sk_state == TCP_CLOSE)
2062 goto adjudge_to_death;
2063
2064 /* As outlined in RFC 2525, section 2.17, we send a RST here because
2065 * data was lost. To witness the awful effects of the old behavior of
2066 * always doing a FIN, run an older 2.1.x kernel or 2.0.x, start a bulk
2067 * GET in an FTP client, suspend the process, wait for the client to
2068 * advertise a zero window, then kill -9 the FTP client, wheee...
2069 * Note: timeout is always zero in such a case.
2070 */
2071 if (unlikely(tcp_sk(sk)->repair)) {
2072 sk->sk_prot->disconnect(sk, 0);
2073 } else if (data_was_unread) {
2074 /* Unread data was tossed, zap the connection. */
2075 NET_INC_STATS_USER(sock_net(sk), LINUX_MIB_TCPABORTONCLOSE);
2076 tcp_set_state(sk, TCP_CLOSE);
2077 tcp_send_active_reset(sk, sk->sk_allocation);
2078 } else if (sock_flag(sk, SOCK_LINGER) && !sk->sk_lingertime) {
2079 /* Check zero linger _after_ checking for unread data. */
2080 sk->sk_prot->disconnect(sk, 0);
2081 NET_INC_STATS_USER(sock_net(sk), LINUX_MIB_TCPABORTONDATA);
2082 } else if (tcp_close_state(sk)) {
2083 /* We FIN if the application ate all the data before
2084 * zapping the connection.
2085 */
2086
2087 /* RED-PEN. Formally speaking, we have broken TCP state
2088 * machine. State transitions:
2089 *
2090 * TCP_ESTABLISHED -> TCP_FIN_WAIT1
2091 * TCP_SYN_RECV -> TCP_FIN_WAIT1 (forget it, it's impossible)
2092 * TCP_CLOSE_WAIT -> TCP_LAST_ACK
2093 *
2094 * are legal only when FIN has been sent (i.e. in window),
2095 * rather than queued out of window. Purists blame.
2096 *
2097 * F.e. "RFC state" is ESTABLISHED,
2098 * if Linux state is FIN-WAIT-1, but FIN is still not sent.
2099 *
2100 * The visible declinations are that sometimes
2101 * we enter time-wait state, when it is not required really
2102 * (harmless), do not send active resets, when they are
2103 * required by specs (TCP_ESTABLISHED, TCP_CLOSE_WAIT, when
2104 * they look as CLOSING or LAST_ACK for Linux)
2105 * Probably, I missed some more holelets.
2106 * --ANK
2107 * XXX (TFO) - To start off we don't support SYN+ACK+FIN
2108 * in a single packet! (May consider it later but will
2109 * probably need API support or TCP_CORK SYN-ACK until
2110 * data is written and socket is closed.)
2111 */
2112 tcp_send_fin(sk);
2113 }
2114
2115 sk_stream_wait_close(sk, timeout);
2116
2117 adjudge_to_death:
2118 state = sk->sk_state;
2119 sock_hold(sk);
2120 sock_orphan(sk);
2121
2122 /* It is the last release_sock in its life. It will remove backlog. */
2123 release_sock(sk);
2124
2125
2126 /* Now socket is owned by kernel and we acquire BH lock
2127 to finish close. No need to check for user refs.
2128 */
2129 local_bh_disable();
2130 bh_lock_sock(sk);
2131 WARN_ON(sock_owned_by_user(sk));
2132
2133 percpu_counter_inc(sk->sk_prot->orphan_count);
2134
2135 /* Have we already been destroyed by a softirq or backlog? */
2136 if (state != TCP_CLOSE && sk->sk_state == TCP_CLOSE)
2137 goto out;
2138
2139 /* This is a (useful) BSD violating of the RFC. There is a
2140 * problem with TCP as specified in that the other end could
2141 * keep a socket open forever with no application left this end.
2142 * We use a 1 minute timeout (about the same as BSD) then kill
2143 * our end. If they send after that then tough - BUT: long enough
2144 * that we won't make the old 4*rto = almost no time - whoops
2145 * reset mistake.
2146 *
2147 * Nope, it was not mistake. It is really desired behaviour
2148 * f.e. on http servers, when such sockets are useless, but
2149 * consume significant resources. Let's do it with special
2150 * linger2 option. --ANK
2151 */
2152
2153 if (sk->sk_state == TCP_FIN_WAIT2) {
2154 struct tcp_sock *tp = tcp_sk(sk);
2155 if (tp->linger2 < 0) {
2156 tcp_set_state(sk, TCP_CLOSE);
2157 tcp_send_active_reset(sk, GFP_ATOMIC);
2158 NET_INC_STATS_BH(sock_net(sk),
2159 LINUX_MIB_TCPABORTONLINGER);
2160 } else {
2161 const int tmo = tcp_fin_time(sk);
2162
2163 if (tmo > TCP_TIMEWAIT_LEN) {
2164 inet_csk_reset_keepalive_timer(sk,
2165 tmo - TCP_TIMEWAIT_LEN);
2166 } else {
2167 tcp_time_wait(sk, TCP_FIN_WAIT2, tmo);
2168 goto out;
2169 }
2170 }
2171 }
2172 if (sk->sk_state != TCP_CLOSE) {
2173 sk_mem_reclaim(sk);
2174 if (tcp_check_oom(sk, 0)) {
2175 tcp_set_state(sk, TCP_CLOSE);
2176 tcp_send_active_reset(sk, GFP_ATOMIC);
2177 NET_INC_STATS_BH(sock_net(sk),
2178 LINUX_MIB_TCPABORTONMEMORY);
2179 } else if (!check_net(sock_net(sk))) {
2180 /* Not possible to send reset; just close */
2181 tcp_set_state(sk, TCP_CLOSE);
2182 }
2183 }
2184
2185 if (sk->sk_state == TCP_CLOSE) {
2186 struct request_sock *req = tcp_sk(sk)->fastopen_rsk;
2187 /* We could get here with a non-NULL req if the socket is
2188 * aborted (e.g., closed with unread data) before 3WHS
2189 * finishes.
2190 */
2191 if (req)
2192 reqsk_fastopen_remove(sk, req, false);
2193 inet_csk_destroy_sock(sk);
2194 }
2195 /* Otherwise, socket is reprieved until protocol close. */
2196
2197 out:
2198 bh_unlock_sock(sk);
2199 local_bh_enable();
2200 sock_put(sk);
2201 }
2202 EXPORT_SYMBOL(tcp_close);
2203
2204 /* These states need RST on ABORT according to RFC793 */
2205
tcp_need_reset(int state)2206 static inline bool tcp_need_reset(int state)
2207 {
2208 return (1 << state) &
2209 (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT | TCPF_FIN_WAIT1 |
2210 TCPF_FIN_WAIT2 | TCPF_SYN_RECV);
2211 }
2212
tcp_disconnect(struct sock * sk,int flags)2213 int tcp_disconnect(struct sock *sk, int flags)
2214 {
2215 struct inet_sock *inet = inet_sk(sk);
2216 struct inet_connection_sock *icsk = inet_csk(sk);
2217 struct tcp_sock *tp = tcp_sk(sk);
2218 int err = 0;
2219 int old_state = sk->sk_state;
2220
2221 if (old_state != TCP_CLOSE)
2222 tcp_set_state(sk, TCP_CLOSE);
2223
2224 /* ABORT function of RFC793 */
2225 if (old_state == TCP_LISTEN) {
2226 inet_csk_listen_stop(sk);
2227 } else if (unlikely(tp->repair)) {
2228 sk->sk_err = ECONNABORTED;
2229 } else if (tcp_need_reset(old_state) ||
2230 (tp->snd_nxt != tp->write_seq &&
2231 (1 << old_state) & (TCPF_CLOSING | TCPF_LAST_ACK))) {
2232 /* The last check adjusts for discrepancy of Linux wrt. RFC
2233 * states
2234 */
2235 tcp_send_active_reset(sk, gfp_any());
2236 sk->sk_err = ECONNRESET;
2237 } else if (old_state == TCP_SYN_SENT)
2238 sk->sk_err = ECONNRESET;
2239
2240 tcp_clear_xmit_timers(sk);
2241 __skb_queue_purge(&sk->sk_receive_queue);
2242 tcp_write_queue_purge(sk);
2243 skb_rbtree_purge(&tp->out_of_order_queue);
2244
2245 inet->inet_dport = 0;
2246
2247 if (!(sk->sk_userlocks & SOCK_BINDADDR_LOCK))
2248 inet_reset_saddr(sk);
2249
2250 sk->sk_shutdown = 0;
2251 sock_reset_flag(sk, SOCK_DONE);
2252 tp->srtt_us = 0;
2253 tp->write_seq += tp->max_window + 2;
2254 if (tp->write_seq == 0)
2255 tp->write_seq = 1;
2256 tp->snd_cwnd = 2;
2257 icsk->icsk_probes_out = 0;
2258 tp->packets_out = 0;
2259 tp->snd_ssthresh = TCP_INFINITE_SSTHRESH;
2260 tp->snd_cwnd_cnt = 0;
2261 tp->window_clamp = 0;
2262 if (icsk->icsk_ca_ops->release)
2263 icsk->icsk_ca_ops->release(sk);
2264 memset(icsk->icsk_ca_priv, 0, sizeof(icsk->icsk_ca_priv));
2265 tcp_set_ca_state(sk, TCP_CA_Open);
2266 tcp_clear_retrans(tp);
2267 tp->total_retrans = 0;
2268 inet_csk_delack_init(sk);
2269 /* Initialize rcv_mss to TCP_MIN_MSS to avoid division by 0
2270 * issue in __tcp_select_window()
2271 */
2272 icsk->icsk_ack.rcv_mss = TCP_MIN_MSS;
2273 tcp_init_send_head(sk);
2274 memset(&tp->rx_opt, 0, sizeof(tp->rx_opt));
2275 __sk_dst_reset(sk);
2276 dst_release(sk->sk_rx_dst);
2277 sk->sk_rx_dst = NULL;
2278 tcp_saved_syn_free(tp);
2279 tp->segs_in = 0;
2280 tp->segs_out = 0;
2281 tp->bytes_acked = 0;
2282 tp->bytes_received = 0;
2283
2284 WARN_ON(inet->inet_num && !icsk->icsk_bind_hash);
2285
2286 if (sk->sk_frag.page) {
2287 put_page(sk->sk_frag.page);
2288 sk->sk_frag.page = NULL;
2289 sk->sk_frag.offset = 0;
2290 }
2291
2292 sk->sk_error_report(sk);
2293 return err;
2294 }
2295 EXPORT_SYMBOL(tcp_disconnect);
2296
tcp_can_repair_sock(const struct sock * sk)2297 static inline bool tcp_can_repair_sock(const struct sock *sk)
2298 {
2299 return ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN) &&
2300 ((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_ESTABLISHED));
2301 }
2302
tcp_repair_options_est(struct tcp_sock * tp,struct tcp_repair_opt __user * optbuf,unsigned int len)2303 static int tcp_repair_options_est(struct tcp_sock *tp,
2304 struct tcp_repair_opt __user *optbuf, unsigned int len)
2305 {
2306 struct tcp_repair_opt opt;
2307
2308 while (len >= sizeof(opt)) {
2309 if (copy_from_user(&opt, optbuf, sizeof(opt)))
2310 return -EFAULT;
2311
2312 optbuf++;
2313 len -= sizeof(opt);
2314
2315 switch (opt.opt_code) {
2316 case TCPOPT_MSS:
2317 tp->rx_opt.mss_clamp = opt.opt_val;
2318 break;
2319 case TCPOPT_WINDOW:
2320 {
2321 u16 snd_wscale = opt.opt_val & 0xFFFF;
2322 u16 rcv_wscale = opt.opt_val >> 16;
2323
2324 if (snd_wscale > 14 || rcv_wscale > 14)
2325 return -EFBIG;
2326
2327 tp->rx_opt.snd_wscale = snd_wscale;
2328 tp->rx_opt.rcv_wscale = rcv_wscale;
2329 tp->rx_opt.wscale_ok = 1;
2330 }
2331 break;
2332 case TCPOPT_SACK_PERM:
2333 if (opt.opt_val != 0)
2334 return -EINVAL;
2335
2336 tp->rx_opt.sack_ok |= TCP_SACK_SEEN;
2337 if (sysctl_tcp_fack)
2338 tcp_enable_fack(tp);
2339 break;
2340 case TCPOPT_TIMESTAMP:
2341 if (opt.opt_val != 0)
2342 return -EINVAL;
2343
2344 tp->rx_opt.tstamp_ok = 1;
2345 break;
2346 }
2347 }
2348
2349 return 0;
2350 }
2351
2352 /*
2353 * Socket option code for TCP.
2354 */
do_tcp_setsockopt(struct sock * sk,int level,int optname,char __user * optval,unsigned int optlen)2355 static int do_tcp_setsockopt(struct sock *sk, int level,
2356 int optname, char __user *optval, unsigned int optlen)
2357 {
2358 struct tcp_sock *tp = tcp_sk(sk);
2359 struct inet_connection_sock *icsk = inet_csk(sk);
2360 int val;
2361 int err = 0;
2362
2363 /* These are data/string values, all the others are ints */
2364 switch (optname) {
2365 case TCP_CONGESTION: {
2366 char name[TCP_CA_NAME_MAX];
2367
2368 if (optlen < 1)
2369 return -EINVAL;
2370
2371 val = strncpy_from_user(name, optval,
2372 min_t(long, TCP_CA_NAME_MAX-1, optlen));
2373 if (val < 0)
2374 return -EFAULT;
2375 name[val] = 0;
2376
2377 lock_sock(sk);
2378 err = tcp_set_congestion_control(sk, name);
2379 release_sock(sk);
2380 return err;
2381 }
2382 default:
2383 /* fallthru */
2384 break;
2385 }
2386
2387 if (optlen < sizeof(int))
2388 return -EINVAL;
2389
2390 if (get_user(val, (int __user *)optval))
2391 return -EFAULT;
2392
2393 lock_sock(sk);
2394
2395 switch (optname) {
2396 case TCP_MAXSEG:
2397 /* Values greater than interface MTU won't take effect. However
2398 * at the point when this call is done we typically don't yet
2399 * know which interface is going to be used */
2400 if (val < TCP_MIN_MSS || val > MAX_TCP_WINDOW) {
2401 err = -EINVAL;
2402 break;
2403 }
2404 tp->rx_opt.user_mss = val;
2405 break;
2406
2407 case TCP_NODELAY:
2408 if (val) {
2409 /* TCP_NODELAY is weaker than TCP_CORK, so that
2410 * this option on corked socket is remembered, but
2411 * it is not activated until cork is cleared.
2412 *
2413 * However, when TCP_NODELAY is set we make
2414 * an explicit push, which overrides even TCP_CORK
2415 * for currently queued segments.
2416 */
2417 tp->nonagle |= TCP_NAGLE_OFF|TCP_NAGLE_PUSH;
2418 tcp_push_pending_frames(sk);
2419 } else {
2420 tp->nonagle &= ~TCP_NAGLE_OFF;
2421 }
2422 break;
2423
2424 case TCP_THIN_LINEAR_TIMEOUTS:
2425 if (val < 0 || val > 1)
2426 err = -EINVAL;
2427 else
2428 tp->thin_lto = val;
2429 break;
2430
2431 case TCP_THIN_DUPACK:
2432 if (val < 0 || val > 1)
2433 err = -EINVAL;
2434 else {
2435 tp->thin_dupack = val;
2436 if (tp->thin_dupack)
2437 tcp_disable_early_retrans(tp);
2438 }
2439 break;
2440
2441 case TCP_REPAIR:
2442 if (!tcp_can_repair_sock(sk))
2443 err = -EPERM;
2444 else if (val == 1) {
2445 tp->repair = 1;
2446 sk->sk_reuse = SK_FORCE_REUSE;
2447 tp->repair_queue = TCP_NO_QUEUE;
2448 } else if (val == 0) {
2449 tp->repair = 0;
2450 sk->sk_reuse = SK_NO_REUSE;
2451 tcp_send_window_probe(sk);
2452 } else
2453 err = -EINVAL;
2454
2455 break;
2456
2457 case TCP_REPAIR_QUEUE:
2458 if (!tp->repair)
2459 err = -EPERM;
2460 else if ((unsigned int)val < TCP_QUEUES_NR)
2461 tp->repair_queue = val;
2462 else
2463 err = -EINVAL;
2464 break;
2465
2466 case TCP_QUEUE_SEQ:
2467 if (sk->sk_state != TCP_CLOSE)
2468 err = -EPERM;
2469 else if (tp->repair_queue == TCP_SEND_QUEUE)
2470 tp->write_seq = val;
2471 else if (tp->repair_queue == TCP_RECV_QUEUE)
2472 tp->rcv_nxt = val;
2473 else
2474 err = -EINVAL;
2475 break;
2476
2477 case TCP_REPAIR_OPTIONS:
2478 if (!tp->repair)
2479 err = -EINVAL;
2480 else if (sk->sk_state == TCP_ESTABLISHED)
2481 err = tcp_repair_options_est(tp,
2482 (struct tcp_repair_opt __user *)optval,
2483 optlen);
2484 else
2485 err = -EPERM;
2486 break;
2487
2488 case TCP_CORK:
2489 /* When set indicates to always queue non-full frames.
2490 * Later the user clears this option and we transmit
2491 * any pending partial frames in the queue. This is
2492 * meant to be used alongside sendfile() to get properly
2493 * filled frames when the user (for example) must write
2494 * out headers with a write() call first and then use
2495 * sendfile to send out the data parts.
2496 *
2497 * TCP_CORK can be set together with TCP_NODELAY and it is
2498 * stronger than TCP_NODELAY.
2499 */
2500 if (val) {
2501 tp->nonagle |= TCP_NAGLE_CORK;
2502 } else {
2503 tp->nonagle &= ~TCP_NAGLE_CORK;
2504 if (tp->nonagle&TCP_NAGLE_OFF)
2505 tp->nonagle |= TCP_NAGLE_PUSH;
2506 tcp_push_pending_frames(sk);
2507 }
2508 break;
2509
2510 case TCP_KEEPIDLE:
2511 if (val < 1 || val > MAX_TCP_KEEPIDLE)
2512 err = -EINVAL;
2513 else {
2514 tp->keepalive_time = val * HZ;
2515 if (sock_flag(sk, SOCK_KEEPOPEN) &&
2516 !((1 << sk->sk_state) &
2517 (TCPF_CLOSE | TCPF_LISTEN))) {
2518 u32 elapsed = keepalive_time_elapsed(tp);
2519 if (tp->keepalive_time > elapsed)
2520 elapsed = tp->keepalive_time - elapsed;
2521 else
2522 elapsed = 0;
2523 inet_csk_reset_keepalive_timer(sk, elapsed);
2524 }
2525 }
2526 break;
2527 case TCP_KEEPINTVL:
2528 if (val < 1 || val > MAX_TCP_KEEPINTVL)
2529 err = -EINVAL;
2530 else
2531 tp->keepalive_intvl = val * HZ;
2532 break;
2533 case TCP_KEEPCNT:
2534 if (val < 1 || val > MAX_TCP_KEEPCNT)
2535 err = -EINVAL;
2536 else
2537 tp->keepalive_probes = val;
2538 break;
2539 case TCP_SYNCNT:
2540 if (val < 1 || val > MAX_TCP_SYNCNT)
2541 err = -EINVAL;
2542 else
2543 icsk->icsk_syn_retries = val;
2544 break;
2545
2546 case TCP_SAVE_SYN:
2547 if (val < 0 || val > 1)
2548 err = -EINVAL;
2549 else
2550 tp->save_syn = val;
2551 break;
2552
2553 case TCP_LINGER2:
2554 if (val < 0)
2555 tp->linger2 = -1;
2556 else if (val > sysctl_tcp_fin_timeout / HZ)
2557 tp->linger2 = 0;
2558 else
2559 tp->linger2 = val * HZ;
2560 break;
2561
2562 case TCP_DEFER_ACCEPT:
2563 /* Translate value in seconds to number of retransmits */
2564 icsk->icsk_accept_queue.rskq_defer_accept =
2565 secs_to_retrans(val, TCP_TIMEOUT_INIT / HZ,
2566 TCP_RTO_MAX / HZ);
2567 break;
2568
2569 case TCP_WINDOW_CLAMP:
2570 if (!val) {
2571 if (sk->sk_state != TCP_CLOSE) {
2572 err = -EINVAL;
2573 break;
2574 }
2575 tp->window_clamp = 0;
2576 } else
2577 tp->window_clamp = val < SOCK_MIN_RCVBUF / 2 ?
2578 SOCK_MIN_RCVBUF / 2 : val;
2579 break;
2580
2581 case TCP_QUICKACK:
2582 if (!val) {
2583 icsk->icsk_ack.pingpong = 1;
2584 } else {
2585 icsk->icsk_ack.pingpong = 0;
2586 if ((1 << sk->sk_state) &
2587 (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT) &&
2588 inet_csk_ack_scheduled(sk)) {
2589 icsk->icsk_ack.pending |= ICSK_ACK_PUSHED;
2590 tcp_cleanup_rbuf(sk, 1);
2591 if (!(val & 1))
2592 icsk->icsk_ack.pingpong = 1;
2593 }
2594 }
2595 break;
2596
2597 #ifdef CONFIG_TCP_MD5SIG
2598 case TCP_MD5SIG:
2599 err = tp->af_specific->md5_parse(sk, optval, optlen);
2600 break;
2601 #endif
2602 case TCP_USER_TIMEOUT:
2603 /* Cap the max time in ms TCP will retry or probe the window
2604 * before giving up and aborting (ETIMEDOUT) a connection.
2605 */
2606 if (val < 0)
2607 err = -EINVAL;
2608 else
2609 icsk->icsk_user_timeout = msecs_to_jiffies(val);
2610 break;
2611
2612 case TCP_FASTOPEN:
2613 if (val >= 0 && ((1 << sk->sk_state) & (TCPF_CLOSE |
2614 TCPF_LISTEN))) {
2615 tcp_fastopen_init_key_once(true);
2616
2617 fastopen_queue_tune(sk, val);
2618 } else {
2619 err = -EINVAL;
2620 }
2621 break;
2622 case TCP_TIMESTAMP:
2623 if (!tp->repair)
2624 err = -EPERM;
2625 else
2626 tp->tsoffset = val - tcp_time_stamp;
2627 break;
2628 case TCP_NOTSENT_LOWAT:
2629 tp->notsent_lowat = val;
2630 sk->sk_write_space(sk);
2631 break;
2632 default:
2633 err = -ENOPROTOOPT;
2634 break;
2635 }
2636
2637 release_sock(sk);
2638 return err;
2639 }
2640
tcp_setsockopt(struct sock * sk,int level,int optname,char __user * optval,unsigned int optlen)2641 int tcp_setsockopt(struct sock *sk, int level, int optname, char __user *optval,
2642 unsigned int optlen)
2643 {
2644 const struct inet_connection_sock *icsk = inet_csk(sk);
2645
2646 if (level != SOL_TCP)
2647 return icsk->icsk_af_ops->setsockopt(sk, level, optname,
2648 optval, optlen);
2649 return do_tcp_setsockopt(sk, level, optname, optval, optlen);
2650 }
2651 EXPORT_SYMBOL(tcp_setsockopt);
2652
2653 #ifdef CONFIG_COMPAT
compat_tcp_setsockopt(struct sock * sk,int level,int optname,char __user * optval,unsigned int optlen)2654 int compat_tcp_setsockopt(struct sock *sk, int level, int optname,
2655 char __user *optval, unsigned int optlen)
2656 {
2657 if (level != SOL_TCP)
2658 return inet_csk_compat_setsockopt(sk, level, optname,
2659 optval, optlen);
2660 return do_tcp_setsockopt(sk, level, optname, optval, optlen);
2661 }
2662 EXPORT_SYMBOL(compat_tcp_setsockopt);
2663 #endif
2664
2665 /* Return information about state of tcp endpoint in API format. */
tcp_get_info(struct sock * sk,struct tcp_info * info)2666 void tcp_get_info(struct sock *sk, struct tcp_info *info)
2667 {
2668 const struct tcp_sock *tp = tcp_sk(sk); /* iff sk_type == SOCK_STREAM */
2669 const struct inet_connection_sock *icsk = inet_csk(sk);
2670 u32 now = tcp_time_stamp;
2671 unsigned int start;
2672 u64 rate64;
2673 u32 rate;
2674
2675 memset(info, 0, sizeof(*info));
2676 if (sk->sk_type != SOCK_STREAM)
2677 return;
2678
2679 info->tcpi_state = sk_state_load(sk);
2680
2681 info->tcpi_ca_state = icsk->icsk_ca_state;
2682 info->tcpi_retransmits = icsk->icsk_retransmits;
2683 info->tcpi_probes = icsk->icsk_probes_out;
2684 info->tcpi_backoff = icsk->icsk_backoff;
2685
2686 if (tp->rx_opt.tstamp_ok)
2687 info->tcpi_options |= TCPI_OPT_TIMESTAMPS;
2688 if (tcp_is_sack(tp))
2689 info->tcpi_options |= TCPI_OPT_SACK;
2690 if (tp->rx_opt.wscale_ok) {
2691 info->tcpi_options |= TCPI_OPT_WSCALE;
2692 info->tcpi_snd_wscale = tp->rx_opt.snd_wscale;
2693 info->tcpi_rcv_wscale = tp->rx_opt.rcv_wscale;
2694 }
2695
2696 if (tp->ecn_flags & TCP_ECN_OK)
2697 info->tcpi_options |= TCPI_OPT_ECN;
2698 if (tp->ecn_flags & TCP_ECN_SEEN)
2699 info->tcpi_options |= TCPI_OPT_ECN_SEEN;
2700 if (tp->syn_data_acked)
2701 info->tcpi_options |= TCPI_OPT_SYN_DATA;
2702
2703 info->tcpi_rto = jiffies_to_usecs(icsk->icsk_rto);
2704 info->tcpi_ato = jiffies_to_usecs(icsk->icsk_ack.ato);
2705 info->tcpi_snd_mss = tp->mss_cache;
2706 info->tcpi_rcv_mss = icsk->icsk_ack.rcv_mss;
2707
2708 if (info->tcpi_state == TCP_LISTEN) {
2709 info->tcpi_unacked = sk->sk_ack_backlog;
2710 info->tcpi_sacked = sk->sk_max_ack_backlog;
2711 } else {
2712 info->tcpi_unacked = tp->packets_out;
2713 info->tcpi_sacked = tp->sacked_out;
2714 }
2715 info->tcpi_lost = tp->lost_out;
2716 info->tcpi_retrans = tp->retrans_out;
2717 info->tcpi_fackets = tp->fackets_out;
2718
2719 info->tcpi_last_data_sent = jiffies_to_msecs(now - tp->lsndtime);
2720 info->tcpi_last_data_recv = jiffies_to_msecs(now - icsk->icsk_ack.lrcvtime);
2721 info->tcpi_last_ack_recv = jiffies_to_msecs(now - tp->rcv_tstamp);
2722
2723 info->tcpi_pmtu = icsk->icsk_pmtu_cookie;
2724 info->tcpi_rcv_ssthresh = tp->rcv_ssthresh;
2725 info->tcpi_rtt = tp->srtt_us >> 3;
2726 info->tcpi_rttvar = tp->mdev_us >> 2;
2727 info->tcpi_snd_ssthresh = tp->snd_ssthresh;
2728 info->tcpi_snd_cwnd = tp->snd_cwnd;
2729 info->tcpi_advmss = tp->advmss;
2730 info->tcpi_reordering = tp->reordering;
2731
2732 info->tcpi_rcv_rtt = jiffies_to_usecs(tp->rcv_rtt_est.rtt)>>3;
2733 info->tcpi_rcv_space = tp->rcvq_space.space;
2734
2735 info->tcpi_total_retrans = tp->total_retrans;
2736
2737 rate = READ_ONCE(sk->sk_pacing_rate);
2738 rate64 = rate != ~0U ? rate : ~0ULL;
2739 put_unaligned(rate64, &info->tcpi_pacing_rate);
2740
2741 rate = READ_ONCE(sk->sk_max_pacing_rate);
2742 rate64 = rate != ~0U ? rate : ~0ULL;
2743 put_unaligned(rate64, &info->tcpi_max_pacing_rate);
2744
2745 do {
2746 start = u64_stats_fetch_begin_irq(&tp->syncp);
2747 put_unaligned(tp->bytes_acked, &info->tcpi_bytes_acked);
2748 put_unaligned(tp->bytes_received, &info->tcpi_bytes_received);
2749 } while (u64_stats_fetch_retry_irq(&tp->syncp, start));
2750 info->tcpi_segs_out = tp->segs_out;
2751 info->tcpi_segs_in = tp->segs_in;
2752 }
2753 EXPORT_SYMBOL_GPL(tcp_get_info);
2754
do_tcp_getsockopt(struct sock * sk,int level,int optname,char __user * optval,int __user * optlen)2755 static int do_tcp_getsockopt(struct sock *sk, int level,
2756 int optname, char __user *optval, int __user *optlen)
2757 {
2758 struct inet_connection_sock *icsk = inet_csk(sk);
2759 struct tcp_sock *tp = tcp_sk(sk);
2760 int val, len;
2761
2762 if (get_user(len, optlen))
2763 return -EFAULT;
2764
2765 len = min_t(unsigned int, len, sizeof(int));
2766
2767 if (len < 0)
2768 return -EINVAL;
2769
2770 switch (optname) {
2771 case TCP_MAXSEG:
2772 val = tp->mss_cache;
2773 if (!val && ((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN)))
2774 val = tp->rx_opt.user_mss;
2775 if (tp->repair)
2776 val = tp->rx_opt.mss_clamp;
2777 break;
2778 case TCP_NODELAY:
2779 val = !!(tp->nonagle&TCP_NAGLE_OFF);
2780 break;
2781 case TCP_CORK:
2782 val = !!(tp->nonagle&TCP_NAGLE_CORK);
2783 break;
2784 case TCP_KEEPIDLE:
2785 val = keepalive_time_when(tp) / HZ;
2786 break;
2787 case TCP_KEEPINTVL:
2788 val = keepalive_intvl_when(tp) / HZ;
2789 break;
2790 case TCP_KEEPCNT:
2791 val = keepalive_probes(tp);
2792 break;
2793 case TCP_SYNCNT:
2794 val = icsk->icsk_syn_retries ? : sysctl_tcp_syn_retries;
2795 break;
2796 case TCP_LINGER2:
2797 val = tp->linger2;
2798 if (val >= 0)
2799 val = (val ? : sysctl_tcp_fin_timeout) / HZ;
2800 break;
2801 case TCP_DEFER_ACCEPT:
2802 val = retrans_to_secs(icsk->icsk_accept_queue.rskq_defer_accept,
2803 TCP_TIMEOUT_INIT / HZ, TCP_RTO_MAX / HZ);
2804 break;
2805 case TCP_WINDOW_CLAMP:
2806 val = tp->window_clamp;
2807 break;
2808 case TCP_INFO: {
2809 struct tcp_info info;
2810
2811 if (get_user(len, optlen))
2812 return -EFAULT;
2813
2814 tcp_get_info(sk, &info);
2815
2816 len = min_t(unsigned int, len, sizeof(info));
2817 if (put_user(len, optlen))
2818 return -EFAULT;
2819 if (copy_to_user(optval, &info, len))
2820 return -EFAULT;
2821 return 0;
2822 }
2823 case TCP_CC_INFO: {
2824 const struct tcp_congestion_ops *ca_ops;
2825 union tcp_cc_info info;
2826 size_t sz = 0;
2827 int attr;
2828
2829 if (get_user(len, optlen))
2830 return -EFAULT;
2831
2832 ca_ops = icsk->icsk_ca_ops;
2833 if (ca_ops && ca_ops->get_info)
2834 sz = ca_ops->get_info(sk, ~0U, &attr, &info);
2835
2836 len = min_t(unsigned int, len, sz);
2837 if (put_user(len, optlen))
2838 return -EFAULT;
2839 if (copy_to_user(optval, &info, len))
2840 return -EFAULT;
2841 return 0;
2842 }
2843 case TCP_QUICKACK:
2844 val = !icsk->icsk_ack.pingpong;
2845 break;
2846
2847 case TCP_CONGESTION:
2848 if (get_user(len, optlen))
2849 return -EFAULT;
2850 len = min_t(unsigned int, len, TCP_CA_NAME_MAX);
2851 if (put_user(len, optlen))
2852 return -EFAULT;
2853 if (copy_to_user(optval, icsk->icsk_ca_ops->name, len))
2854 return -EFAULT;
2855 return 0;
2856
2857 case TCP_THIN_LINEAR_TIMEOUTS:
2858 val = tp->thin_lto;
2859 break;
2860 case TCP_THIN_DUPACK:
2861 val = tp->thin_dupack;
2862 break;
2863
2864 case TCP_REPAIR:
2865 val = tp->repair;
2866 break;
2867
2868 case TCP_REPAIR_QUEUE:
2869 if (tp->repair)
2870 val = tp->repair_queue;
2871 else
2872 return -EINVAL;
2873 break;
2874
2875 case TCP_QUEUE_SEQ:
2876 if (tp->repair_queue == TCP_SEND_QUEUE)
2877 val = tp->write_seq;
2878 else if (tp->repair_queue == TCP_RECV_QUEUE)
2879 val = tp->rcv_nxt;
2880 else
2881 return -EINVAL;
2882 break;
2883
2884 case TCP_USER_TIMEOUT:
2885 val = jiffies_to_msecs(icsk->icsk_user_timeout);
2886 break;
2887
2888 case TCP_FASTOPEN:
2889 val = icsk->icsk_accept_queue.fastopenq.max_qlen;
2890 break;
2891
2892 case TCP_TIMESTAMP:
2893 val = tcp_time_stamp + tp->tsoffset;
2894 break;
2895 case TCP_NOTSENT_LOWAT:
2896 val = tp->notsent_lowat;
2897 break;
2898 case TCP_SAVE_SYN:
2899 val = tp->save_syn;
2900 break;
2901 case TCP_SAVED_SYN: {
2902 if (get_user(len, optlen))
2903 return -EFAULT;
2904
2905 lock_sock(sk);
2906 if (tp->saved_syn) {
2907 if (len < tp->saved_syn[0]) {
2908 if (put_user(tp->saved_syn[0], optlen)) {
2909 release_sock(sk);
2910 return -EFAULT;
2911 }
2912 release_sock(sk);
2913 return -EINVAL;
2914 }
2915 len = tp->saved_syn[0];
2916 if (put_user(len, optlen)) {
2917 release_sock(sk);
2918 return -EFAULT;
2919 }
2920 if (copy_to_user(optval, tp->saved_syn + 1, len)) {
2921 release_sock(sk);
2922 return -EFAULT;
2923 }
2924 tcp_saved_syn_free(tp);
2925 release_sock(sk);
2926 } else {
2927 release_sock(sk);
2928 len = 0;
2929 if (put_user(len, optlen))
2930 return -EFAULT;
2931 }
2932 return 0;
2933 }
2934 default:
2935 return -ENOPROTOOPT;
2936 }
2937
2938 if (put_user(len, optlen))
2939 return -EFAULT;
2940 if (copy_to_user(optval, &val, len))
2941 return -EFAULT;
2942 return 0;
2943 }
2944
tcp_getsockopt(struct sock * sk,int level,int optname,char __user * optval,int __user * optlen)2945 int tcp_getsockopt(struct sock *sk, int level, int optname, char __user *optval,
2946 int __user *optlen)
2947 {
2948 struct inet_connection_sock *icsk = inet_csk(sk);
2949
2950 if (level != SOL_TCP)
2951 return icsk->icsk_af_ops->getsockopt(sk, level, optname,
2952 optval, optlen);
2953 return do_tcp_getsockopt(sk, level, optname, optval, optlen);
2954 }
2955 EXPORT_SYMBOL(tcp_getsockopt);
2956
2957 #ifdef CONFIG_COMPAT
compat_tcp_getsockopt(struct sock * sk,int level,int optname,char __user * optval,int __user * optlen)2958 int compat_tcp_getsockopt(struct sock *sk, int level, int optname,
2959 char __user *optval, int __user *optlen)
2960 {
2961 if (level != SOL_TCP)
2962 return inet_csk_compat_getsockopt(sk, level, optname,
2963 optval, optlen);
2964 return do_tcp_getsockopt(sk, level, optname, optval, optlen);
2965 }
2966 EXPORT_SYMBOL(compat_tcp_getsockopt);
2967 #endif
2968
2969 #ifdef CONFIG_TCP_MD5SIG
2970 static DEFINE_PER_CPU(struct tcp_md5sig_pool, tcp_md5sig_pool);
2971 static DEFINE_MUTEX(tcp_md5sig_mutex);
2972 static bool tcp_md5sig_pool_populated = false;
2973
__tcp_alloc_md5sig_pool(void)2974 static void __tcp_alloc_md5sig_pool(void)
2975 {
2976 int cpu;
2977
2978 for_each_possible_cpu(cpu) {
2979 if (!per_cpu(tcp_md5sig_pool, cpu).md5_desc.tfm) {
2980 struct crypto_hash *hash;
2981
2982 hash = crypto_alloc_hash("md5", 0, CRYPTO_ALG_ASYNC);
2983 if (IS_ERR_OR_NULL(hash))
2984 return;
2985 per_cpu(tcp_md5sig_pool, cpu).md5_desc.tfm = hash;
2986 }
2987 }
2988 /* before setting tcp_md5sig_pool_populated, we must commit all writes
2989 * to memory. See smp_rmb() in tcp_get_md5sig_pool()
2990 */
2991 smp_wmb();
2992 tcp_md5sig_pool_populated = true;
2993 }
2994
tcp_alloc_md5sig_pool(void)2995 bool tcp_alloc_md5sig_pool(void)
2996 {
2997 if (unlikely(!tcp_md5sig_pool_populated)) {
2998 mutex_lock(&tcp_md5sig_mutex);
2999
3000 if (!tcp_md5sig_pool_populated)
3001 __tcp_alloc_md5sig_pool();
3002
3003 mutex_unlock(&tcp_md5sig_mutex);
3004 }
3005 return tcp_md5sig_pool_populated;
3006 }
3007 EXPORT_SYMBOL(tcp_alloc_md5sig_pool);
3008
3009
3010 /**
3011 * tcp_get_md5sig_pool - get md5sig_pool for this user
3012 *
3013 * We use percpu structure, so if we succeed, we exit with preemption
3014 * and BH disabled, to make sure another thread or softirq handling
3015 * wont try to get same context.
3016 */
tcp_get_md5sig_pool(void)3017 struct tcp_md5sig_pool *tcp_get_md5sig_pool(void)
3018 {
3019 local_bh_disable();
3020
3021 if (tcp_md5sig_pool_populated) {
3022 /* coupled with smp_wmb() in __tcp_alloc_md5sig_pool() */
3023 smp_rmb();
3024 return this_cpu_ptr(&tcp_md5sig_pool);
3025 }
3026 local_bh_enable();
3027 return NULL;
3028 }
3029 EXPORT_SYMBOL(tcp_get_md5sig_pool);
3030
tcp_md5_hash_header(struct tcp_md5sig_pool * hp,const struct tcphdr * th)3031 int tcp_md5_hash_header(struct tcp_md5sig_pool *hp,
3032 const struct tcphdr *th)
3033 {
3034 struct scatterlist sg;
3035 struct tcphdr hdr;
3036 int err;
3037
3038 /* We are not allowed to change tcphdr, make a local copy */
3039 memcpy(&hdr, th, sizeof(hdr));
3040 hdr.check = 0;
3041
3042 /* options aren't included in the hash */
3043 sg_init_one(&sg, &hdr, sizeof(hdr));
3044 err = crypto_hash_update(&hp->md5_desc, &sg, sizeof(hdr));
3045 return err;
3046 }
3047 EXPORT_SYMBOL(tcp_md5_hash_header);
3048
tcp_md5_hash_skb_data(struct tcp_md5sig_pool * hp,const struct sk_buff * skb,unsigned int header_len)3049 int tcp_md5_hash_skb_data(struct tcp_md5sig_pool *hp,
3050 const struct sk_buff *skb, unsigned int header_len)
3051 {
3052 struct scatterlist sg;
3053 const struct tcphdr *tp = tcp_hdr(skb);
3054 struct hash_desc *desc = &hp->md5_desc;
3055 unsigned int i;
3056 const unsigned int head_data_len = skb_headlen(skb) > header_len ?
3057 skb_headlen(skb) - header_len : 0;
3058 const struct skb_shared_info *shi = skb_shinfo(skb);
3059 struct sk_buff *frag_iter;
3060
3061 sg_init_table(&sg, 1);
3062
3063 sg_set_buf(&sg, ((u8 *) tp) + header_len, head_data_len);
3064 if (crypto_hash_update(desc, &sg, head_data_len))
3065 return 1;
3066
3067 for (i = 0; i < shi->nr_frags; ++i) {
3068 const struct skb_frag_struct *f = &shi->frags[i];
3069 unsigned int offset = f->page_offset;
3070 struct page *page = skb_frag_page(f) + (offset >> PAGE_SHIFT);
3071
3072 sg_set_page(&sg, page, skb_frag_size(f),
3073 offset_in_page(offset));
3074 if (crypto_hash_update(desc, &sg, skb_frag_size(f)))
3075 return 1;
3076 }
3077
3078 skb_walk_frags(skb, frag_iter)
3079 if (tcp_md5_hash_skb_data(hp, frag_iter, 0))
3080 return 1;
3081
3082 return 0;
3083 }
3084 EXPORT_SYMBOL(tcp_md5_hash_skb_data);
3085
tcp_md5_hash_key(struct tcp_md5sig_pool * hp,const struct tcp_md5sig_key * key)3086 int tcp_md5_hash_key(struct tcp_md5sig_pool *hp, const struct tcp_md5sig_key *key)
3087 {
3088 u8 keylen = READ_ONCE(key->keylen); /* paired with WRITE_ONCE() in tcp_md5_do_add */
3089 struct scatterlist sg;
3090
3091 sg_init_one(&sg, key->key, keylen);
3092
3093 /* tcp_md5_do_add() might change key->key under us */
3094 return crypto_hash_update(&hp->md5_desc, &sg, key->keylen);
3095 }
3096 EXPORT_SYMBOL(tcp_md5_hash_key);
3097
3098 #endif
3099
tcp_done(struct sock * sk)3100 void tcp_done(struct sock *sk)
3101 {
3102 struct request_sock *req = tcp_sk(sk)->fastopen_rsk;
3103
3104 if (sk->sk_state == TCP_SYN_SENT || sk->sk_state == TCP_SYN_RECV)
3105 TCP_INC_STATS_BH(sock_net(sk), TCP_MIB_ATTEMPTFAILS);
3106
3107 tcp_set_state(sk, TCP_CLOSE);
3108 tcp_clear_xmit_timers(sk);
3109 if (req)
3110 reqsk_fastopen_remove(sk, req, false);
3111
3112 sk->sk_shutdown = SHUTDOWN_MASK;
3113
3114 if (!sock_flag(sk, SOCK_DEAD))
3115 sk->sk_state_change(sk);
3116 else
3117 inet_csk_destroy_sock(sk);
3118 }
3119 EXPORT_SYMBOL_GPL(tcp_done);
3120
tcp_abort(struct sock * sk,int err)3121 int tcp_abort(struct sock *sk, int err)
3122 {
3123 if (!sk_fullsock(sk)) {
3124 if (sk->sk_state == TCP_NEW_SYN_RECV) {
3125 struct request_sock *req = inet_reqsk(sk);
3126
3127 local_bh_disable();
3128 inet_csk_reqsk_queue_drop_and_put(req->rsk_listener,
3129 req);
3130 local_bh_enable();
3131 return 0;
3132 }
3133 sock_gen_put(sk);
3134 return -EOPNOTSUPP;
3135 }
3136
3137 /* Don't race with userspace socket closes such as tcp_close. */
3138 lock_sock(sk);
3139
3140 if (sk->sk_state == TCP_LISTEN) {
3141 tcp_set_state(sk, TCP_CLOSE);
3142 inet_csk_listen_stop(sk);
3143 }
3144
3145 /* Don't race with BH socket closes such as inet_csk_listen_stop. */
3146 local_bh_disable();
3147 bh_lock_sock(sk);
3148
3149 if (!sock_flag(sk, SOCK_DEAD)) {
3150 sk->sk_err = err;
3151 /* This barrier is coupled with smp_rmb() in tcp_poll() */
3152 smp_wmb();
3153 sk->sk_error_report(sk);
3154 if (tcp_need_reset(sk->sk_state))
3155 tcp_send_active_reset(sk, GFP_ATOMIC);
3156 tcp_done(sk);
3157 }
3158
3159 bh_unlock_sock(sk);
3160 local_bh_enable();
3161 release_sock(sk);
3162 sock_put(sk);
3163 return 0;
3164 }
3165 EXPORT_SYMBOL_GPL(tcp_abort);
3166
3167 extern struct tcp_congestion_ops tcp_reno;
3168
3169 static __initdata unsigned long thash_entries;
set_thash_entries(char * str)3170 static int __init set_thash_entries(char *str)
3171 {
3172 ssize_t ret;
3173
3174 if (!str)
3175 return 0;
3176
3177 ret = kstrtoul(str, 0, &thash_entries);
3178 if (ret)
3179 return 0;
3180
3181 return 1;
3182 }
3183 __setup("thash_entries=", set_thash_entries);
3184
tcp_init_mem(void)3185 static void __init tcp_init_mem(void)
3186 {
3187 unsigned long limit = nr_free_buffer_pages() / 16;
3188
3189 limit = max(limit, 128UL);
3190 sysctl_tcp_mem[0] = limit / 4 * 3; /* 4.68 % */
3191 sysctl_tcp_mem[1] = limit; /* 6.25 % */
3192 sysctl_tcp_mem[2] = sysctl_tcp_mem[0] * 2; /* 9.37 % */
3193 }
3194
tcp_init(void)3195 void __init tcp_init(void)
3196 {
3197 unsigned long limit;
3198 int max_rshare, max_wshare, cnt;
3199 unsigned int i;
3200
3201 BUILD_BUG_ON(TCP_MIN_SND_MSS <= MAX_TCP_OPTION_SPACE);
3202 sock_skb_cb_check_size(sizeof(struct tcp_skb_cb));
3203
3204 percpu_counter_init(&tcp_sockets_allocated, 0, GFP_KERNEL);
3205 percpu_counter_init(&tcp_orphan_count, 0, GFP_KERNEL);
3206 tcp_hashinfo.bind_bucket_cachep =
3207 kmem_cache_create("tcp_bind_bucket",
3208 sizeof(struct inet_bind_bucket), 0,
3209 SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
3210
3211 /* Size and allocate the main established and bind bucket
3212 * hash tables.
3213 *
3214 * The methodology is similar to that of the buffer cache.
3215 */
3216 tcp_hashinfo.ehash =
3217 alloc_large_system_hash("TCP established",
3218 sizeof(struct inet_ehash_bucket),
3219 thash_entries,
3220 17, /* one slot per 128 KB of memory */
3221 0,
3222 NULL,
3223 &tcp_hashinfo.ehash_mask,
3224 0,
3225 thash_entries ? 0 : 512 * 1024);
3226 for (i = 0; i <= tcp_hashinfo.ehash_mask; i++)
3227 INIT_HLIST_NULLS_HEAD(&tcp_hashinfo.ehash[i].chain, i);
3228
3229 if (inet_ehash_locks_alloc(&tcp_hashinfo))
3230 panic("TCP: failed to alloc ehash_locks");
3231 tcp_hashinfo.bhash =
3232 alloc_large_system_hash("TCP bind",
3233 sizeof(struct inet_bind_hashbucket),
3234 tcp_hashinfo.ehash_mask + 1,
3235 17, /* one slot per 128 KB of memory */
3236 0,
3237 &tcp_hashinfo.bhash_size,
3238 NULL,
3239 0,
3240 64 * 1024);
3241 tcp_hashinfo.bhash_size = 1U << tcp_hashinfo.bhash_size;
3242 for (i = 0; i < tcp_hashinfo.bhash_size; i++) {
3243 spin_lock_init(&tcp_hashinfo.bhash[i].lock);
3244 INIT_HLIST_HEAD(&tcp_hashinfo.bhash[i].chain);
3245 }
3246
3247
3248 cnt = tcp_hashinfo.ehash_mask + 1;
3249
3250 tcp_death_row.sysctl_max_tw_buckets = cnt / 2;
3251 sysctl_tcp_max_orphans = cnt / 2;
3252 sysctl_max_syn_backlog = max(128, cnt / 256);
3253
3254 tcp_init_mem();
3255 /* Set per-socket limits to no more than 1/128 the pressure threshold */
3256 limit = nr_free_buffer_pages() << (PAGE_SHIFT - 7);
3257 max_wshare = min(4UL*1024*1024, limit);
3258 max_rshare = min(6UL*1024*1024, limit);
3259
3260 sysctl_tcp_wmem[0] = SK_MEM_QUANTUM;
3261 sysctl_tcp_wmem[1] = 16*1024;
3262 sysctl_tcp_wmem[2] = max(64*1024, max_wshare);
3263
3264 sysctl_tcp_rmem[0] = SK_MEM_QUANTUM;
3265 sysctl_tcp_rmem[1] = 87380;
3266 sysctl_tcp_rmem[2] = max(87380, max_rshare);
3267
3268 pr_info("Hash tables configured (established %u bind %u)\n",
3269 tcp_hashinfo.ehash_mask + 1, tcp_hashinfo.bhash_size);
3270
3271 tcp_metrics_init();
3272 BUG_ON(tcp_register_congestion_control(&tcp_reno) != 0);
3273 tcp_tasklet_init();
3274 }
3275