Lines Matching +full:tx +full:- +full:ts +full:- +full:routes
1 // SPDX-License-Identifier: GPL-2.0-or-later
12 * Corey Minyard <wf-rch!minyard@relay.EU.net>
13 * Florian La Roche, <flla@stud.uni-sb.de>
25 * sk->inuse=1 and was trying to connect
51 * Alan Cox : sk->keepopen now seems to work
53 * Alan Cox : Fixed assorted sk->rqueue->next errors
63 * Alan Cox : FIN with no memory -> CRASH
113 * Alan Cox : Changed the semantics of sk->socket to
164 * Matt Day : poll()->select() match BSD precisely on error
166 * Marc Tamsky : Various sk->prot->retransmits and
167 * sk->retransmits misupdating fixed.
172 * Alan Cox : Look up device on a retransmit - routes may
214 * waiting for final ack in three-way handshake.
341 val--; in tcp_enter_memory_pressure()
356 jiffies_to_msecs(jiffies - val)); in tcp_leave_memory_pressure()
387 while (--retrans) { in retrans_to_secs()
399 u32 rate = READ_ONCE(tp->rate_delivered); in tcp_compute_delivery_rate()
400 u32 intv = READ_ONCE(tp->rate_interval_us); in tcp_compute_delivery_rate()
404 rate64 = (u64)rate * tp->mss_cache * USEC_PER_SEC; in tcp_compute_delivery_rate()
410 /* Address-family independent initialization for a tcp_sock.
420 tp->out_of_order_queue = RB_ROOT; in tcp_init_sock()
421 sk->tcp_rtx_queue = RB_ROOT; in tcp_init_sock()
423 INIT_LIST_HEAD(&tp->tsq_node); in tcp_init_sock()
424 INIT_LIST_HEAD(&tp->tsorted_sent_queue); in tcp_init_sock()
426 icsk->icsk_rto = TCP_TIMEOUT_INIT; in tcp_init_sock()
427 icsk->icsk_rto_min = TCP_RTO_MIN; in tcp_init_sock()
428 icsk->icsk_delack_max = TCP_DELACK_MAX; in tcp_init_sock()
429 tp->mdev_us = jiffies_to_usecs(TCP_TIMEOUT_INIT); in tcp_init_sock()
430 minmax_reset(&tp->rtt_min, tcp_jiffies32, ~0U); in tcp_init_sock()
433 * initial SYN frame in their delayed-ACK and congestion control in tcp_init_sock()
435 * efficiently to them. -DaveM in tcp_init_sock()
437 tp->snd_cwnd = TCP_INIT_CWND; in tcp_init_sock()
440 tp->app_limited = ~0U; in tcp_init_sock()
441 tp->rate_app_limited = 1; in tcp_init_sock()
443 /* See draft-stevens-tcpca-spec-01 for discussion of the in tcp_init_sock()
446 tp->snd_ssthresh = TCP_INFINITE_SSTHRESH; in tcp_init_sock()
447 tp->snd_cwnd_clamp = ~0; in tcp_init_sock()
448 tp->mss_cache = TCP_MSS_DEFAULT; in tcp_init_sock()
450 tp->reordering = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_reordering); in tcp_init_sock()
453 tp->tsoffset = 0; in tcp_init_sock()
454 tp->rack.reo_wnd_steps = 1; in tcp_init_sock()
456 sk->sk_write_space = sk_stream_write_space; in tcp_init_sock()
459 icsk->icsk_sync_mss = tcp_sync_mss; in tcp_init_sock()
461 WRITE_ONCE(sk->sk_sndbuf, READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_wmem[1])); in tcp_init_sock()
462 WRITE_ONCE(sk->sk_rcvbuf, READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_rmem[1])); in tcp_init_sock()
465 sk->sk_route_forced_caps = NETIF_F_GSO; in tcp_init_sock()
467 icsk->nata_retries_enabled = 0; in tcp_init_sock()
468 icsk->nata_retries_type = NATA_NA; in tcp_init_sock()
469 icsk->nata_syn_rto = TCP_TIMEOUT_INIT; in tcp_init_sock()
470 icsk->nata_data_rto = TCP_TIMEOUT_INIT; in tcp_init_sock()
471 icsk->nata_data_retries = 0; in tcp_init_sock()
484 sock_tx_timestamp(sk, tsflags, &shinfo->tx_flags); in tcp_tx_timestamp()
486 tcb->txstamp_ack = 1; in tcp_tx_timestamp()
488 shinfo->tskey = TCP_SKB_CB(skb)->seq + skb->len - 1; in tcp_tx_timestamp()
495 int avail = READ_ONCE(tp->rcv_nxt) - READ_ONCE(tp->copied_seq); in tcp_stream_is_readable()
502 if (tcp_receive_window(tp) <= inet_csk(sk)->icsk_ack.rcv_mss) in tcp_stream_is_readable()
505 if (sk->sk_prot->stream_memory_read) in tcp_stream_is_readable()
506 return sk->sk_prot->stream_memory_read(sk); in tcp_stream_is_readable()
520 struct sock *sk = sock->sk; in tcp_poll()
549 * Check-me. in tcp_poll()
554 * in state CLOSE_WAIT. One solution is evident --- to set EPOLLHUP in tcp_poll()
560 * why EPOLLHUP is incompatible with EPOLLOUT. --ANK in tcp_poll()
563 * blocking on fresh not-connected or disconnected socket. --ANK in tcp_poll()
565 shutdown = READ_ONCE(sk->sk_shutdown); in tcp_poll()
573 (state != TCP_SYN_RECV || rcu_access_pointer(tp->fastopen_rsk))) { in tcp_poll()
576 if (READ_ONCE(tp->urg_seq) == READ_ONCE(tp->copied_seq) && in tcp_poll()
578 tp->urg_data) in tcp_poll()
589 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); in tcp_poll()
603 if (tp->urg_data & TCP_URG_VALID) in tcp_poll()
605 } else if (state == TCP_SYN_SENT && inet_sk(sk)->defer_connect) { in tcp_poll()
614 if (sk->sk_err || !skb_queue_empty_lockless(&sk->sk_error_queue)) in tcp_poll()
629 if (sk->sk_state == TCP_LISTEN) in tcp_ioctl()
630 return -EINVAL; in tcp_ioctl()
637 answ = tp->urg_data && in tcp_ioctl()
638 READ_ONCE(tp->urg_seq) == READ_ONCE(tp->copied_seq); in tcp_ioctl()
641 if (sk->sk_state == TCP_LISTEN) in tcp_ioctl()
642 return -EINVAL; in tcp_ioctl()
644 if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV)) in tcp_ioctl()
647 answ = READ_ONCE(tp->write_seq) - tp->snd_una; in tcp_ioctl()
650 if (sk->sk_state == TCP_LISTEN) in tcp_ioctl()
651 return -EINVAL; in tcp_ioctl()
653 if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV)) in tcp_ioctl()
656 answ = READ_ONCE(tp->write_seq) - in tcp_ioctl()
657 READ_ONCE(tp->snd_nxt); in tcp_ioctl()
660 return -ENOIOCTLCMD; in tcp_ioctl()
669 TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_PSH; in tcp_mark_push()
670 tp->pushed_seq = tp->write_seq; in tcp_mark_push()
675 return after(tp->write_seq, tp->pushed_seq + (tp->max_window >> 1)); in forced_push()
683 skb->csum = 0; in skb_entail()
684 tcb->seq = tcb->end_seq = tp->write_seq; in skb_entail()
685 tcb->tcp_flags = TCPHDR_ACK; in skb_entail()
686 tcb->sacked = 0; in skb_entail()
689 sk_wmem_queued_add(sk, skb->truesize); in skb_entail()
690 sk_mem_charge(sk, skb->truesize); in skb_entail()
691 if (tp->nonagle & TCP_NAGLE_PUSH) in skb_entail()
692 tp->nonagle &= ~TCP_NAGLE_PUSH; in skb_entail()
700 tp->snd_up = tp->write_seq; in tcp_mark_urg()
705 * Because TX completion will happen shortly, it gives a chance
711 * or if TX completion was delayed after we processed ACK packet.
716 return skb->len < size_goal && in tcp_should_autocork()
717 READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_autocorking) && in tcp_should_autocork()
719 refcount_read(&sk->sk_wmem_alloc) > skb->truesize; in tcp_should_autocork()
739 if (!test_bit(TSQ_THROTTLED, &sk->sk_tsq_flags)) { in tcp_push()
741 set_bit(TSQ_THROTTLED, &sk->sk_tsq_flags); in tcp_push()
744 /* It is possible TX completion already happened in tcp_push()
747 if (refcount_read(&sk->sk_wmem_alloc) > skb->truesize) in tcp_push()
760 struct tcp_splice_state *tss = rd_desc->arg.data; in tcp_splice_data_recv()
763 ret = skb_splice_bits(skb, skb->sk, offset, tss->pipe, in tcp_splice_data_recv()
764 min(rd_desc->count, len), tss->flags); in tcp_splice_data_recv()
766 rd_desc->count -= ret; in tcp_splice_data_recv()
775 .count = tss->len, in __tcp_splice_read()
782 * tcp_splice_read - splice data from TCP socket to a pipe
797 struct sock *sk = sock->sk; in tcp_splice_read()
812 return -ESPIPE; in tcp_splice_read()
818 timeo = sock_rcvtimeo(sk, sock->file->f_flags & O_NONBLOCK); in tcp_splice_read()
828 if (sk->sk_err) { in tcp_splice_read()
832 if (sk->sk_shutdown & RCV_SHUTDOWN) in tcp_splice_read()
834 if (sk->sk_state == TCP_CLOSE) { in tcp_splice_read()
839 ret = -ENOTCONN; in tcp_splice_read()
843 ret = -EAGAIN; in tcp_splice_read()
850 if (!skb_queue_empty(&sk->sk_receive_queue)) in tcp_splice_read()
859 tss.len -= ret; in tcp_splice_read()
867 if (sk->sk_err || sk->sk_state == TCP_CLOSE || in tcp_splice_read()
868 (sk->sk_shutdown & RCV_SHUTDOWN) || in tcp_splice_read()
888 skb = sk->sk_tx_skb_cache; in sk_stream_alloc_skb()
890 skb->truesize = SKB_TRUESIZE(skb_end_offset(skb)); in sk_stream_alloc_skb()
891 sk->sk_tx_skb_cache = NULL; in sk_stream_alloc_skb()
893 INIT_LIST_HEAD(&skb->tcp_tsorted_anchor); in sk_stream_alloc_skb()
894 skb_shinfo(skb)->tx_flags = 0; in sk_stream_alloc_skb()
899 /* The TCP header must be at least 32-bit aligned. */ in sk_stream_alloc_skb()
905 skb = alloc_skb_fclone(size + sk->sk_prot->max_header, gfp); in sk_stream_alloc_skb()
911 sk_forced_mem_schedule(sk, skb->truesize); in sk_stream_alloc_skb()
913 mem_scheduled = sk_wmem_schedule(sk, skb->truesize); in sk_stream_alloc_skb()
916 skb_reserve(skb, sk->sk_prot->max_header); in sk_stream_alloc_skb()
921 skb->reserved_tailroom = skb->end - skb->tail - size; in sk_stream_alloc_skb()
922 INIT_LIST_HEAD(&skb->tcp_tsorted_anchor); in sk_stream_alloc_skb()
927 sk->sk_prot->enter_memory_pressure(sk); in sk_stream_alloc_skb()
943 new_size_goal = sk->sk_gso_max_size - 1 - MAX_TCP_HEADER; in tcp_xmit_size_goal()
947 size_goal = tp->gso_segs * mss_now; in tcp_xmit_size_goal()
950 tp->gso_segs = min_t(u16, new_size_goal / mss_now, in tcp_xmit_size_goal()
951 sk->sk_gso_max_segs); in tcp_xmit_size_goal()
952 size_goal = tp->gso_segs * mss_now; in tcp_xmit_size_goal()
976 if (skb && TCP_SKB_CB(skb)->seq == TCP_SKB_CB(skb)->end_seq) { in tcp_remove_empty_skb()
996 return -EINVAL; in do_tcp_sendpages()
1002 if (((1 << sk->sk_state) & ~(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)) && in do_tcp_sendpages()
1014 err = -EPIPE; in do_tcp_sendpages()
1015 if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN)) in do_tcp_sendpages()
1023 if (!skb || (copy = size_goal - skb->len) <= 0 || in do_tcp_sendpages()
1029 skb = sk_stream_alloc_skb(sk, 0, sk->sk_allocation, in do_tcp_sendpages()
1035 skb->decrypted = !!(flags & MSG_SENDPAGE_DECRYPTED); in do_tcp_sendpages()
1044 i = skb_shinfo(skb)->nr_frags; in do_tcp_sendpages()
1054 skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy); in do_tcp_sendpages()
1061 skb_shinfo(skb)->tx_flags |= SKBTX_SHARED_FRAG; in do_tcp_sendpages()
1063 skb->len += copy; in do_tcp_sendpages()
1064 skb->data_len += copy; in do_tcp_sendpages()
1065 skb->truesize += copy; in do_tcp_sendpages()
1068 skb->ip_summed = CHECKSUM_PARTIAL; in do_tcp_sendpages()
1069 WRITE_ONCE(tp->write_seq, tp->write_seq + copy); in do_tcp_sendpages()
1070 TCP_SKB_CB(skb)->end_seq += copy; in do_tcp_sendpages()
1074 TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_PSH; in do_tcp_sendpages()
1078 size -= copy; in do_tcp_sendpages()
1082 if (skb->len < size_goal || (flags & MSG_OOB)) in do_tcp_sendpages()
1093 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); in do_tcp_sendpages()
1106 tcp_tx_timestamp(sk, sk->sk_tsflags); in do_tcp_sendpages()
1108 tcp_push(sk, flags, mss_now, tp->nonagle, size_goal); in do_tcp_sendpages()
1118 if (unlikely(tcp_rtx_and_write_queues_empty(sk) && err == -EAGAIN)) { in do_tcp_sendpages()
1119 sk->sk_write_space(sk); in do_tcp_sendpages()
1129 if (!(sk->sk_route_caps & NETIF_F_SG)) in tcp_sendpage_locked()
1132 tcp_rate_check_app_limited(sk); /* is sending application-limited? */ in tcp_sendpage_locked()
1153 if (tp->fastopen_req) { in tcp_free_fastopen_req()
1154 kfree(tp->fastopen_req); in tcp_free_fastopen_req()
1155 tp->fastopen_req = NULL; in tcp_free_fastopen_req()
1165 struct sockaddr *uaddr = msg->msg_name; in tcp_sendmsg_fastopen()
1168 if (!(READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_fastopen) & in tcp_sendmsg_fastopen()
1170 (uaddr && msg->msg_namelen >= sizeof(uaddr->sa_family) && in tcp_sendmsg_fastopen()
1171 uaddr->sa_family == AF_UNSPEC)) in tcp_sendmsg_fastopen()
1172 return -EOPNOTSUPP; in tcp_sendmsg_fastopen()
1173 if (tp->fastopen_req) in tcp_sendmsg_fastopen()
1174 return -EALREADY; /* Another Fast Open is in progress */ in tcp_sendmsg_fastopen()
1176 tp->fastopen_req = kzalloc(sizeof(struct tcp_fastopen_request), in tcp_sendmsg_fastopen()
1177 sk->sk_allocation); in tcp_sendmsg_fastopen()
1178 if (unlikely(!tp->fastopen_req)) in tcp_sendmsg_fastopen()
1179 return -ENOBUFS; in tcp_sendmsg_fastopen()
1180 tp->fastopen_req->data = msg; in tcp_sendmsg_fastopen()
1181 tp->fastopen_req->size = size; in tcp_sendmsg_fastopen()
1182 tp->fastopen_req->uarg = uarg; in tcp_sendmsg_fastopen()
1184 if (inet->defer_connect) { in tcp_sendmsg_fastopen()
1189 inet->inet_dport = 0; in tcp_sendmsg_fastopen()
1190 sk->sk_route_caps = 0; in tcp_sendmsg_fastopen()
1193 flags = (msg->msg_flags & MSG_DONTWAIT) ? O_NONBLOCK : 0; in tcp_sendmsg_fastopen()
1194 err = __inet_stream_connect(sk->sk_socket, uaddr, in tcp_sendmsg_fastopen()
1195 msg->msg_namelen, flags, 1); in tcp_sendmsg_fastopen()
1199 if (tp->fastopen_req) { in tcp_sendmsg_fastopen()
1200 *copied = tp->fastopen_req->copied; in tcp_sendmsg_fastopen()
1202 inet->defer_connect = 0; in tcp_sendmsg_fastopen()
1219 flags = msg->msg_flags; in tcp_sendmsg_locked()
1225 err = -ENOBUFS; in tcp_sendmsg_locked()
1229 zc = sk->sk_route_caps & NETIF_F_SG; in tcp_sendmsg_locked()
1231 uarg->zerocopy = 0; in tcp_sendmsg_locked()
1234 if (unlikely(flags & MSG_FASTOPEN || inet_sk(sk)->defer_connect) && in tcp_sendmsg_locked()
1235 !tp->repair) { in tcp_sendmsg_locked()
1237 if (err == -EINPROGRESS && copied_syn > 0) in tcp_sendmsg_locked()
1245 tcp_rate_check_app_limited(sk); /* is sending application-limited? */ in tcp_sendmsg_locked()
1251 if (((1 << sk->sk_state) & ~(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)) && in tcp_sendmsg_locked()
1258 if (unlikely(tp->repair)) { in tcp_sendmsg_locked()
1259 if (tp->repair_queue == TCP_RECV_QUEUE) { in tcp_sendmsg_locked()
1264 err = -EINVAL; in tcp_sendmsg_locked()
1265 if (tp->repair_queue == TCP_NO_QUEUE) in tcp_sendmsg_locked()
1272 if (msg->msg_controllen) { in tcp_sendmsg_locked()
1275 err = -EINVAL; in tcp_sendmsg_locked()
1289 err = -EPIPE; in tcp_sendmsg_locked()
1290 if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN)) in tcp_sendmsg_locked()
1298 copy = size_goal - skb->len; in tcp_sendmsg_locked()
1313 skb = sk_stream_alloc_skb(sk, 0, sk->sk_allocation, in tcp_sendmsg_locked()
1319 skb->ip_summed = CHECKSUM_PARTIAL; in tcp_sendmsg_locked()
1328 if (tp->repair) in tcp_sendmsg_locked()
1329 TCP_SKB_CB(skb)->sacked |= TCPCB_REPAIRED; in tcp_sendmsg_locked()
1340 err = skb_add_data_nocache(sk, skb, &msg->msg_iter, copy); in tcp_sendmsg_locked()
1345 int i = skb_shinfo(skb)->nr_frags; in tcp_sendmsg_locked()
1351 if (!skb_can_coalesce(skb, i, pfrag->page, in tcp_sendmsg_locked()
1352 pfrag->offset)) { in tcp_sendmsg_locked()
1360 copy = min_t(int, copy, pfrag->size - pfrag->offset); in tcp_sendmsg_locked()
1365 err = skb_copy_to_page_nocache(sk, &msg->msg_iter, skb, in tcp_sendmsg_locked()
1366 pfrag->page, in tcp_sendmsg_locked()
1367 pfrag->offset, in tcp_sendmsg_locked()
1374 skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy); in tcp_sendmsg_locked()
1376 skb_fill_page_desc(skb, i, pfrag->page, in tcp_sendmsg_locked()
1377 pfrag->offset, copy); in tcp_sendmsg_locked()
1378 page_ref_inc(pfrag->page); in tcp_sendmsg_locked()
1380 pfrag->offset += copy; in tcp_sendmsg_locked()
1386 if (err == -EMSGSIZE || err == -EEXIST) { in tcp_sendmsg_locked()
1396 TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_PSH; in tcp_sendmsg_locked()
1398 WRITE_ONCE(tp->write_seq, tp->write_seq + copy); in tcp_sendmsg_locked()
1399 TCP_SKB_CB(skb)->end_seq += copy; in tcp_sendmsg_locked()
1405 TCP_SKB_CB(skb)->eor = 1; in tcp_sendmsg_locked()
1409 if (skb->len < size_goal || (flags & MSG_OOB) || unlikely(tp->repair)) in tcp_sendmsg_locked()
1420 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); in tcp_sendmsg_locked()
1435 tcp_push(sk, flags, mss_now, tp->nonagle, size_goal); in tcp_sendmsg_locked()
1452 if (unlikely(tcp_rtx_and_write_queues_empty(sk) && err == -EAGAIN)) { in tcp_sendmsg_locked()
1453 sk->sk_write_space(sk); in tcp_sendmsg_locked()
1482 if (sock_flag(sk, SOCK_URGINLINE) || !tp->urg_data || in tcp_recv_urg()
1483 tp->urg_data == TCP_URG_READ) in tcp_recv_urg()
1484 return -EINVAL; /* Yes this is right ! */ in tcp_recv_urg()
1486 if (sk->sk_state == TCP_CLOSE && !sock_flag(sk, SOCK_DONE)) in tcp_recv_urg()
1487 return -ENOTCONN; in tcp_recv_urg()
1489 if (tp->urg_data & TCP_URG_VALID) { in tcp_recv_urg()
1491 char c = tp->urg_data; in tcp_recv_urg()
1494 tp->urg_data = TCP_URG_READ; in tcp_recv_urg()
1497 msg->msg_flags |= MSG_OOB; in tcp_recv_urg()
1504 msg->msg_flags |= MSG_TRUNC; in tcp_recv_urg()
1506 return err ? -EFAULT : len; in tcp_recv_urg()
1509 if (sk->sk_state == TCP_CLOSE || (sk->sk_shutdown & RCV_SHUTDOWN)) in tcp_recv_urg()
1516 * Mike <pall@rz.uni-karlsruhe.de> in tcp_recv_urg()
1518 return -EAGAIN; in tcp_recv_urg()
1526 /* XXX -- need to support SO_PEEK_OFF */ in tcp_peek_sndq()
1528 skb_rbtree_walk(skb, &sk->tcp_rtx_queue) { in tcp_peek_sndq()
1529 err = skb_copy_datagram_msg(skb, 0, msg, skb->len); in tcp_peek_sndq()
1532 copied += skb->len; in tcp_peek_sndq()
1535 skb_queue_walk(&sk->sk_write_queue, skb) { in tcp_peek_sndq()
1536 err = skb_copy_datagram_msg(skb, 0, msg, skb->len); in tcp_peek_sndq()
1540 copied += skb->len; in tcp_peek_sndq()
1557 struct sk_buff *skb = skb_peek(&sk->sk_receive_queue); in tcp_cleanup_rbuf()
1559 WARN(skb && !before(tp->copied_seq, TCP_SKB_CB(skb)->end_seq), in tcp_cleanup_rbuf()
1561 tp->copied_seq, TCP_SKB_CB(skb)->end_seq, tp->rcv_nxt); in tcp_cleanup_rbuf()
1565 __u16 rcv_mss = icsk->icsk_ack.rcv_mss; in tcp_cleanup_rbuf()
1570 if (/* Once-per-two-segments ACK was not sent by tcp_input.c */ in tcp_cleanup_rbuf()
1571 tp->rcv_nxt - tp->rcv_wup > rcv_mss || in tcp_cleanup_rbuf()
1579 ((icsk->icsk_ack.pending & ICSK_ACK_PUSHED2) || in tcp_cleanup_rbuf()
1580 ((icsk->icsk_ack.pending & ICSK_ACK_PUSHED) && in tcp_cleanup_rbuf()
1582 !atomic_read(&sk->sk_rmem_alloc))) in tcp_cleanup_rbuf()
1586 /* We send an ACK if we can now advertise a non-zero window in tcp_cleanup_rbuf()
1592 if (copied > 0 && !time_to_ack && !(sk->sk_shutdown & RCV_SHUTDOWN)) { in tcp_cleanup_rbuf()
1596 if (2*rcv_window_now <= tp->window_clamp) { in tcp_cleanup_rbuf()
1617 while ((skb = skb_peek(&sk->sk_receive_queue)) != NULL) { in tcp_recv_skb()
1618 offset = seq - TCP_SKB_CB(skb)->seq; in tcp_recv_skb()
1619 if (unlikely(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)) { in tcp_recv_skb()
1621 offset--; in tcp_recv_skb()
1623 if (offset < skb->len || (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)) { in tcp_recv_skb()
1641 * - It is assumed that the socket was locked by the caller.
1642 * - The routine does not block.
1643 * - At present, there is no support for reading OOB data
1652 u32 seq = tp->copied_seq; in tcp_read_sock()
1656 if (sk->sk_state == TCP_LISTEN) in tcp_read_sock()
1657 return -ENOTCONN; in tcp_read_sock()
1659 if (offset < skb->len) { in tcp_read_sock()
1663 len = skb->len - offset; in tcp_read_sock()
1665 if (tp->urg_data) { in tcp_read_sock()
1666 u32 urg_offset = tp->urg_seq - seq; in tcp_read_sock()
1689 skb = tcp_recv_skb(sk, seq - 1, &offset); in tcp_read_sock()
1695 if (offset + 1 != skb->len) in tcp_read_sock()
1698 if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) { in tcp_read_sock()
1704 if (!desc->count) in tcp_read_sock()
1706 WRITE_ONCE(tp->copied_seq, seq); in tcp_read_sock()
1708 WRITE_ONCE(tp->copied_seq, seq); in tcp_read_sock()
1723 return tcp_inq(sock->sk); in tcp_peek_len()
1732 if (sk->sk_userlocks & SOCK_RCVBUF_LOCK) in tcp_set_rcvlowat()
1733 cap = sk->sk_rcvbuf >> 1; in tcp_set_rcvlowat()
1735 cap = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_rmem[2]) >> 1; in tcp_set_rcvlowat()
1737 WRITE_ONCE(sk->sk_rcvlowat, val ? : 1); in tcp_set_rcvlowat()
1742 if (sk->sk_userlocks & SOCK_RCVBUF_LOCK) in tcp_set_rcvlowat()
1746 if (val > sk->sk_rcvbuf) { in tcp_set_rcvlowat()
1747 WRITE_ONCE(sk->sk_rcvbuf, val); in tcp_set_rcvlowat()
1748 tcp_sk(sk)->window_clamp = tcp_win_from_space(sk, val); in tcp_set_rcvlowat()
1761 if (vma->vm_flags & (VM_WRITE | VM_EXEC)) in tcp_mmap()
1762 return -EPERM; in tcp_mmap()
1763 vma->vm_flags &= ~(VM_MAYWRITE | VM_MAYEXEC); in tcp_mmap()
1766 vma->vm_flags |= VM_MIXEDMAP; in tcp_mmap()
1768 vma->vm_ops = &tcp_vm_ops; in tcp_mmap()
1778 if (unlikely(offset_skb >= skb->len)) in skb_advance_to_frag()
1781 offset_skb -= skb_headlen(skb); in skb_advance_to_frag()
1785 frag = skb_shinfo(skb)->frags; in skb_advance_to_frag()
1791 offset_skb -= skb_frag_size(frag); in skb_advance_to_frag()
1807 if (PageCompound(page) || page->mapping) in can_map_frag()
1832 unsigned long copy_address = (unsigned long)zc->copybuf_address; in tcp_copy_straggler_data()
1837 if (copy_address != zc->copybuf_address) in tcp_copy_straggler_data()
1838 return -EINVAL; in tcp_copy_straggler_data()
1847 zc->recv_skip_hint -= copylen; in tcp_copy_straggler_data()
1859 u32 offset, copylen = min_t(u32, copybuf_len, zc->recv_skip_hint); in tcp_zerocopy_handle_leftover_data()
1865 offset = *seq - TCP_SKB_CB(skb)->seq; in tcp_zerocopy_handle_leftover_data()
1869 zc->copybuf_len = tcp_copy_straggler_data(zc, skb, copylen, &offset, in tcp_zerocopy_handle_leftover_data()
1871 return zc->copybuf_len < 0 ? 0 : copylen; in tcp_zerocopy_handle_leftover_data()
1887 bytes_mapped = PAGE_SIZE * (pages_to_map - pages_remaining); in tcp_zerocopy_vm_insert_batch()
1898 *length_with_pending -= bytes_not_mapped; in tcp_zerocopy_vm_insert_batch()
1899 zc->recv_skip_hint += bytes_not_mapped; in tcp_zerocopy_vm_insert_batch()
1908 unsigned long address = (unsigned long)zc->address; in tcp_zerocopy_receive()
1909 s32 copybuf_len = zc->copybuf_len; in tcp_zerocopy_receive()
1918 u32 seq = tp->copied_seq; in tcp_zerocopy_receive()
1922 zc->copybuf_len = 0; in tcp_zerocopy_receive()
1924 if (address & (PAGE_SIZE - 1) || address != zc->address) in tcp_zerocopy_receive()
1925 return -EINVAL; in tcp_zerocopy_receive()
1927 if (sk->sk_state == TCP_LISTEN) in tcp_zerocopy_receive()
1928 return -ENOTCONN; in tcp_zerocopy_receive()
1932 mmap_read_lock(current->mm); in tcp_zerocopy_receive()
1934 vma = find_vma(current->mm, address); in tcp_zerocopy_receive()
1935 if (!vma || vma->vm_start > address || vma->vm_ops != &tcp_vm_ops) { in tcp_zerocopy_receive()
1936 mmap_read_unlock(current->mm); in tcp_zerocopy_receive()
1937 return -EINVAL; in tcp_zerocopy_receive()
1939 vma_len = min_t(unsigned long, zc->length, vma->vm_end - address); in tcp_zerocopy_receive()
1941 aligned_len = avail_len & ~(PAGE_SIZE - 1); in tcp_zerocopy_receive()
1944 zc->length = aligned_len; in tcp_zerocopy_receive()
1945 zc->recv_skip_hint = 0; in tcp_zerocopy_receive()
1947 zc->length = avail_len; in tcp_zerocopy_receive()
1948 zc->recv_skip_hint = avail_len; in tcp_zerocopy_receive()
1952 while (length + PAGE_SIZE <= zc->length) { in tcp_zerocopy_receive()
1955 if (zc->recv_skip_hint < PAGE_SIZE) { in tcp_zerocopy_receive()
1970 if (zc->recv_skip_hint > 0) in tcp_zerocopy_receive()
1972 skb = skb->next; in tcp_zerocopy_receive()
1973 offset = seq - TCP_SKB_CB(skb)->seq; in tcp_zerocopy_receive()
1977 zc->recv_skip_hint = skb->len - offset; in tcp_zerocopy_receive()
1984 zc->recv_skip_hint); in tcp_zerocopy_receive()
1986 zc->recv_skip_hint = mappable_offset; in tcp_zerocopy_receive()
1992 zc->recv_skip_hint -= PAGE_SIZE; in tcp_zerocopy_receive()
2009 mmap_read_unlock(current->mm); in tcp_zerocopy_receive()
2016 WRITE_ONCE(tp->copied_seq, seq); in tcp_zerocopy_receive()
2023 if (length == zc->length) in tcp_zerocopy_receive()
2024 zc->recv_skip_hint = 0; in tcp_zerocopy_receive()
2026 if (!zc->recv_skip_hint && sock_flag(sk, SOCK_DONE)) in tcp_zerocopy_receive()
2027 ret = -EIO; in tcp_zerocopy_receive()
2029 zc->length = length; in tcp_zerocopy_receive()
2037 if (skb->tstamp) in tcp_update_recv_tstamps()
2038 tss->ts[0] = ktime_to_timespec64(skb->tstamp); in tcp_update_recv_tstamps()
2040 tss->ts[0] = (struct timespec64) {0}; in tcp_update_recv_tstamps()
2042 if (skb_hwtstamps(skb)->hwtstamp) in tcp_update_recv_tstamps()
2043 tss->ts[2] = ktime_to_timespec64(skb_hwtstamps(skb)->hwtstamp); in tcp_update_recv_tstamps()
2045 tss->ts[2] = (struct timespec64) {0}; in tcp_update_recv_tstamps()
2055 if (tss->ts[0].tv_sec || tss->ts[0].tv_nsec) { in tcp_recv_timestamp()
2060 .tv_sec = tss->ts[0].tv_sec, in tcp_recv_timestamp()
2061 .tv_nsec = tss->ts[0].tv_nsec, in tcp_recv_timestamp()
2067 .tv_sec = tss->ts[0].tv_sec, in tcp_recv_timestamp()
2068 .tv_nsec = tss->ts[0].tv_nsec, in tcp_recv_timestamp()
2076 .tv_sec = tss->ts[0].tv_sec, in tcp_recv_timestamp()
2077 .tv_usec = tss->ts[0].tv_nsec / 1000, in tcp_recv_timestamp()
2083 .tv_sec = tss->ts[0].tv_sec, in tcp_recv_timestamp()
2084 .tv_usec = tss->ts[0].tv_nsec / 1000, in tcp_recv_timestamp()
2092 if (sk->sk_tsflags & SOF_TIMESTAMPING_SOFTWARE) in tcp_recv_timestamp()
2095 tss->ts[0] = (struct timespec64) {0}; in tcp_recv_timestamp()
2098 if (tss->ts[2].tv_sec || tss->ts[2].tv_nsec) { in tcp_recv_timestamp()
2099 if (sk->sk_tsflags & SOF_TIMESTAMPING_RAW_HARDWARE) in tcp_recv_timestamp()
2102 tss->ts[2] = (struct timespec64) {0}; in tcp_recv_timestamp()
2106 tss->ts[1] = (struct timespec64) {0}; in tcp_recv_timestamp()
2117 u32 copied_seq = READ_ONCE(tp->copied_seq); in tcp_inq_hint()
2118 u32 rcv_nxt = READ_ONCE(tp->rcv_nxt); in tcp_inq_hint()
2121 inq = rcv_nxt - copied_seq; in tcp_inq_hint()
2122 if (unlikely(inq < 0 || copied_seq != READ_ONCE(tp->copied_seq))) { in tcp_inq_hint()
2124 inq = tp->rcv_nxt - tp->copied_seq; in tcp_inq_hint()
2127 /* After receiving a FIN, tell the user-space to continue reading in tcp_inq_hint()
2128 * by returning a non-zero inq. in tcp_inq_hint()
2139 * tricks with *seq access order and skb->users are not required.
2162 if (sk_can_busy_loop(sk) && skb_queue_empty_lockless(&sk->sk_receive_queue) && in tcp_recvmsg()
2163 (sk->sk_state == TCP_ESTABLISHED)) in tcp_recvmsg()
2168 err = -ENOTCONN; in tcp_recvmsg()
2169 if (sk->sk_state == TCP_LISTEN) in tcp_recvmsg()
2172 cmsg_flags = tp->recvmsg_inq ? 1 : 0; in tcp_recvmsg()
2179 if (unlikely(tp->repair)) { in tcp_recvmsg()
2180 err = -EPERM; in tcp_recvmsg()
2184 if (tp->repair_queue == TCP_SEND_QUEUE) in tcp_recvmsg()
2187 err = -EINVAL; in tcp_recvmsg()
2188 if (tp->repair_queue == TCP_NO_QUEUE) in tcp_recvmsg()
2191 /* 'common' recv queue MSG_PEEK-ing */ in tcp_recvmsg()
2194 seq = &tp->copied_seq; in tcp_recvmsg()
2196 peek_seq = tp->copied_seq; in tcp_recvmsg()
2206 if (tp->urg_data && tp->urg_seq == *seq) { in tcp_recvmsg()
2210 copied = timeo ? sock_intr_errno(timeo) : -EAGAIN; in tcp_recvmsg()
2217 last = skb_peek_tail(&sk->sk_receive_queue); in tcp_recvmsg()
2218 skb_queue_walk(&sk->sk_receive_queue, skb) { in tcp_recvmsg()
2223 if (WARN(before(*seq, TCP_SKB_CB(skb)->seq), in tcp_recvmsg()
2225 *seq, TCP_SKB_CB(skb)->seq, tp->rcv_nxt, in tcp_recvmsg()
2229 offset = *seq - TCP_SKB_CB(skb)->seq; in tcp_recvmsg()
2230 if (unlikely(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)) { in tcp_recvmsg()
2232 offset--; in tcp_recvmsg()
2234 if (offset < skb->len) in tcp_recvmsg()
2236 if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) in tcp_recvmsg()
2240 *seq, TCP_SKB_CB(skb)->seq, tp->rcv_nxt, flags); in tcp_recvmsg()
2245 if (copied >= target && !READ_ONCE(sk->sk_backlog.tail)) in tcp_recvmsg()
2249 if (sk->sk_err || in tcp_recvmsg()
2250 sk->sk_state == TCP_CLOSE || in tcp_recvmsg()
2251 (sk->sk_shutdown & RCV_SHUTDOWN) || in tcp_recvmsg()
2259 if (sk->sk_err) { in tcp_recvmsg()
2264 if (sk->sk_shutdown & RCV_SHUTDOWN) in tcp_recvmsg()
2267 if (sk->sk_state == TCP_CLOSE) { in tcp_recvmsg()
2271 copied = -ENOTCONN; in tcp_recvmsg()
2276 copied = -EAGAIN; in tcp_recvmsg()
2297 (peek_seq - copied - urg_hole != tp->copied_seq)) { in tcp_recvmsg()
2299 current->comm, in tcp_recvmsg()
2301 peek_seq = tp->copied_seq; in tcp_recvmsg()
2307 used = skb->len - offset; in tcp_recvmsg()
2312 if (tp->urg_data) { in tcp_recvmsg()
2313 u32 urg_offset = tp->urg_seq - *seq; in tcp_recvmsg()
2320 used--; in tcp_recvmsg()
2334 copied = -EFAULT; in tcp_recvmsg()
2341 len -= used; in tcp_recvmsg()
2346 if (tp->urg_data && after(tp->copied_seq, tp->urg_seq)) { in tcp_recvmsg()
2347 tp->urg_data = 0; in tcp_recvmsg()
2351 if (TCP_SKB_CB(skb)->has_rxtstamp) { in tcp_recvmsg()
2356 if (used + offset < skb->len) in tcp_recvmsg()
2359 if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) in tcp_recvmsg()
2374 * on connected socket. I was just happy when found this 8) --ANK in tcp_recvmsg()
2409 int oldstate = sk->sk_state; in tcp_set_state()
2445 sk->sk_prot->unhash(sk); in tcp_set_state()
2446 if (inet_csk(sk)->icsk_bind_hash && in tcp_set_state()
2447 !(sk->sk_userlocks & SOCK_BINDPORT_LOCK)) in tcp_set_state()
2488 int next = (int)new_state[sk->sk_state]; in tcp_close_state()
2511 if ((1 << sk->sk_state) & in tcp_shutdown()
2557 net_info_ratelimited("out of memory -- consider tuning tcp_mem\n"); in tcp_check_oom()
2567 WRITE_ONCE(sk->sk_shutdown, SHUTDOWN_MASK); in __tcp_close()
2569 if (sk->sk_state == TCP_LISTEN) { in __tcp_close()
2579 * descriptor close, not protocol-sourced closes, because the in __tcp_close()
2582 while ((skb = __skb_dequeue(&sk->sk_receive_queue)) != NULL) { in __tcp_close()
2583 u32 len = TCP_SKB_CB(skb)->end_seq - TCP_SKB_CB(skb)->seq; in __tcp_close()
2585 if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) in __tcp_close()
2586 len--; in __tcp_close()
2593 /* If socket has been already reset (e.g. in tcp_reset()) - kill it. */ in __tcp_close()
2594 if (sk->sk_state == TCP_CLOSE) in __tcp_close()
2601 * advertise a zero window, then kill -9 the FTP client, wheee... in __tcp_close()
2604 if (unlikely(tcp_sk(sk)->repair)) { in __tcp_close()
2605 sk->sk_prot->disconnect(sk, 0); in __tcp_close()
2610 tcp_send_active_reset(sk, sk->sk_allocation); in __tcp_close()
2611 } else if (sock_flag(sk, SOCK_LINGER) && !sk->sk_lingertime) { in __tcp_close()
2613 sk->sk_prot->disconnect(sk, 0); in __tcp_close()
2620 /* RED-PEN. Formally speaking, we have broken TCP state in __tcp_close()
2623 * TCP_ESTABLISHED -> TCP_FIN_WAIT1 in __tcp_close()
2624 * TCP_SYN_RECV -> TCP_FIN_WAIT1 (it is difficult) in __tcp_close()
2625 * TCP_CLOSE_WAIT -> TCP_LAST_ACK in __tcp_close()
2631 * if Linux state is FIN-WAIT-1, but FIN is still not sent. in __tcp_close()
2634 * we enter time-wait state, when it is not required really in __tcp_close()
2639 * --ANK in __tcp_close()
2640 * XXX (TFO) - To start off we don't support SYN+ACK+FIN in __tcp_close()
2642 * probably need API support or TCP_CORK SYN-ACK until in __tcp_close()
2651 state = sk->sk_state; in __tcp_close()
2663 if (state != TCP_CLOSE && sk->sk_state == TCP_CLOSE) in __tcp_close()
2670 * our end. If they send after that then tough - BUT: long enough in __tcp_close()
2671 * that we won't make the old 4*rto = almost no time - whoops in __tcp_close()
2677 * linger2 option. --ANK in __tcp_close()
2680 if (sk->sk_state == TCP_FIN_WAIT2) { in __tcp_close()
2682 if (tp->linger2 < 0) { in __tcp_close()
2692 tmo - TCP_TIMEWAIT_LEN); in __tcp_close()
2699 if (sk->sk_state != TCP_CLOSE) { in __tcp_close()
2712 if (sk->sk_state == TCP_CLOSE) { in __tcp_close()
2715 req = rcu_dereference_protected(tcp_sk(sk)->fastopen_rsk, in __tcp_close()
2717 /* We could get here with a non-NULL req if the socket is in __tcp_close()
2737 if (!sk->sk_net_refcnt) in tcp_close()
2754 struct rb_node *p = rb_first(&sk->tcp_rtx_queue); in tcp_rtx_queue_purge()
2756 tcp_sk(sk)->highest_sack = NULL; in tcp_rtx_queue_purge()
2762 * list_del(&skb->tcp_tsorted_anchor) in tcp_rtx_queue_purge()
2774 while ((skb = __skb_dequeue(&sk->sk_write_queue)) != NULL) { in tcp_write_queue_purge()
2779 skb = sk->sk_tx_skb_cache; in tcp_write_queue_purge()
2782 sk->sk_tx_skb_cache = NULL; in tcp_write_queue_purge()
2784 INIT_LIST_HEAD(&tcp_sk(sk)->tsorted_sent_queue); in tcp_write_queue_purge()
2787 tcp_sk(sk)->packets_out = 0; in tcp_write_queue_purge()
2788 inet_csk(sk)->icsk_backoff = 0; in tcp_write_queue_purge()
2796 int old_state = sk->sk_state; in tcp_disconnect()
2802 if (sk->sk_wait_pending) in tcp_disconnect()
2803 return -EBUSY; in tcp_disconnect()
2811 } else if (unlikely(tp->repair)) { in tcp_disconnect()
2812 sk->sk_err = ECONNABORTED; in tcp_disconnect()
2814 (tp->snd_nxt != tp->write_seq && in tcp_disconnect()
2820 sk->sk_err = ECONNRESET; in tcp_disconnect()
2822 sk->sk_err = ECONNRESET; in tcp_disconnect()
2825 __skb_queue_purge(&sk->sk_receive_queue); in tcp_disconnect()
2826 if (sk->sk_rx_skb_cache) { in tcp_disconnect()
2827 __kfree_skb(sk->sk_rx_skb_cache); in tcp_disconnect()
2828 sk->sk_rx_skb_cache = NULL; in tcp_disconnect()
2830 WRITE_ONCE(tp->copied_seq, tp->rcv_nxt); in tcp_disconnect()
2831 tp->urg_data = 0; in tcp_disconnect()
2834 skb_rbtree_purge(&tp->out_of_order_queue); in tcp_disconnect()
2836 inet->inet_dport = 0; in tcp_disconnect()
2838 if (!(sk->sk_userlocks & SOCK_BINDADDR_LOCK)) in tcp_disconnect()
2841 WRITE_ONCE(sk->sk_shutdown, 0); in tcp_disconnect()
2843 tp->srtt_us = 0; in tcp_disconnect()
2844 tp->mdev_us = jiffies_to_usecs(TCP_TIMEOUT_INIT); in tcp_disconnect()
2845 tp->rcv_rtt_last_tsecr = 0; in tcp_disconnect()
2847 seq = tp->write_seq + tp->max_window + 2; in tcp_disconnect()
2850 WRITE_ONCE(tp->write_seq, seq); in tcp_disconnect()
2852 icsk->icsk_backoff = 0; in tcp_disconnect()
2853 icsk->icsk_probes_out = 0; in tcp_disconnect()
2854 icsk->icsk_probes_tstamp = 0; in tcp_disconnect()
2855 icsk->icsk_rto = TCP_TIMEOUT_INIT; in tcp_disconnect()
2856 icsk->icsk_rto_min = TCP_RTO_MIN; in tcp_disconnect()
2857 icsk->icsk_delack_max = TCP_DELACK_MAX; in tcp_disconnect()
2859 icsk->nata_retries_enabled = 0; in tcp_disconnect()
2860 icsk->nata_retries_type = NATA_NA; in tcp_disconnect()
2861 icsk->nata_syn_rto = TCP_TIMEOUT_INIT; in tcp_disconnect()
2862 icsk->nata_data_rto = TCP_TIMEOUT_INIT; in tcp_disconnect()
2863 icsk->nata_data_retries = 0; in tcp_disconnect()
2865 tp->snd_ssthresh = TCP_INFINITE_SSTHRESH; in tcp_disconnect()
2866 tp->snd_cwnd = TCP_INIT_CWND; in tcp_disconnect()
2867 tp->snd_cwnd_cnt = 0; in tcp_disconnect()
2868 tp->is_cwnd_limited = 0; in tcp_disconnect()
2869 tp->max_packets_out = 0; in tcp_disconnect()
2870 tp->window_clamp = 0; in tcp_disconnect()
2871 tp->delivered = 0; in tcp_disconnect()
2872 tp->delivered_ce = 0; in tcp_disconnect()
2873 if (icsk->icsk_ca_ops->release) in tcp_disconnect()
2874 icsk->icsk_ca_ops->release(sk); in tcp_disconnect()
2875 memset(icsk->icsk_ca_priv, 0, sizeof(icsk->icsk_ca_priv)); in tcp_disconnect()
2876 icsk->icsk_ca_initialized = 0; in tcp_disconnect()
2878 tp->is_sack_reneg = 0; in tcp_disconnect()
2880 tp->total_retrans = 0; in tcp_disconnect()
2885 icsk->icsk_ack.rcv_mss = TCP_MIN_MSS; in tcp_disconnect()
2886 memset(&tp->rx_opt, 0, sizeof(tp->rx_opt)); in tcp_disconnect()
2888 dst_release(xchg((__force struct dst_entry **)&sk->sk_rx_dst, NULL)); in tcp_disconnect()
2890 tp->compressed_ack = 0; in tcp_disconnect()
2891 tp->segs_in = 0; in tcp_disconnect()
2892 tp->segs_out = 0; in tcp_disconnect()
2893 tp->bytes_sent = 0; in tcp_disconnect()
2894 tp->bytes_acked = 0; in tcp_disconnect()
2895 tp->bytes_received = 0; in tcp_disconnect()
2896 tp->bytes_retrans = 0; in tcp_disconnect()
2897 tp->data_segs_in = 0; in tcp_disconnect()
2898 tp->data_segs_out = 0; in tcp_disconnect()
2899 tp->duplicate_sack[0].start_seq = 0; in tcp_disconnect()
2900 tp->duplicate_sack[0].end_seq = 0; in tcp_disconnect()
2901 tp->dsack_dups = 0; in tcp_disconnect()
2902 tp->reord_seen = 0; in tcp_disconnect()
2903 tp->retrans_out = 0; in tcp_disconnect()
2904 tp->sacked_out = 0; in tcp_disconnect()
2905 tp->tlp_high_seq = 0; in tcp_disconnect()
2906 tp->last_oow_ack_time = 0; in tcp_disconnect()
2908 tp->app_limited = ~0U; in tcp_disconnect()
2909 tp->rate_app_limited = 1; in tcp_disconnect()
2910 tp->rack.mstamp = 0; in tcp_disconnect()
2911 tp->rack.advanced = 0; in tcp_disconnect()
2912 tp->rack.reo_wnd_steps = 1; in tcp_disconnect()
2913 tp->rack.last_delivered = 0; in tcp_disconnect()
2914 tp->rack.reo_wnd_persist = 0; in tcp_disconnect()
2915 tp->rack.dsack_seen = 0; in tcp_disconnect()
2916 tp->syn_data_acked = 0; in tcp_disconnect()
2917 tp->rx_opt.saw_tstamp = 0; in tcp_disconnect()
2918 tp->rx_opt.dsack = 0; in tcp_disconnect()
2919 tp->rx_opt.num_sacks = 0; in tcp_disconnect()
2920 tp->rcv_ooopack = 0; in tcp_disconnect()
2925 inet->defer_connect = 0; in tcp_disconnect()
2926 tp->fastopen_client_fail = 0; in tcp_disconnect()
2928 WARN_ON(inet->inet_num && !icsk->icsk_bind_hash); in tcp_disconnect()
2930 if (sk->sk_frag.page) { in tcp_disconnect()
2931 put_page(sk->sk_frag.page); in tcp_disconnect()
2932 sk->sk_frag.page = NULL; in tcp_disconnect()
2933 sk->sk_frag.offset = 0; in tcp_disconnect()
2936 sk->sk_error_report(sk); in tcp_disconnect()
2943 return ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN) && in tcp_can_repair_sock()
2944 (sk->sk_state != TCP_LISTEN); in tcp_can_repair_sock()
2951 if (!tp->repair) in tcp_repair_set_window()
2952 return -EPERM; in tcp_repair_set_window()
2955 return -EINVAL; in tcp_repair_set_window()
2958 return -EFAULT; in tcp_repair_set_window()
2961 return -EINVAL; in tcp_repair_set_window()
2963 if (after(opt.snd_wl1, tp->rcv_nxt + opt.rcv_wnd)) in tcp_repair_set_window()
2964 return -EINVAL; in tcp_repair_set_window()
2966 if (after(opt.rcv_wup, tp->rcv_nxt)) in tcp_repair_set_window()
2967 return -EINVAL; in tcp_repair_set_window()
2969 tp->snd_wl1 = opt.snd_wl1; in tcp_repair_set_window()
2970 tp->snd_wnd = opt.snd_wnd; in tcp_repair_set_window()
2971 tp->max_window = opt.max_window; in tcp_repair_set_window()
2973 tp->rcv_wnd = opt.rcv_wnd; in tcp_repair_set_window()
2974 tp->rcv_wup = opt.rcv_wup; in tcp_repair_set_window()
2988 return -EFAULT; in tcp_repair_options_est()
2991 len -= sizeof(opt); in tcp_repair_options_est()
2995 tp->rx_opt.mss_clamp = opt.opt_val; in tcp_repair_options_est()
3004 return -EFBIG; in tcp_repair_options_est()
3006 tp->rx_opt.snd_wscale = snd_wscale; in tcp_repair_options_est()
3007 tp->rx_opt.rcv_wscale = rcv_wscale; in tcp_repair_options_est()
3008 tp->rx_opt.wscale_ok = 1; in tcp_repair_options_est()
3013 return -EINVAL; in tcp_repair_options_est()
3015 tp->rx_opt.sack_ok |= TCP_SACK_SEEN; in tcp_repair_options_est()
3019 return -EINVAL; in tcp_repair_options_est()
3021 tp->rx_opt.tstamp_ok = 1; in tcp_repair_options_est()
3044 /* When set indicates to always queue non-full frames. Later the user clears
3058 tp->nonagle |= TCP_NAGLE_CORK; in __tcp_sock_set_cork()
3060 tp->nonagle &= ~TCP_NAGLE_CORK; in __tcp_sock_set_cork()
3061 if (tp->nonagle & TCP_NAGLE_OFF) in __tcp_sock_set_cork()
3062 tp->nonagle |= TCP_NAGLE_PUSH; in __tcp_sock_set_cork()
3084 tcp_sk(sk)->nonagle |= TCP_NAGLE_OFF|TCP_NAGLE_PUSH; in __tcp_sock_set_nodelay()
3087 tcp_sk(sk)->nonagle &= ~TCP_NAGLE_OFF; in __tcp_sock_set_nodelay()
3107 if ((1 << sk->sk_state) & (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT) && in __tcp_sock_set_quickack()
3109 inet_csk(sk)->icsk_ack.pending |= ICSK_ACK_PUSHED; in __tcp_sock_set_quickack()
3127 return -EINVAL; in tcp_sock_set_syncnt()
3130 WRITE_ONCE(inet_csk(sk)->icsk_syn_retries, val); in tcp_sock_set_syncnt()
3139 WRITE_ONCE(inet_csk(sk)->icsk_user_timeout, val); in tcp_sock_set_user_timeout()
3149 return -EINVAL; in tcp_sock_set_keepidle_locked()
3152 WRITE_ONCE(tp->keepalive_time, val * HZ); in tcp_sock_set_keepidle_locked()
3154 !((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN))) { in tcp_sock_set_keepidle_locked()
3157 if (tp->keepalive_time > elapsed) in tcp_sock_set_keepidle_locked()
3158 elapsed = tp->keepalive_time - elapsed; in tcp_sock_set_keepidle_locked()
3181 return -EINVAL; in tcp_sock_set_keepintvl()
3184 WRITE_ONCE(tcp_sk(sk)->keepalive_intvl, val * HZ); in tcp_sock_set_keepintvl()
3193 return -EINVAL; in tcp_sock_set_keepcnt()
3197 WRITE_ONCE(tcp_sk(sk)->keepalive_probes, val); in tcp_sock_set_keepcnt()
3221 return -EINVAL; in do_tcp_setsockopt()
3224 min_t(long, TCP_CA_NAME_MAX-1, optlen)); in do_tcp_setsockopt()
3226 return -EFAULT; in do_tcp_setsockopt()
3231 ns_capable(sock_net(sk)->user_ns, in do_tcp_setsockopt()
3240 return -EINVAL; in do_tcp_setsockopt()
3243 min_t(long, TCP_ULP_NAME_MAX - 1, in do_tcp_setsockopt()
3246 return -EFAULT; in do_tcp_setsockopt()
3263 return -EINVAL; in do_tcp_setsockopt()
3266 return -EFAULT; in do_tcp_setsockopt()
3279 return -EINVAL; in do_tcp_setsockopt()
3282 return -EFAULT; in do_tcp_setsockopt()
3293 err = -EINVAL; in do_tcp_setsockopt()
3296 tp->rx_opt.user_mss = val; in do_tcp_setsockopt()
3305 err = -EINVAL; in do_tcp_setsockopt()
3307 tp->thin_lto = val; in do_tcp_setsockopt()
3312 err = -EINVAL; in do_tcp_setsockopt()
3317 err = -EPERM; in do_tcp_setsockopt()
3319 tp->repair = 1; in do_tcp_setsockopt()
3320 sk->sk_reuse = SK_FORCE_REUSE; in do_tcp_setsockopt()
3321 tp->repair_queue = TCP_NO_QUEUE; in do_tcp_setsockopt()
3323 tp->repair = 0; in do_tcp_setsockopt()
3324 sk->sk_reuse = SK_NO_REUSE; in do_tcp_setsockopt()
3327 tp->repair = 0; in do_tcp_setsockopt()
3328 sk->sk_reuse = SK_NO_REUSE; in do_tcp_setsockopt()
3330 err = -EINVAL; in do_tcp_setsockopt()
3335 if (!tp->repair) in do_tcp_setsockopt()
3336 err = -EPERM; in do_tcp_setsockopt()
3338 tp->repair_queue = val; in do_tcp_setsockopt()
3340 err = -EINVAL; in do_tcp_setsockopt()
3344 if (sk->sk_state != TCP_CLOSE) { in do_tcp_setsockopt()
3345 err = -EPERM; in do_tcp_setsockopt()
3346 } else if (tp->repair_queue == TCP_SEND_QUEUE) { in do_tcp_setsockopt()
3348 err = -EPERM; in do_tcp_setsockopt()
3350 WRITE_ONCE(tp->write_seq, val); in do_tcp_setsockopt()
3351 } else if (tp->repair_queue == TCP_RECV_QUEUE) { in do_tcp_setsockopt()
3352 if (tp->rcv_nxt != tp->copied_seq) { in do_tcp_setsockopt()
3353 err = -EPERM; in do_tcp_setsockopt()
3355 WRITE_ONCE(tp->rcv_nxt, val); in do_tcp_setsockopt()
3356 WRITE_ONCE(tp->copied_seq, val); in do_tcp_setsockopt()
3359 err = -EINVAL; in do_tcp_setsockopt()
3364 if (!tp->repair) in do_tcp_setsockopt()
3365 err = -EINVAL; in do_tcp_setsockopt()
3366 else if (sk->sk_state == TCP_ESTABLISHED && !tp->bytes_sent) in do_tcp_setsockopt()
3369 err = -EPERM; in do_tcp_setsockopt()
3381 err = -EINVAL; in do_tcp_setsockopt()
3383 WRITE_ONCE(tp->keepalive_intvl, val * HZ); in do_tcp_setsockopt()
3387 err = -EINVAL; in do_tcp_setsockopt()
3389 WRITE_ONCE(tp->keepalive_probes, val); in do_tcp_setsockopt()
3393 err = -EINVAL; in do_tcp_setsockopt()
3395 WRITE_ONCE(icsk->icsk_syn_retries, val); in do_tcp_setsockopt()
3401 err = -EINVAL; in do_tcp_setsockopt()
3403 tp->save_syn = val; in do_tcp_setsockopt()
3408 WRITE_ONCE(tp->linger2, -1); in do_tcp_setsockopt()
3410 WRITE_ONCE(tp->linger2, TCP_FIN_TIMEOUT_MAX); in do_tcp_setsockopt()
3412 WRITE_ONCE(tp->linger2, val * HZ); in do_tcp_setsockopt()
3417 WRITE_ONCE(icsk->icsk_accept_queue.rskq_defer_accept, in do_tcp_setsockopt()
3424 if (sk->sk_state != TCP_CLOSE) { in do_tcp_setsockopt()
3425 err = -EINVAL; in do_tcp_setsockopt()
3428 tp->window_clamp = 0; in do_tcp_setsockopt()
3430 tp->window_clamp = val < SOCK_MIN_RCVBUF / 2 ? in do_tcp_setsockopt()
3441 err = tp->af_specific->md5_parse(sk, optname, optval, optlen); in do_tcp_setsockopt()
3449 err = -EINVAL; in do_tcp_setsockopt()
3451 WRITE_ONCE(icsk->icsk_user_timeout, val); in do_tcp_setsockopt()
3455 if (val >= 0 && ((1 << sk->sk_state) & (TCPF_CLOSE | in do_tcp_setsockopt()
3461 err = -EINVAL; in do_tcp_setsockopt()
3466 err = -EINVAL; in do_tcp_setsockopt()
3467 } else if (READ_ONCE(net->ipv4.sysctl_tcp_fastopen) & in do_tcp_setsockopt()
3469 if (sk->sk_state == TCP_CLOSE) in do_tcp_setsockopt()
3470 tp->fastopen_connect = val; in do_tcp_setsockopt()
3472 err = -EINVAL; in do_tcp_setsockopt()
3474 err = -EOPNOTSUPP; in do_tcp_setsockopt()
3479 err = -EINVAL; in do_tcp_setsockopt()
3480 else if (!((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN))) in do_tcp_setsockopt()
3481 err = -EINVAL; in do_tcp_setsockopt()
3483 tp->fastopen_no_cookie = val; in do_tcp_setsockopt()
3486 if (!tp->repair) in do_tcp_setsockopt()
3487 err = -EPERM; in do_tcp_setsockopt()
3489 tp->tsoffset = val - tcp_time_stamp_raw(); in do_tcp_setsockopt()
3495 WRITE_ONCE(tp->notsent_lowat, val); in do_tcp_setsockopt()
3496 sk->sk_write_space(sk); in do_tcp_setsockopt()
3500 err = -EINVAL; in do_tcp_setsockopt()
3502 tp->recvmsg_inq = val; in do_tcp_setsockopt()
3507 WRITE_ONCE(tp->tcp_tx_delay, val); in do_tcp_setsockopt()
3520 err = -ENOPROTOOPT; in do_tcp_setsockopt()
3535 return READ_ONCE(icsk->icsk_af_ops)->setsockopt(sk, level, optname, in tcp_setsockopt()
3548 stats[i] = tp->chrono_stat[i - 1]; in tcp_get_info_chrono_stats()
3549 if (i == tp->chrono_type) in tcp_get_info_chrono_stats()
3550 stats[i] += tcp_jiffies32 - tp->chrono_start; in tcp_get_info_chrono_stats()
3555 info->tcpi_busy_time = total; in tcp_get_info_chrono_stats()
3556 info->tcpi_rwnd_limited = stats[TCP_CHRONO_RWND_LIMITED]; in tcp_get_info_chrono_stats()
3557 info->tcpi_sndbuf_limited = stats[TCP_CHRONO_SNDBUF_LIMITED]; in tcp_get_info_chrono_stats()
3571 if (sk->sk_type != SOCK_STREAM) in tcp_get_info()
3574 info->tcpi_state = inet_sk_state_load(sk); in tcp_get_info()
3577 rate = READ_ONCE(sk->sk_pacing_rate); in tcp_get_info()
3579 info->tcpi_pacing_rate = rate64; in tcp_get_info()
3581 rate = READ_ONCE(sk->sk_max_pacing_rate); in tcp_get_info()
3583 info->tcpi_max_pacing_rate = rate64; in tcp_get_info()
3585 info->tcpi_reordering = tp->reordering; in tcp_get_info()
3586 info->tcpi_snd_cwnd = tp->snd_cwnd; in tcp_get_info()
3588 if (info->tcpi_state == TCP_LISTEN) { in tcp_get_info()
3590 * tcpi_unacked -> Number of children ready for accept() in tcp_get_info()
3591 * tcpi_sacked -> max backlog in tcp_get_info()
3593 info->tcpi_unacked = READ_ONCE(sk->sk_ack_backlog); in tcp_get_info()
3594 info->tcpi_sacked = READ_ONCE(sk->sk_max_ack_backlog); in tcp_get_info()
3600 info->tcpi_ca_state = icsk->icsk_ca_state; in tcp_get_info()
3601 info->tcpi_retransmits = icsk->icsk_retransmits; in tcp_get_info()
3602 info->tcpi_probes = icsk->icsk_probes_out; in tcp_get_info()
3603 info->tcpi_backoff = icsk->icsk_backoff; in tcp_get_info()
3605 if (tp->rx_opt.tstamp_ok) in tcp_get_info()
3606 info->tcpi_options |= TCPI_OPT_TIMESTAMPS; in tcp_get_info()
3608 info->tcpi_options |= TCPI_OPT_SACK; in tcp_get_info()
3609 if (tp->rx_opt.wscale_ok) { in tcp_get_info()
3610 info->tcpi_options |= TCPI_OPT_WSCALE; in tcp_get_info()
3611 info->tcpi_snd_wscale = tp->rx_opt.snd_wscale; in tcp_get_info()
3612 info->tcpi_rcv_wscale = tp->rx_opt.rcv_wscale; in tcp_get_info()
3615 if (tp->ecn_flags & TCP_ECN_OK) in tcp_get_info()
3616 info->tcpi_options |= TCPI_OPT_ECN; in tcp_get_info()
3617 if (tp->ecn_flags & TCP_ECN_SEEN) in tcp_get_info()
3618 info->tcpi_options |= TCPI_OPT_ECN_SEEN; in tcp_get_info()
3619 if (tp->syn_data_acked) in tcp_get_info()
3620 info->tcpi_options |= TCPI_OPT_SYN_DATA; in tcp_get_info()
3622 info->tcpi_rto = jiffies_to_usecs(icsk->icsk_rto); in tcp_get_info()
3623 info->tcpi_ato = jiffies_to_usecs(icsk->icsk_ack.ato); in tcp_get_info()
3624 info->tcpi_snd_mss = tp->mss_cache; in tcp_get_info()
3625 info->tcpi_rcv_mss = icsk->icsk_ack.rcv_mss; in tcp_get_info()
3627 info->tcpi_unacked = tp->packets_out; in tcp_get_info()
3628 info->tcpi_sacked = tp->sacked_out; in tcp_get_info()
3630 info->tcpi_lost = tp->lost_out; in tcp_get_info()
3631 info->tcpi_retrans = tp->retrans_out; in tcp_get_info()
3634 info->tcpi_last_data_sent = jiffies_to_msecs(now - tp->lsndtime); in tcp_get_info()
3635 info->tcpi_last_data_recv = jiffies_to_msecs(now - icsk->icsk_ack.lrcvtime); in tcp_get_info()
3636 info->tcpi_last_ack_recv = jiffies_to_msecs(now - tp->rcv_tstamp); in tcp_get_info()
3638 info->tcpi_pmtu = icsk->icsk_pmtu_cookie; in tcp_get_info()
3639 info->tcpi_rcv_ssthresh = tp->rcv_ssthresh; in tcp_get_info()
3640 info->tcpi_rtt = tp->srtt_us >> 3; in tcp_get_info()
3641 info->tcpi_rttvar = tp->mdev_us >> 2; in tcp_get_info()
3642 info->tcpi_snd_ssthresh = tp->snd_ssthresh; in tcp_get_info()
3643 info->tcpi_advmss = tp->advmss; in tcp_get_info()
3645 info->tcpi_rcv_rtt = tp->rcv_rtt_est.rtt_us >> 3; in tcp_get_info()
3646 info->tcpi_rcv_space = tp->rcvq_space.space; in tcp_get_info()
3648 info->tcpi_total_retrans = tp->total_retrans; in tcp_get_info()
3650 info->tcpi_bytes_acked = tp->bytes_acked; in tcp_get_info()
3651 info->tcpi_bytes_received = tp->bytes_received; in tcp_get_info()
3652 info->tcpi_notsent_bytes = max_t(int, 0, tp->write_seq - tp->snd_nxt); in tcp_get_info()
3655 info->tcpi_segs_out = tp->segs_out; in tcp_get_info()
3656 info->tcpi_segs_in = tp->segs_in; in tcp_get_info()
3658 info->tcpi_min_rtt = tcp_min_rtt(tp); in tcp_get_info()
3659 info->tcpi_data_segs_in = tp->data_segs_in; in tcp_get_info()
3660 info->tcpi_data_segs_out = tp->data_segs_out; in tcp_get_info()
3662 info->tcpi_delivery_rate_app_limited = tp->rate_app_limited ? 1 : 0; in tcp_get_info()
3665 info->tcpi_delivery_rate = rate64; in tcp_get_info()
3666 info->tcpi_delivered = tp->delivered; in tcp_get_info()
3667 info->tcpi_delivered_ce = tp->delivered_ce; in tcp_get_info()
3668 info->tcpi_bytes_sent = tp->bytes_sent; in tcp_get_info()
3669 info->tcpi_bytes_retrans = tp->bytes_retrans; in tcp_get_info()
3670 info->tcpi_dsack_dups = tp->dsack_dups; in tcp_get_info()
3671 info->tcpi_reord_seen = tp->reord_seen; in tcp_get_info()
3672 info->tcpi_rcv_ooopack = tp->rcv_ooopack; in tcp_get_info()
3673 info->tcpi_snd_wnd = tp->snd_wnd; in tcp_get_info()
3674 info->tcpi_fastopen_client_fail = tp->fastopen_client_fail; in tcp_get_info()
3731 tp->data_segs_out, TCP_NLA_PAD); in tcp_get_timestamping_opt_stats()
3733 tp->total_retrans, TCP_NLA_PAD); in tcp_get_timestamping_opt_stats()
3735 rate = READ_ONCE(sk->sk_pacing_rate); in tcp_get_timestamping_opt_stats()
3742 nla_put_u32(stats, TCP_NLA_SND_CWND, tp->snd_cwnd); in tcp_get_timestamping_opt_stats()
3743 nla_put_u32(stats, TCP_NLA_REORDERING, tp->reordering); in tcp_get_timestamping_opt_stats()
3746 nla_put_u8(stats, TCP_NLA_RECUR_RETRANS, inet_csk(sk)->icsk_retransmits); in tcp_get_timestamping_opt_stats()
3747 nla_put_u8(stats, TCP_NLA_DELIVERY_RATE_APP_LMT, !!tp->rate_app_limited); in tcp_get_timestamping_opt_stats()
3748 nla_put_u32(stats, TCP_NLA_SND_SSTHRESH, tp->snd_ssthresh); in tcp_get_timestamping_opt_stats()
3749 nla_put_u32(stats, TCP_NLA_DELIVERED, tp->delivered); in tcp_get_timestamping_opt_stats()
3750 nla_put_u32(stats, TCP_NLA_DELIVERED_CE, tp->delivered_ce); in tcp_get_timestamping_opt_stats()
3752 nla_put_u32(stats, TCP_NLA_SNDQ_SIZE, tp->write_seq - tp->snd_una); in tcp_get_timestamping_opt_stats()
3753 nla_put_u8(stats, TCP_NLA_CA_STATE, inet_csk(sk)->icsk_ca_state); in tcp_get_timestamping_opt_stats()
3755 nla_put_u64_64bit(stats, TCP_NLA_BYTES_SENT, tp->bytes_sent, in tcp_get_timestamping_opt_stats()
3757 nla_put_u64_64bit(stats, TCP_NLA_BYTES_RETRANS, tp->bytes_retrans, in tcp_get_timestamping_opt_stats()
3759 nla_put_u32(stats, TCP_NLA_DSACK_DUPS, tp->dsack_dups); in tcp_get_timestamping_opt_stats()
3760 nla_put_u32(stats, TCP_NLA_REORD_SEEN, tp->reord_seen); in tcp_get_timestamping_opt_stats()
3761 nla_put_u32(stats, TCP_NLA_SRTT, tp->srtt_us >> 3); in tcp_get_timestamping_opt_stats()
3762 nla_put_u16(stats, TCP_NLA_TIMEOUT_REHASH, tp->timeout_rehash); in tcp_get_timestamping_opt_stats()
3764 max_t(int, 0, tp->write_seq - tp->snd_nxt)); in tcp_get_timestamping_opt_stats()
3765 nla_put_u64_64bit(stats, TCP_NLA_EDT, orig_skb->skb_mstamp_ns, in tcp_get_timestamping_opt_stats()
3780 return -EFAULT; in do_tcp_getsockopt()
3785 return -EINVAL; in do_tcp_getsockopt()
3789 val = tp->mss_cache; in do_tcp_getsockopt()
3790 if (tp->rx_opt.user_mss && in do_tcp_getsockopt()
3791 ((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN))) in do_tcp_getsockopt()
3792 val = tp->rx_opt.user_mss; in do_tcp_getsockopt()
3793 if (tp->repair) in do_tcp_getsockopt()
3794 val = tp->rx_opt.mss_clamp; in do_tcp_getsockopt()
3797 val = !!(tp->nonagle&TCP_NAGLE_OFF); in do_tcp_getsockopt()
3800 val = !!(tp->nonagle&TCP_NAGLE_CORK); in do_tcp_getsockopt()
3812 val = READ_ONCE(icsk->icsk_syn_retries) ? : in do_tcp_getsockopt()
3813 READ_ONCE(net->ipv4.sysctl_tcp_syn_retries); in do_tcp_getsockopt()
3816 val = READ_ONCE(tp->linger2); in do_tcp_getsockopt()
3818 val = (val ? : READ_ONCE(net->ipv4.sysctl_tcp_fin_timeout)) / HZ; in do_tcp_getsockopt()
3821 val = READ_ONCE(icsk->icsk_accept_queue.rskq_defer_accept); in do_tcp_getsockopt()
3826 val = tp->window_clamp; in do_tcp_getsockopt()
3832 return -EFAULT; in do_tcp_getsockopt()
3838 return -EFAULT; in do_tcp_getsockopt()
3840 return -EFAULT; in do_tcp_getsockopt()
3850 return -EFAULT; in do_tcp_getsockopt()
3852 ca_ops = icsk->icsk_ca_ops; in do_tcp_getsockopt()
3853 if (ca_ops && ca_ops->get_info) in do_tcp_getsockopt()
3854 sz = ca_ops->get_info(sk, ~0U, &attr, &info); in do_tcp_getsockopt()
3858 return -EFAULT; in do_tcp_getsockopt()
3860 return -EFAULT; in do_tcp_getsockopt()
3869 return -EFAULT; in do_tcp_getsockopt()
3872 return -EFAULT; in do_tcp_getsockopt()
3873 if (copy_to_user(optval, icsk->icsk_ca_ops->name, len)) in do_tcp_getsockopt()
3874 return -EFAULT; in do_tcp_getsockopt()
3879 return -EFAULT; in do_tcp_getsockopt()
3881 if (!icsk->icsk_ulp_ops) { in do_tcp_getsockopt()
3883 return -EFAULT; in do_tcp_getsockopt()
3887 return -EFAULT; in do_tcp_getsockopt()
3888 if (copy_to_user(optval, icsk->icsk_ulp_ops->name, len)) in do_tcp_getsockopt()
3889 return -EFAULT; in do_tcp_getsockopt()
3897 return -EFAULT; in do_tcp_getsockopt()
3903 return -EFAULT; in do_tcp_getsockopt()
3905 return -EFAULT; in do_tcp_getsockopt()
3909 val = tp->thin_lto; in do_tcp_getsockopt()
3917 val = tp->repair; in do_tcp_getsockopt()
3921 if (tp->repair) in do_tcp_getsockopt()
3922 val = tp->repair_queue; in do_tcp_getsockopt()
3924 return -EINVAL; in do_tcp_getsockopt()
3931 return -EFAULT; in do_tcp_getsockopt()
3934 return -EINVAL; in do_tcp_getsockopt()
3936 if (!tp->repair) in do_tcp_getsockopt()
3937 return -EPERM; in do_tcp_getsockopt()
3939 opt.snd_wl1 = tp->snd_wl1; in do_tcp_getsockopt()
3940 opt.snd_wnd = tp->snd_wnd; in do_tcp_getsockopt()
3941 opt.max_window = tp->max_window; in do_tcp_getsockopt()
3942 opt.rcv_wnd = tp->rcv_wnd; in do_tcp_getsockopt()
3943 opt.rcv_wup = tp->rcv_wup; in do_tcp_getsockopt()
3946 return -EFAULT; in do_tcp_getsockopt()
3950 if (tp->repair_queue == TCP_SEND_QUEUE) in do_tcp_getsockopt()
3951 val = tp->write_seq; in do_tcp_getsockopt()
3952 else if (tp->repair_queue == TCP_RECV_QUEUE) in do_tcp_getsockopt()
3953 val = tp->rcv_nxt; in do_tcp_getsockopt()
3955 return -EINVAL; in do_tcp_getsockopt()
3959 val = READ_ONCE(icsk->icsk_user_timeout); in do_tcp_getsockopt()
3963 val = READ_ONCE(icsk->icsk_accept_queue.fastopenq.max_qlen); in do_tcp_getsockopt()
3967 val = tp->fastopen_connect; in do_tcp_getsockopt()
3971 val = tp->fastopen_no_cookie; in do_tcp_getsockopt()
3975 val = READ_ONCE(tp->tcp_tx_delay); in do_tcp_getsockopt()
3979 val = tcp_time_stamp_raw() + tp->tsoffset; in do_tcp_getsockopt()
3982 val = READ_ONCE(tp->notsent_lowat); in do_tcp_getsockopt()
3985 val = tp->recvmsg_inq; in do_tcp_getsockopt()
3988 val = tp->save_syn; in do_tcp_getsockopt()
3992 return -EFAULT; in do_tcp_getsockopt()
3995 if (tp->saved_syn) { in do_tcp_getsockopt()
3996 if (len < tcp_saved_syn_len(tp->saved_syn)) { in do_tcp_getsockopt()
3997 if (put_user(tcp_saved_syn_len(tp->saved_syn), in do_tcp_getsockopt()
4000 return -EFAULT; in do_tcp_getsockopt()
4003 return -EINVAL; in do_tcp_getsockopt()
4005 len = tcp_saved_syn_len(tp->saved_syn); in do_tcp_getsockopt()
4008 return -EFAULT; in do_tcp_getsockopt()
4010 if (copy_to_user(optval, tp->saved_syn->data, len)) { in do_tcp_getsockopt()
4012 return -EFAULT; in do_tcp_getsockopt()
4020 return -EFAULT; in do_tcp_getsockopt()
4030 return -EFAULT; in do_tcp_getsockopt()
4033 return -EINVAL; in do_tcp_getsockopt()
4037 return -EFAULT; in do_tcp_getsockopt()
4040 return -EFAULT; in do_tcp_getsockopt()
4064 err = -EFAULT; in do_tcp_getsockopt()
4069 return -ENOPROTOOPT; in do_tcp_getsockopt()
4073 return -EFAULT; in do_tcp_getsockopt()
4075 return -EFAULT; in do_tcp_getsockopt()
4098 return READ_ONCE(icsk->icsk_af_ops)->getsockopt(sk, level, optname, in tcp_getsockopt()
4173 * tcp_get_md5sig_pool - get md5sig_pool for this user
4199 struct ahash_request *req = hp->md5_req; in tcp_md5_hash_skb_data()
4202 skb_headlen(skb) - header_len : 0; in tcp_md5_hash_skb_data()
4213 for (i = 0; i < shi->nr_frags; ++i) { in tcp_md5_hash_skb_data()
4214 const skb_frag_t *f = &shi->frags[i]; in tcp_md5_hash_skb_data()
4235 u8 keylen = READ_ONCE(key->keylen); /* paired with WRITE_ONCE() in tcp_md5_do_add */ in tcp_md5_hash_key()
4238 sg_init_one(&sg, key->key, keylen); in tcp_md5_hash_key()
4239 ahash_request_set_crypt(hp->md5_req, &sg, NULL, keylen); in tcp_md5_hash_key()
4241 /* We use data_race() because tcp_md5_do_add() might change key->key under us */ in tcp_md5_hash_key()
4242 return data_race(crypto_ahash_update(hp->md5_req)); in tcp_md5_hash_key()
4256 req = rcu_dereference_protected(tcp_sk(sk)->fastopen_rsk, 1); in tcp_done()
4258 if (sk->sk_state == TCP_SYN_SENT || sk->sk_state == TCP_SYN_RECV) in tcp_done()
4266 WRITE_ONCE(sk->sk_shutdown, SHUTDOWN_MASK); in tcp_done()
4269 sk->sk_state_change(sk); in tcp_done()
4278 if (sk->sk_state == TCP_NEW_SYN_RECV) { in tcp_abort()
4282 inet_csk_reqsk_queue_drop(req->rsk_listener, req); in tcp_abort()
4286 return -EOPNOTSUPP; in tcp_abort()
4296 if (sk->sk_state == TCP_LISTEN) { in tcp_abort()
4306 sk->sk_err = err; in tcp_abort()
4309 sk->sk_error_report(sk); in tcp_abort()
4310 if (tcp_need_reset(sk->sk_state)) in tcp_abort()
4419 /* Set per-socket limits to no more than 1/128 the pressure threshold */ in tcp_init()
4420 limit = nr_free_buffer_pages() << (PAGE_SHIFT - 7); in tcp_init()