1 // SPDX-License-Identifier: GPL-2.0
2 /* Multipath TCP
3 *
4 * Copyright (c) 2017 - 2019, Intel Corporation.
5 */
6
7 #define pr_fmt(fmt) "MPTCP: " fmt
8
9 #include <linux/kernel.h>
10 #include <linux/module.h>
11 #include <linux/netdevice.h>
12 #include <crypto/algapi.h>
13 #include <crypto/sha.h>
14 #include <net/sock.h>
15 #include <net/inet_common.h>
16 #include <net/inet_hashtables.h>
17 #include <net/protocol.h>
18 #include <net/tcp.h>
19 #if IS_ENABLED(CONFIG_MPTCP_IPV6)
20 #include <net/ip6_route.h>
21 #endif
22 #include <net/mptcp.h>
23 #include <uapi/linux/mptcp.h>
24 #include "protocol.h"
25 #include "mib.h"
26
SUBFLOW_REQ_INC_STATS(struct request_sock * req,enum linux_mptcp_mib_field field)27 static void SUBFLOW_REQ_INC_STATS(struct request_sock *req,
28 enum linux_mptcp_mib_field field)
29 {
30 MPTCP_INC_STATS(sock_net(req_to_sk(req)), field);
31 }
32
subflow_req_destructor(struct request_sock * req)33 static void subflow_req_destructor(struct request_sock *req)
34 {
35 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req);
36
37 pr_debug("subflow_req=%p", subflow_req);
38
39 if (subflow_req->msk)
40 sock_put((struct sock *)subflow_req->msk);
41
42 mptcp_token_destroy_request(req);
43 }
44
subflow_generate_hmac(u64 key1,u64 key2,u32 nonce1,u32 nonce2,void * hmac)45 static void subflow_generate_hmac(u64 key1, u64 key2, u32 nonce1, u32 nonce2,
46 void *hmac)
47 {
48 u8 msg[8];
49
50 put_unaligned_be32(nonce1, &msg[0]);
51 put_unaligned_be32(nonce2, &msg[4]);
52
53 mptcp_crypto_hmac_sha(key1, key2, msg, 8, hmac);
54 }
55
mptcp_can_accept_new_subflow(const struct mptcp_sock * msk)56 static bool mptcp_can_accept_new_subflow(const struct mptcp_sock *msk)
57 {
58 return mptcp_is_fully_established((void *)msk) &&
59 READ_ONCE(msk->pm.accept_subflow);
60 }
61
62 /* validate received token and create truncated hmac and nonce for SYN-ACK */
subflow_token_join_request(struct request_sock * req,const struct sk_buff * skb)63 static struct mptcp_sock *subflow_token_join_request(struct request_sock *req,
64 const struct sk_buff *skb)
65 {
66 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req);
67 u8 hmac[SHA256_DIGEST_SIZE];
68 struct mptcp_sock *msk;
69 int local_id;
70
71 msk = mptcp_token_get_sock(sock_net(req_to_sk(req)), subflow_req->token);
72 if (!msk) {
73 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINNOTOKEN);
74 return NULL;
75 }
76
77 local_id = mptcp_pm_get_local_id(msk, (struct sock_common *)req);
78 if (local_id < 0) {
79 sock_put((struct sock *)msk);
80 return NULL;
81 }
82 subflow_req->local_id = local_id;
83
84 get_random_bytes(&subflow_req->local_nonce, sizeof(u32));
85
86 subflow_generate_hmac(msk->local_key, msk->remote_key,
87 subflow_req->local_nonce,
88 subflow_req->remote_nonce, hmac);
89
90 subflow_req->thmac = get_unaligned_be64(hmac);
91 return msk;
92 }
93
__subflow_init_req(struct request_sock * req,const struct sock * sk_listener)94 static int __subflow_init_req(struct request_sock *req, const struct sock *sk_listener)
95 {
96 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req);
97
98 subflow_req->mp_capable = 0;
99 subflow_req->mp_join = 0;
100 subflow_req->msk = NULL;
101 mptcp_token_init_request(req);
102
103 #ifdef CONFIG_TCP_MD5SIG
104 /* no MPTCP if MD5SIG is enabled on this socket or we may run out of
105 * TCP option space.
106 */
107 if (rcu_access_pointer(tcp_sk(sk_listener)->md5sig_info))
108 return -EINVAL;
109 #endif
110
111 return 0;
112 }
113
subflow_init_req(struct request_sock * req,const struct sock * sk_listener,struct sk_buff * skb)114 static void subflow_init_req(struct request_sock *req,
115 const struct sock *sk_listener,
116 struct sk_buff *skb)
117 {
118 struct mptcp_subflow_context *listener = mptcp_subflow_ctx(sk_listener);
119 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req);
120 struct mptcp_options_received mp_opt;
121 int ret;
122
123 pr_debug("subflow_req=%p, listener=%p", subflow_req, listener);
124
125 ret = __subflow_init_req(req, sk_listener);
126 if (ret)
127 return;
128
129 mptcp_get_options(skb, &mp_opt);
130
131 if (mp_opt.mp_capable) {
132 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_MPCAPABLEPASSIVE);
133
134 if (mp_opt.mp_join)
135 return;
136 } else if (mp_opt.mp_join) {
137 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINSYNRX);
138 }
139
140 if (mp_opt.mp_capable && listener->request_mptcp) {
141 int err, retries = 4;
142
143 subflow_req->ssn_offset = TCP_SKB_CB(skb)->seq;
144 again:
145 do {
146 get_random_bytes(&subflow_req->local_key, sizeof(subflow_req->local_key));
147 } while (subflow_req->local_key == 0);
148
149 if (unlikely(req->syncookie)) {
150 mptcp_crypto_key_sha(subflow_req->local_key,
151 &subflow_req->token,
152 &subflow_req->idsn);
153 if (mptcp_token_exists(subflow_req->token)) {
154 if (retries-- > 0)
155 goto again;
156 } else {
157 subflow_req->mp_capable = 1;
158 }
159 return;
160 }
161
162 err = mptcp_token_new_request(req);
163 if (err == 0)
164 subflow_req->mp_capable = 1;
165 else if (retries-- > 0)
166 goto again;
167
168 } else if (mp_opt.mp_join && listener->request_mptcp) {
169 subflow_req->ssn_offset = TCP_SKB_CB(skb)->seq;
170 subflow_req->mp_join = 1;
171 subflow_req->backup = mp_opt.backup;
172 subflow_req->remote_id = mp_opt.join_id;
173 subflow_req->token = mp_opt.token;
174 subflow_req->remote_nonce = mp_opt.nonce;
175 subflow_req->msk = subflow_token_join_request(req, skb);
176
177 if (unlikely(req->syncookie) && subflow_req->msk) {
178 if (mptcp_can_accept_new_subflow(subflow_req->msk))
179 subflow_init_req_cookie_join_save(subflow_req, skb);
180 }
181
182 pr_debug("token=%u, remote_nonce=%u msk=%p", subflow_req->token,
183 subflow_req->remote_nonce, subflow_req->msk);
184 }
185 }
186
mptcp_subflow_init_cookie_req(struct request_sock * req,const struct sock * sk_listener,struct sk_buff * skb)187 int mptcp_subflow_init_cookie_req(struct request_sock *req,
188 const struct sock *sk_listener,
189 struct sk_buff *skb)
190 {
191 struct mptcp_subflow_context *listener = mptcp_subflow_ctx(sk_listener);
192 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req);
193 struct mptcp_options_received mp_opt;
194 int err;
195
196 err = __subflow_init_req(req, sk_listener);
197 if (err)
198 return err;
199
200 mptcp_get_options(skb, &mp_opt);
201
202 if (mp_opt.mp_capable && mp_opt.mp_join)
203 return -EINVAL;
204
205 if (mp_opt.mp_capable && listener->request_mptcp) {
206 if (mp_opt.sndr_key == 0)
207 return -EINVAL;
208
209 subflow_req->local_key = mp_opt.rcvr_key;
210 err = mptcp_token_new_request(req);
211 if (err)
212 return err;
213
214 subflow_req->mp_capable = 1;
215 subflow_req->ssn_offset = TCP_SKB_CB(skb)->seq - 1;
216 } else if (mp_opt.mp_join && listener->request_mptcp) {
217 if (!mptcp_token_join_cookie_init_state(subflow_req, skb))
218 return -EINVAL;
219
220 if (mptcp_can_accept_new_subflow(subflow_req->msk))
221 subflow_req->mp_join = 1;
222
223 subflow_req->ssn_offset = TCP_SKB_CB(skb)->seq - 1;
224 }
225
226 return 0;
227 }
228 EXPORT_SYMBOL_GPL(mptcp_subflow_init_cookie_req);
229
subflow_v4_init_req(struct request_sock * req,const struct sock * sk_listener,struct sk_buff * skb)230 static void subflow_v4_init_req(struct request_sock *req,
231 const struct sock *sk_listener,
232 struct sk_buff *skb)
233 {
234 tcp_rsk(req)->is_mptcp = 1;
235
236 tcp_request_sock_ipv4_ops.init_req(req, sk_listener, skb);
237
238 subflow_init_req(req, sk_listener, skb);
239 }
240
241 #if IS_ENABLED(CONFIG_MPTCP_IPV6)
subflow_v6_init_req(struct request_sock * req,const struct sock * sk_listener,struct sk_buff * skb)242 static void subflow_v6_init_req(struct request_sock *req,
243 const struct sock *sk_listener,
244 struct sk_buff *skb)
245 {
246 tcp_rsk(req)->is_mptcp = 1;
247
248 tcp_request_sock_ipv6_ops.init_req(req, sk_listener, skb);
249
250 subflow_init_req(req, sk_listener, skb);
251 }
252 #endif
253
254 /* validate received truncated hmac and create hmac for third ACK */
subflow_thmac_valid(struct mptcp_subflow_context * subflow)255 static bool subflow_thmac_valid(struct mptcp_subflow_context *subflow)
256 {
257 u8 hmac[SHA256_DIGEST_SIZE];
258 u64 thmac;
259
260 subflow_generate_hmac(subflow->remote_key, subflow->local_key,
261 subflow->remote_nonce, subflow->local_nonce,
262 hmac);
263
264 thmac = get_unaligned_be64(hmac);
265 pr_debug("subflow=%p, token=%u, thmac=%llu, subflow->thmac=%llu\n",
266 subflow, subflow->token,
267 (unsigned long long)thmac,
268 (unsigned long long)subflow->thmac);
269
270 return thmac == subflow->thmac;
271 }
272
mptcp_subflow_reset(struct sock * ssk)273 void mptcp_subflow_reset(struct sock *ssk)
274 {
275 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
276 struct sock *sk = subflow->conn;
277
278 tcp_send_active_reset(ssk, GFP_ATOMIC);
279 tcp_done(ssk);
280 if (!test_and_set_bit(MPTCP_WORK_CLOSE_SUBFLOW, &mptcp_sk(sk)->flags) &&
281 schedule_work(&mptcp_sk(sk)->work))
282 sock_hold(sk);
283 }
284
subflow_finish_connect(struct sock * sk,const struct sk_buff * skb)285 static void subflow_finish_connect(struct sock *sk, const struct sk_buff *skb)
286 {
287 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
288 struct mptcp_options_received mp_opt;
289 struct sock *parent = subflow->conn;
290
291 subflow->icsk_af_ops->sk_rx_dst_set(sk, skb);
292
293 if (inet_sk_state_load(parent) == TCP_SYN_SENT) {
294 inet_sk_state_store(parent, TCP_ESTABLISHED);
295 parent->sk_state_change(parent);
296 }
297
298 /* be sure no special action on any packet other than syn-ack */
299 if (subflow->conn_finished)
300 return;
301
302 subflow->rel_write_seq = 1;
303 subflow->conn_finished = 1;
304 subflow->ssn_offset = TCP_SKB_CB(skb)->seq;
305 pr_debug("subflow=%p synack seq=%x", subflow, subflow->ssn_offset);
306
307 mptcp_get_options(skb, &mp_opt);
308 if (subflow->request_mptcp) {
309 if (!mp_opt.mp_capable) {
310 MPTCP_INC_STATS(sock_net(sk),
311 MPTCP_MIB_MPCAPABLEACTIVEFALLBACK);
312 mptcp_do_fallback(sk);
313 pr_fallback(mptcp_sk(subflow->conn));
314 goto fallback;
315 }
316
317 subflow->mp_capable = 1;
318 subflow->can_ack = 1;
319 subflow->remote_key = mp_opt.sndr_key;
320 pr_debug("subflow=%p, remote_key=%llu", subflow,
321 subflow->remote_key);
322 mptcp_finish_connect(sk);
323 } else if (subflow->request_join) {
324 u8 hmac[SHA256_DIGEST_SIZE];
325
326 if (!mp_opt.mp_join)
327 goto do_reset;
328
329 subflow->thmac = mp_opt.thmac;
330 subflow->remote_nonce = mp_opt.nonce;
331 pr_debug("subflow=%p, thmac=%llu, remote_nonce=%u", subflow,
332 subflow->thmac, subflow->remote_nonce);
333
334 if (!subflow_thmac_valid(subflow)) {
335 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_JOINACKMAC);
336 goto do_reset;
337 }
338
339 if (!mptcp_finish_join(sk))
340 goto do_reset;
341
342 subflow_generate_hmac(subflow->local_key, subflow->remote_key,
343 subflow->local_nonce,
344 subflow->remote_nonce,
345 hmac);
346 memcpy(subflow->hmac, hmac, MPTCPOPT_HMAC_LEN);
347
348 subflow->mp_join = 1;
349 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_JOINSYNACKRX);
350 } else if (mptcp_check_fallback(sk)) {
351 fallback:
352 mptcp_rcv_space_init(mptcp_sk(parent), sk);
353 }
354 return;
355
356 do_reset:
357 mptcp_subflow_reset(sk);
358 }
359
360 static struct request_sock_ops mptcp_subflow_v4_request_sock_ops __ro_after_init;
361 static struct tcp_request_sock_ops subflow_request_sock_ipv4_ops __ro_after_init;
362
subflow_v4_conn_request(struct sock * sk,struct sk_buff * skb)363 static int subflow_v4_conn_request(struct sock *sk, struct sk_buff *skb)
364 {
365 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
366
367 pr_debug("subflow=%p", subflow);
368
369 /* Never answer to SYNs sent to broadcast or multicast */
370 if (skb_rtable(skb)->rt_flags & (RTCF_BROADCAST | RTCF_MULTICAST))
371 goto drop;
372
373 return tcp_conn_request(&mptcp_subflow_v4_request_sock_ops,
374 &subflow_request_sock_ipv4_ops,
375 sk, skb);
376 drop:
377 tcp_listendrop(sk);
378 return 0;
379 }
380
subflow_v4_req_destructor(struct request_sock * req)381 static void subflow_v4_req_destructor(struct request_sock *req)
382 {
383 subflow_req_destructor(req);
384 tcp_request_sock_ops.destructor(req);
385 }
386
387 #if IS_ENABLED(CONFIG_MPTCP_IPV6)
388 static struct request_sock_ops mptcp_subflow_v6_request_sock_ops __ro_after_init;
389 static struct tcp_request_sock_ops subflow_request_sock_ipv6_ops __ro_after_init;
390 static struct inet_connection_sock_af_ops subflow_v6_specific __ro_after_init;
391 static struct inet_connection_sock_af_ops subflow_v6m_specific __ro_after_init;
392
subflow_v6_conn_request(struct sock * sk,struct sk_buff * skb)393 static int subflow_v6_conn_request(struct sock *sk, struct sk_buff *skb)
394 {
395 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
396
397 pr_debug("subflow=%p", subflow);
398
399 if (skb->protocol == htons(ETH_P_IP))
400 return subflow_v4_conn_request(sk, skb);
401
402 if (!ipv6_unicast_destination(skb))
403 goto drop;
404
405 if (ipv6_addr_v4mapped(&ipv6_hdr(skb)->saddr)) {
406 __IP6_INC_STATS(sock_net(sk), NULL, IPSTATS_MIB_INHDRERRORS);
407 return 0;
408 }
409
410 return tcp_conn_request(&mptcp_subflow_v6_request_sock_ops,
411 &subflow_request_sock_ipv6_ops, sk, skb);
412
413 drop:
414 tcp_listendrop(sk);
415 return 0; /* don't send reset */
416 }
417
subflow_v6_req_destructor(struct request_sock * req)418 static void subflow_v6_req_destructor(struct request_sock *req)
419 {
420 subflow_req_destructor(req);
421 tcp6_request_sock_ops.destructor(req);
422 }
423 #endif
424
mptcp_subflow_reqsk_alloc(const struct request_sock_ops * ops,struct sock * sk_listener,bool attach_listener)425 struct request_sock *mptcp_subflow_reqsk_alloc(const struct request_sock_ops *ops,
426 struct sock *sk_listener,
427 bool attach_listener)
428 {
429 if (ops->family == AF_INET)
430 ops = &mptcp_subflow_v4_request_sock_ops;
431 #if IS_ENABLED(CONFIG_MPTCP_IPV6)
432 else if (ops->family == AF_INET6)
433 ops = &mptcp_subflow_v6_request_sock_ops;
434 #endif
435
436 return inet_reqsk_alloc(ops, sk_listener, attach_listener);
437 }
438 EXPORT_SYMBOL(mptcp_subflow_reqsk_alloc);
439
440 /* validate hmac received in third ACK */
subflow_hmac_valid(const struct request_sock * req,const struct mptcp_options_received * mp_opt)441 static bool subflow_hmac_valid(const struct request_sock *req,
442 const struct mptcp_options_received *mp_opt)
443 {
444 const struct mptcp_subflow_request_sock *subflow_req;
445 u8 hmac[SHA256_DIGEST_SIZE];
446 struct mptcp_sock *msk;
447
448 subflow_req = mptcp_subflow_rsk(req);
449 msk = subflow_req->msk;
450 if (!msk)
451 return false;
452
453 subflow_generate_hmac(msk->remote_key, msk->local_key,
454 subflow_req->remote_nonce,
455 subflow_req->local_nonce, hmac);
456
457 return !crypto_memneq(hmac, mp_opt->hmac, MPTCPOPT_HMAC_LEN);
458 }
459
mptcp_sock_destruct(struct sock * sk)460 static void mptcp_sock_destruct(struct sock *sk)
461 {
462 /* if new mptcp socket isn't accepted, it is free'd
463 * from the tcp listener sockets request queue, linked
464 * from req->sk. The tcp socket is released.
465 * This calls the ULP release function which will
466 * also remove the mptcp socket, via
467 * sock_put(ctx->conn).
468 *
469 * Problem is that the mptcp socket will be in
470 * ESTABLISHED state and will not have the SOCK_DEAD flag.
471 * Both result in warnings from inet_sock_destruct.
472 */
473 if ((1 << sk->sk_state) & (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)) {
474 sk->sk_state = TCP_CLOSE;
475 WARN_ON_ONCE(sk->sk_socket);
476 sock_orphan(sk);
477 }
478
479 mptcp_destroy_common(mptcp_sk(sk));
480 inet_sock_destruct(sk);
481 }
482
mptcp_force_close(struct sock * sk)483 static void mptcp_force_close(struct sock *sk)
484 {
485 inet_sk_state_store(sk, TCP_CLOSE);
486 sk_common_release(sk);
487 }
488
subflow_ulp_fallback(struct sock * sk,struct mptcp_subflow_context * old_ctx)489 static void subflow_ulp_fallback(struct sock *sk,
490 struct mptcp_subflow_context *old_ctx)
491 {
492 struct inet_connection_sock *icsk = inet_csk(sk);
493
494 mptcp_subflow_tcp_fallback(sk, old_ctx);
495 icsk->icsk_ulp_ops = NULL;
496 rcu_assign_pointer(icsk->icsk_ulp_data, NULL);
497 tcp_sk(sk)->is_mptcp = 0;
498 }
499
subflow_drop_ctx(struct sock * ssk)500 static void subflow_drop_ctx(struct sock *ssk)
501 {
502 struct mptcp_subflow_context *ctx = mptcp_subflow_ctx(ssk);
503
504 if (!ctx)
505 return;
506
507 subflow_ulp_fallback(ssk, ctx);
508 if (ctx->conn)
509 sock_put(ctx->conn);
510
511 kfree_rcu(ctx, rcu);
512 }
513
mptcp_subflow_fully_established(struct mptcp_subflow_context * subflow,struct mptcp_options_received * mp_opt)514 void mptcp_subflow_fully_established(struct mptcp_subflow_context *subflow,
515 struct mptcp_options_received *mp_opt)
516 {
517 struct mptcp_sock *msk = mptcp_sk(subflow->conn);
518
519 subflow->remote_key = mp_opt->sndr_key;
520 subflow->fully_established = 1;
521 subflow->can_ack = 1;
522 WRITE_ONCE(msk->fully_established, true);
523 }
524
subflow_syn_recv_sock(const struct sock * sk,struct sk_buff * skb,struct request_sock * req,struct dst_entry * dst,struct request_sock * req_unhash,bool * own_req)525 static struct sock *subflow_syn_recv_sock(const struct sock *sk,
526 struct sk_buff *skb,
527 struct request_sock *req,
528 struct dst_entry *dst,
529 struct request_sock *req_unhash,
530 bool *own_req)
531 {
532 struct mptcp_subflow_context *listener = mptcp_subflow_ctx(sk);
533 struct mptcp_subflow_request_sock *subflow_req;
534 struct mptcp_options_received mp_opt;
535 bool fallback, fallback_is_fatal;
536 struct sock *new_msk = NULL;
537 struct sock *child;
538
539 pr_debug("listener=%p, req=%p, conn=%p", listener, req, listener->conn);
540
541 /* After child creation we must look for 'mp_capable' even when options
542 * are not parsed
543 */
544 mp_opt.mp_capable = 0;
545
546 /* hopefully temporary handling for MP_JOIN+syncookie */
547 subflow_req = mptcp_subflow_rsk(req);
548 fallback_is_fatal = tcp_rsk(req)->is_mptcp && subflow_req->mp_join;
549 fallback = !tcp_rsk(req)->is_mptcp;
550 if (fallback)
551 goto create_child;
552
553 /* if the sk is MP_CAPABLE, we try to fetch the client key */
554 if (subflow_req->mp_capable) {
555 /* we can receive and accept an in-window, out-of-order pkt,
556 * which may not carry the MP_CAPABLE opt even on mptcp enabled
557 * paths: always try to extract the peer key, and fallback
558 * for packets missing it.
559 * Even OoO DSS packets coming legitly after dropped or
560 * reordered MPC will cause fallback, but we don't have other
561 * options.
562 */
563 mptcp_get_options(skb, &mp_opt);
564 if (!mp_opt.mp_capable) {
565 fallback = true;
566 goto create_child;
567 }
568
569 new_msk = mptcp_sk_clone(listener->conn, &mp_opt, req);
570 if (!new_msk)
571 fallback = true;
572 } else if (subflow_req->mp_join) {
573 mptcp_get_options(skb, &mp_opt);
574 if (!mp_opt.mp_join || !subflow_hmac_valid(req, &mp_opt) ||
575 !mptcp_can_accept_new_subflow(subflow_req->msk)) {
576 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINACKMAC);
577 fallback = true;
578 }
579 }
580
581 create_child:
582 child = listener->icsk_af_ops->syn_recv_sock(sk, skb, req, dst,
583 req_unhash, own_req);
584
585 if (child && *own_req) {
586 struct mptcp_subflow_context *ctx = mptcp_subflow_ctx(child);
587
588 tcp_rsk(req)->drop_req = false;
589
590 /* we need to fallback on ctx allocation failure and on pre-reqs
591 * checking above. In the latter scenario we additionally need
592 * to reset the context to non MPTCP status.
593 */
594 if (!ctx || fallback) {
595 if (fallback_is_fatal)
596 goto dispose_child;
597
598 subflow_drop_ctx(child);
599 goto out;
600 }
601
602 if (ctx->mp_capable) {
603 /* this can't race with mptcp_close(), as the msk is
604 * not yet exposted to user-space
605 */
606 inet_sk_state_store((void *)new_msk, TCP_ESTABLISHED);
607
608 /* new mpc subflow takes ownership of the newly
609 * created mptcp socket
610 */
611 new_msk->sk_destruct = mptcp_sock_destruct;
612 mptcp_pm_new_connection(mptcp_sk(new_msk), 1);
613 mptcp_token_accept(subflow_req, mptcp_sk(new_msk));
614 ctx->conn = new_msk;
615 new_msk = NULL;
616
617 /* with OoO packets we can reach here without ingress
618 * mpc option
619 */
620 if (mp_opt.mp_capable)
621 mptcp_subflow_fully_established(ctx, &mp_opt);
622 } else if (ctx->mp_join) {
623 struct mptcp_sock *owner;
624
625 owner = subflow_req->msk;
626 if (!owner)
627 goto dispose_child;
628
629 /* move the msk reference ownership to the subflow */
630 subflow_req->msk = NULL;
631 ctx->conn = (struct sock *)owner;
632 if (!mptcp_finish_join(child))
633 goto dispose_child;
634
635 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINACKRX);
636 tcp_rsk(req)->drop_req = true;
637 }
638 }
639
640 out:
641 /* dispose of the left over mptcp master, if any */
642 if (unlikely(new_msk))
643 mptcp_force_close(new_msk);
644
645 /* check for expected invariant - should never trigger, just help
646 * catching eariler subtle bugs
647 */
648 WARN_ON_ONCE(child && *own_req && tcp_sk(child)->is_mptcp &&
649 (!mptcp_subflow_ctx(child) ||
650 !mptcp_subflow_ctx(child)->conn));
651 return child;
652
653 dispose_child:
654 subflow_drop_ctx(child);
655 tcp_rsk(req)->drop_req = true;
656 inet_csk_prepare_for_destroy_sock(child);
657 tcp_done(child);
658 req->rsk_ops->send_reset(sk, skb);
659
660 /* The last child reference will be released by the caller */
661 return child;
662 }
663
664 static struct inet_connection_sock_af_ops subflow_specific __ro_after_init;
665
666 enum mapping_status {
667 MAPPING_OK,
668 MAPPING_INVALID,
669 MAPPING_EMPTY,
670 MAPPING_DATA_FIN,
671 MAPPING_DUMMY
672 };
673
expand_seq(u64 old_seq,u16 old_data_len,u64 seq)674 static u64 expand_seq(u64 old_seq, u16 old_data_len, u64 seq)
675 {
676 if ((u32)seq == (u32)old_seq)
677 return old_seq;
678
679 /* Assume map covers data not mapped yet. */
680 return seq | ((old_seq + old_data_len + 1) & GENMASK_ULL(63, 32));
681 }
682
dbg_bad_map(struct mptcp_subflow_context * subflow,u32 ssn)683 static void dbg_bad_map(struct mptcp_subflow_context *subflow, u32 ssn)
684 {
685 pr_debug("Bad mapping: ssn=%d map_seq=%d map_data_len=%d",
686 ssn, subflow->map_subflow_seq, subflow->map_data_len);
687 }
688
skb_is_fully_mapped(struct sock * ssk,struct sk_buff * skb)689 static bool skb_is_fully_mapped(struct sock *ssk, struct sk_buff *skb)
690 {
691 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
692 unsigned int skb_consumed;
693
694 skb_consumed = tcp_sk(ssk)->copied_seq - TCP_SKB_CB(skb)->seq;
695 if (WARN_ON_ONCE(skb_consumed >= skb->len))
696 return true;
697
698 return skb->len - skb_consumed <= subflow->map_data_len -
699 mptcp_subflow_get_map_offset(subflow);
700 }
701
validate_mapping(struct sock * ssk,struct sk_buff * skb)702 static bool validate_mapping(struct sock *ssk, struct sk_buff *skb)
703 {
704 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
705 u32 ssn = tcp_sk(ssk)->copied_seq - subflow->ssn_offset;
706
707 if (unlikely(before(ssn, subflow->map_subflow_seq))) {
708 /* Mapping covers data later in the subflow stream,
709 * currently unsupported.
710 */
711 dbg_bad_map(subflow, ssn);
712 return false;
713 }
714 if (unlikely(!before(ssn, subflow->map_subflow_seq +
715 subflow->map_data_len))) {
716 /* Mapping does covers past subflow data, invalid */
717 dbg_bad_map(subflow, ssn);
718 return false;
719 }
720 return true;
721 }
722
get_mapping_status(struct sock * ssk,struct mptcp_sock * msk)723 static enum mapping_status get_mapping_status(struct sock *ssk,
724 struct mptcp_sock *msk)
725 {
726 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
727 struct mptcp_ext *mpext;
728 struct sk_buff *skb;
729 u16 data_len;
730 u64 map_seq;
731
732 skb = skb_peek(&ssk->sk_receive_queue);
733 if (!skb)
734 return MAPPING_EMPTY;
735
736 if (mptcp_check_fallback(ssk))
737 return MAPPING_DUMMY;
738
739 mpext = mptcp_get_ext(skb);
740 if (!mpext || !mpext->use_map) {
741 if (!subflow->map_valid && !skb->len) {
742 /* the TCP stack deliver 0 len FIN pkt to the receive
743 * queue, that is the only 0len pkts ever expected here,
744 * and we can admit no mapping only for 0 len pkts
745 */
746 if (!(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN))
747 WARN_ONCE(1, "0len seq %d:%d flags %x",
748 TCP_SKB_CB(skb)->seq,
749 TCP_SKB_CB(skb)->end_seq,
750 TCP_SKB_CB(skb)->tcp_flags);
751 sk_eat_skb(ssk, skb);
752 return MAPPING_EMPTY;
753 }
754
755 if (!subflow->map_valid)
756 return MAPPING_INVALID;
757
758 goto validate_seq;
759 }
760
761 pr_debug("seq=%llu is64=%d ssn=%u data_len=%u data_fin=%d",
762 mpext->data_seq, mpext->dsn64, mpext->subflow_seq,
763 mpext->data_len, mpext->data_fin);
764
765 data_len = mpext->data_len;
766 if (data_len == 0) {
767 MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_INFINITEMAPRX);
768 return MAPPING_INVALID;
769 }
770
771 if (mpext->data_fin == 1) {
772 if (data_len == 1) {
773 bool updated = mptcp_update_rcv_data_fin(msk, mpext->data_seq,
774 mpext->dsn64);
775 pr_debug("DATA_FIN with no payload seq=%llu", mpext->data_seq);
776 if (subflow->map_valid) {
777 /* A DATA_FIN might arrive in a DSS
778 * option before the previous mapping
779 * has been fully consumed. Continue
780 * handling the existing mapping.
781 */
782 skb_ext_del(skb, SKB_EXT_MPTCP);
783 return MAPPING_OK;
784 } else {
785 if (updated && schedule_work(&msk->work))
786 sock_hold((struct sock *)msk);
787
788 return MAPPING_DATA_FIN;
789 }
790 } else {
791 u64 data_fin_seq = mpext->data_seq + data_len - 1;
792
793 /* If mpext->data_seq is a 32-bit value, data_fin_seq
794 * must also be limited to 32 bits.
795 */
796 if (!mpext->dsn64)
797 data_fin_seq &= GENMASK_ULL(31, 0);
798
799 mptcp_update_rcv_data_fin(msk, data_fin_seq, mpext->dsn64);
800 pr_debug("DATA_FIN with mapping seq=%llu dsn64=%d",
801 data_fin_seq, mpext->dsn64);
802 }
803
804 /* Adjust for DATA_FIN using 1 byte of sequence space */
805 data_len--;
806 }
807
808 if (!mpext->dsn64) {
809 map_seq = expand_seq(subflow->map_seq, subflow->map_data_len,
810 mpext->data_seq);
811 pr_debug("expanded seq=%llu", subflow->map_seq);
812 } else {
813 map_seq = mpext->data_seq;
814 }
815 WRITE_ONCE(mptcp_sk(subflow->conn)->use_64bit_ack, !!mpext->dsn64);
816
817 if (subflow->map_valid) {
818 /* Allow replacing only with an identical map */
819 if (subflow->map_seq == map_seq &&
820 subflow->map_subflow_seq == mpext->subflow_seq &&
821 subflow->map_data_len == data_len) {
822 skb_ext_del(skb, SKB_EXT_MPTCP);
823 return MAPPING_OK;
824 }
825
826 /* If this skb data are fully covered by the current mapping,
827 * the new map would need caching, which is not supported
828 */
829 if (skb_is_fully_mapped(ssk, skb)) {
830 MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_DSSNOMATCH);
831 return MAPPING_INVALID;
832 }
833
834 /* will validate the next map after consuming the current one */
835 return MAPPING_OK;
836 }
837
838 subflow->map_seq = map_seq;
839 subflow->map_subflow_seq = mpext->subflow_seq;
840 subflow->map_data_len = data_len;
841 subflow->map_valid = 1;
842 subflow->mpc_map = mpext->mpc_map;
843 pr_debug("new map seq=%llu subflow_seq=%u data_len=%u",
844 subflow->map_seq, subflow->map_subflow_seq,
845 subflow->map_data_len);
846
847 validate_seq:
848 /* we revalidate valid mapping on new skb, because we must ensure
849 * the current skb is completely covered by the available mapping
850 */
851 if (!validate_mapping(ssk, skb))
852 return MAPPING_INVALID;
853
854 skb_ext_del(skb, SKB_EXT_MPTCP);
855 return MAPPING_OK;
856 }
857
mptcp_subflow_discard_data(struct sock * ssk,struct sk_buff * skb,u64 limit)858 static void mptcp_subflow_discard_data(struct sock *ssk, struct sk_buff *skb,
859 u64 limit)
860 {
861 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
862 bool fin = TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN;
863 u32 incr;
864
865 incr = limit >= skb->len ? skb->len + fin : limit;
866
867 pr_debug("discarding=%d len=%d seq=%d", incr, skb->len,
868 subflow->map_subflow_seq);
869 MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_DUPDATA);
870 tcp_sk(ssk)->copied_seq += incr;
871 if (!before(tcp_sk(ssk)->copied_seq, TCP_SKB_CB(skb)->end_seq))
872 sk_eat_skb(ssk, skb);
873 if (mptcp_subflow_get_map_offset(subflow) >= subflow->map_data_len)
874 subflow->map_valid = 0;
875 if (incr)
876 tcp_cleanup_rbuf(ssk, incr);
877 }
878
subflow_check_data_avail(struct sock * ssk)879 static bool subflow_check_data_avail(struct sock *ssk)
880 {
881 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
882 enum mapping_status status;
883 struct mptcp_sock *msk;
884 struct sk_buff *skb;
885
886 pr_debug("msk=%p ssk=%p data_avail=%d skb=%p", subflow->conn, ssk,
887 subflow->data_avail, skb_peek(&ssk->sk_receive_queue));
888 if (!skb_peek(&ssk->sk_receive_queue))
889 subflow->data_avail = 0;
890 if (subflow->data_avail)
891 return true;
892
893 msk = mptcp_sk(subflow->conn);
894 for (;;) {
895 u64 ack_seq;
896 u64 old_ack;
897
898 status = get_mapping_status(ssk, msk);
899 pr_debug("msk=%p ssk=%p status=%d", msk, ssk, status);
900 if (status == MAPPING_INVALID) {
901 ssk->sk_err = EBADMSG;
902 goto fatal;
903 }
904 if (status == MAPPING_DUMMY) {
905 __mptcp_do_fallback(msk);
906 skb = skb_peek(&ssk->sk_receive_queue);
907 subflow->map_valid = 1;
908 subflow->map_seq = READ_ONCE(msk->ack_seq);
909 subflow->map_data_len = skb->len;
910 subflow->map_subflow_seq = tcp_sk(ssk)->copied_seq -
911 subflow->ssn_offset;
912 subflow->data_avail = MPTCP_SUBFLOW_DATA_AVAIL;
913 return true;
914 }
915
916 if (status != MAPPING_OK)
917 return false;
918
919 skb = skb_peek(&ssk->sk_receive_queue);
920 if (WARN_ON_ONCE(!skb))
921 return false;
922
923 /* if msk lacks the remote key, this subflow must provide an
924 * MP_CAPABLE-based mapping
925 */
926 if (unlikely(!READ_ONCE(msk->can_ack))) {
927 if (!subflow->mpc_map) {
928 ssk->sk_err = EBADMSG;
929 goto fatal;
930 }
931 WRITE_ONCE(msk->remote_key, subflow->remote_key);
932 WRITE_ONCE(msk->ack_seq, subflow->map_seq);
933 WRITE_ONCE(msk->can_ack, true);
934 }
935
936 old_ack = READ_ONCE(msk->ack_seq);
937 ack_seq = mptcp_subflow_get_mapped_dsn(subflow);
938 pr_debug("msk ack_seq=%llx subflow ack_seq=%llx", old_ack,
939 ack_seq);
940 if (ack_seq == old_ack) {
941 subflow->data_avail = MPTCP_SUBFLOW_DATA_AVAIL;
942 break;
943 } else if (after64(ack_seq, old_ack)) {
944 subflow->data_avail = MPTCP_SUBFLOW_OOO_DATA;
945 break;
946 }
947
948 /* only accept in-sequence mapping. Old values are spurious
949 * retransmission
950 */
951 mptcp_subflow_discard_data(ssk, skb, old_ack - ack_seq);
952 }
953 return true;
954
955 fatal:
956 /* fatal protocol error, close the socket */
957 /* This barrier is coupled with smp_rmb() in tcp_poll() */
958 smp_wmb();
959 ssk->sk_error_report(ssk);
960 tcp_set_state(ssk, TCP_CLOSE);
961 tcp_send_active_reset(ssk, GFP_ATOMIC);
962 subflow->data_avail = 0;
963 return false;
964 }
965
mptcp_subflow_data_available(struct sock * sk)966 bool mptcp_subflow_data_available(struct sock *sk)
967 {
968 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
969
970 /* check if current mapping is still valid */
971 if (subflow->map_valid &&
972 mptcp_subflow_get_map_offset(subflow) >= subflow->map_data_len) {
973 subflow->map_valid = 0;
974 subflow->data_avail = 0;
975
976 pr_debug("Done with mapping: seq=%u data_len=%u",
977 subflow->map_subflow_seq,
978 subflow->map_data_len);
979 }
980
981 return subflow_check_data_avail(sk);
982 }
983
984 /* If ssk has an mptcp parent socket, use the mptcp rcvbuf occupancy,
985 * not the ssk one.
986 *
987 * In mptcp, rwin is about the mptcp-level connection data.
988 *
989 * Data that is still on the ssk rx queue can thus be ignored,
990 * as far as mptcp peer is concerened that data is still inflight.
991 * DSS ACK is updated when skb is moved to the mptcp rx queue.
992 */
mptcp_space(const struct sock * ssk,int * space,int * full_space)993 void mptcp_space(const struct sock *ssk, int *space, int *full_space)
994 {
995 const struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
996 const struct sock *sk = subflow->conn;
997
998 *space = tcp_space(sk);
999 *full_space = tcp_full_space(sk);
1000 }
1001
subflow_data_ready(struct sock * sk)1002 static void subflow_data_ready(struct sock *sk)
1003 {
1004 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
1005 u16 state = 1 << inet_sk_state_load(sk);
1006 struct sock *parent = subflow->conn;
1007 struct mptcp_sock *msk;
1008
1009 msk = mptcp_sk(parent);
1010 if (state & TCPF_LISTEN) {
1011 /* MPJ subflow are removed from accept queue before reaching here,
1012 * avoid stray wakeups
1013 */
1014 if (reqsk_queue_empty(&inet_csk(sk)->icsk_accept_queue))
1015 return;
1016
1017 set_bit(MPTCP_DATA_READY, &msk->flags);
1018 parent->sk_data_ready(parent);
1019 return;
1020 }
1021
1022 WARN_ON_ONCE(!__mptcp_check_fallback(msk) && !subflow->mp_capable &&
1023 !subflow->mp_join && !(state & TCPF_CLOSE));
1024
1025 if (mptcp_subflow_data_available(sk))
1026 mptcp_data_ready(parent, sk);
1027 }
1028
subflow_write_space(struct sock * sk)1029 static void subflow_write_space(struct sock *sk)
1030 {
1031 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
1032 struct sock *parent = subflow->conn;
1033
1034 if (!sk_stream_is_writeable(sk))
1035 return;
1036
1037 if (sk_stream_is_writeable(parent)) {
1038 set_bit(MPTCP_SEND_SPACE, &mptcp_sk(parent)->flags);
1039 smp_mb__after_atomic();
1040 /* set SEND_SPACE before sk_stream_write_space clears NOSPACE */
1041 sk_stream_write_space(parent);
1042 }
1043 }
1044
1045 static const struct inet_connection_sock_af_ops *
subflow_default_af_ops(struct sock * sk)1046 subflow_default_af_ops(struct sock *sk)
1047 {
1048 #if IS_ENABLED(CONFIG_MPTCP_IPV6)
1049 if (sk->sk_family == AF_INET6)
1050 return &subflow_v6_specific;
1051 #endif
1052 return &subflow_specific;
1053 }
1054
1055 #if IS_ENABLED(CONFIG_MPTCP_IPV6)
mptcpv6_handle_mapped(struct sock * sk,bool mapped)1056 void mptcpv6_handle_mapped(struct sock *sk, bool mapped)
1057 {
1058 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
1059 struct inet_connection_sock *icsk = inet_csk(sk);
1060 const struct inet_connection_sock_af_ops *target;
1061
1062 target = mapped ? &subflow_v6m_specific : subflow_default_af_ops(sk);
1063
1064 pr_debug("subflow=%p family=%d ops=%p target=%p mapped=%d",
1065 subflow, sk->sk_family, icsk->icsk_af_ops, target, mapped);
1066
1067 if (likely(icsk->icsk_af_ops == target))
1068 return;
1069
1070 subflow->icsk_af_ops = icsk->icsk_af_ops;
1071 icsk->icsk_af_ops = target;
1072 }
1073 #endif
1074
mptcp_info2sockaddr(const struct mptcp_addr_info * info,struct sockaddr_storage * addr)1075 static void mptcp_info2sockaddr(const struct mptcp_addr_info *info,
1076 struct sockaddr_storage *addr)
1077 {
1078 memset(addr, 0, sizeof(*addr));
1079 addr->ss_family = info->family;
1080 if (addr->ss_family == AF_INET) {
1081 struct sockaddr_in *in_addr = (struct sockaddr_in *)addr;
1082
1083 in_addr->sin_addr = info->addr;
1084 in_addr->sin_port = info->port;
1085 }
1086 #if IS_ENABLED(CONFIG_MPTCP_IPV6)
1087 else if (addr->ss_family == AF_INET6) {
1088 struct sockaddr_in6 *in6_addr = (struct sockaddr_in6 *)addr;
1089
1090 in6_addr->sin6_addr = info->addr6;
1091 in6_addr->sin6_port = info->port;
1092 }
1093 #endif
1094 }
1095
__mptcp_subflow_connect(struct sock * sk,const struct mptcp_addr_info * loc,const struct mptcp_addr_info * remote)1096 int __mptcp_subflow_connect(struct sock *sk, const struct mptcp_addr_info *loc,
1097 const struct mptcp_addr_info *remote)
1098 {
1099 struct mptcp_sock *msk = mptcp_sk(sk);
1100 struct mptcp_subflow_context *subflow;
1101 struct sockaddr_storage addr;
1102 int remote_id = remote->id;
1103 int local_id = loc->id;
1104 struct socket *sf;
1105 struct sock *ssk;
1106 u32 remote_token;
1107 int addrlen;
1108 int err;
1109
1110 if (!mptcp_is_fully_established(sk))
1111 return -ENOTCONN;
1112
1113 err = mptcp_subflow_create_socket(sk, &sf);
1114 if (err)
1115 return err;
1116
1117 ssk = sf->sk;
1118 subflow = mptcp_subflow_ctx(ssk);
1119 do {
1120 get_random_bytes(&subflow->local_nonce, sizeof(u32));
1121 } while (!subflow->local_nonce);
1122
1123 if (!local_id) {
1124 err = mptcp_pm_get_local_id(msk, (struct sock_common *)ssk);
1125 if (err < 0)
1126 goto failed;
1127
1128 local_id = err;
1129 }
1130
1131 subflow->remote_key = msk->remote_key;
1132 subflow->local_key = msk->local_key;
1133 subflow->token = msk->token;
1134 mptcp_info2sockaddr(loc, &addr);
1135
1136 addrlen = sizeof(struct sockaddr_in);
1137 #if IS_ENABLED(CONFIG_MPTCP_IPV6)
1138 if (loc->family == AF_INET6)
1139 addrlen = sizeof(struct sockaddr_in6);
1140 #endif
1141 ssk->sk_bound_dev_if = loc->ifindex;
1142 err = kernel_bind(sf, (struct sockaddr *)&addr, addrlen);
1143 if (err)
1144 goto failed;
1145
1146 mptcp_crypto_key_sha(subflow->remote_key, &remote_token, NULL);
1147 pr_debug("msk=%p remote_token=%u local_id=%d remote_id=%d", msk,
1148 remote_token, local_id, remote_id);
1149 subflow->remote_token = remote_token;
1150 subflow->local_id = local_id;
1151 subflow->remote_id = remote_id;
1152 subflow->request_join = 1;
1153 subflow->request_bkup = !!(loc->flags & MPTCP_PM_ADDR_FLAG_BACKUP);
1154 mptcp_info2sockaddr(remote, &addr);
1155
1156 err = kernel_connect(sf, (struct sockaddr *)&addr, addrlen, O_NONBLOCK);
1157 if (err && err != -EINPROGRESS)
1158 goto failed;
1159
1160 spin_lock_bh(&msk->join_list_lock);
1161 list_add_tail(&subflow->node, &msk->join_list);
1162 spin_unlock_bh(&msk->join_list_lock);
1163
1164 return err;
1165
1166 failed:
1167 sock_release(sf);
1168 return err;
1169 }
1170
mptcp_subflow_create_socket(struct sock * sk,struct socket ** new_sock)1171 int mptcp_subflow_create_socket(struct sock *sk, struct socket **new_sock)
1172 {
1173 struct mptcp_subflow_context *subflow;
1174 struct net *net = sock_net(sk);
1175 struct socket *sf;
1176 int err;
1177
1178 /* un-accepted server sockets can reach here - on bad configuration
1179 * bail early to avoid greater trouble later
1180 */
1181 if (unlikely(!sk->sk_socket))
1182 return -EINVAL;
1183
1184 err = sock_create_kern(net, sk->sk_family, SOCK_STREAM, IPPROTO_TCP,
1185 &sf);
1186 if (err)
1187 return err;
1188
1189 lock_sock(sf->sk);
1190
1191 /* kernel sockets do not by default acquire net ref, but TCP timer
1192 * needs it.
1193 */
1194 sf->sk->sk_net_refcnt = 1;
1195 get_net(net);
1196 #ifdef CONFIG_PROC_FS
1197 this_cpu_add(*net->core.sock_inuse, 1);
1198 #endif
1199 err = tcp_set_ulp(sf->sk, "mptcp");
1200 release_sock(sf->sk);
1201
1202 if (err) {
1203 sock_release(sf);
1204 return err;
1205 }
1206
1207 /* the newly created socket really belongs to the owning MPTCP master
1208 * socket, even if for additional subflows the allocation is performed
1209 * by a kernel workqueue. Adjust inode references, so that the
1210 * procfs/diag interaces really show this one belonging to the correct
1211 * user.
1212 */
1213 SOCK_INODE(sf)->i_ino = SOCK_INODE(sk->sk_socket)->i_ino;
1214 SOCK_INODE(sf)->i_uid = SOCK_INODE(sk->sk_socket)->i_uid;
1215 SOCK_INODE(sf)->i_gid = SOCK_INODE(sk->sk_socket)->i_gid;
1216
1217 subflow = mptcp_subflow_ctx(sf->sk);
1218 pr_debug("subflow=%p", subflow);
1219
1220 *new_sock = sf;
1221 sock_hold(sk);
1222 subflow->conn = sk;
1223
1224 return 0;
1225 }
1226
subflow_create_ctx(struct sock * sk,gfp_t priority)1227 static struct mptcp_subflow_context *subflow_create_ctx(struct sock *sk,
1228 gfp_t priority)
1229 {
1230 struct inet_connection_sock *icsk = inet_csk(sk);
1231 struct mptcp_subflow_context *ctx;
1232
1233 ctx = kzalloc(sizeof(*ctx), priority);
1234 if (!ctx)
1235 return NULL;
1236
1237 rcu_assign_pointer(icsk->icsk_ulp_data, ctx);
1238 INIT_LIST_HEAD(&ctx->node);
1239
1240 pr_debug("subflow=%p", ctx);
1241
1242 ctx->tcp_sock = sk;
1243
1244 return ctx;
1245 }
1246
__subflow_state_change(struct sock * sk)1247 static void __subflow_state_change(struct sock *sk)
1248 {
1249 struct socket_wq *wq;
1250
1251 rcu_read_lock();
1252 wq = rcu_dereference(sk->sk_wq);
1253 if (skwq_has_sleeper(wq))
1254 wake_up_interruptible_all(&wq->wait);
1255 rcu_read_unlock();
1256 }
1257
subflow_is_done(const struct sock * sk)1258 static bool subflow_is_done(const struct sock *sk)
1259 {
1260 return sk->sk_shutdown & RCV_SHUTDOWN || sk->sk_state == TCP_CLOSE;
1261 }
1262
subflow_state_change(struct sock * sk)1263 static void subflow_state_change(struct sock *sk)
1264 {
1265 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
1266 struct sock *parent = subflow->conn;
1267
1268 __subflow_state_change(sk);
1269
1270 if (subflow_simultaneous_connect(sk)) {
1271 mptcp_do_fallback(sk);
1272 mptcp_rcv_space_init(mptcp_sk(parent), sk);
1273 pr_fallback(mptcp_sk(parent));
1274 subflow->conn_finished = 1;
1275 if (inet_sk_state_load(parent) == TCP_SYN_SENT) {
1276 inet_sk_state_store(parent, TCP_ESTABLISHED);
1277 parent->sk_state_change(parent);
1278 }
1279 }
1280
1281 /* as recvmsg() does not acquire the subflow socket for ssk selection
1282 * a fin packet carrying a DSS can be unnoticed if we don't trigger
1283 * the data available machinery here.
1284 */
1285 if (mptcp_subflow_data_available(sk))
1286 mptcp_data_ready(parent, sk);
1287
1288 if (__mptcp_check_fallback(mptcp_sk(parent)) &&
1289 !(parent->sk_shutdown & RCV_SHUTDOWN) &&
1290 !subflow->rx_eof && subflow_is_done(sk)) {
1291 subflow->rx_eof = 1;
1292 mptcp_subflow_eof(parent);
1293 }
1294 }
1295
subflow_ulp_init(struct sock * sk)1296 static int subflow_ulp_init(struct sock *sk)
1297 {
1298 struct inet_connection_sock *icsk = inet_csk(sk);
1299 struct mptcp_subflow_context *ctx;
1300 struct tcp_sock *tp = tcp_sk(sk);
1301 int err = 0;
1302
1303 /* disallow attaching ULP to a socket unless it has been
1304 * created with sock_create_kern()
1305 */
1306 if (!sk->sk_kern_sock) {
1307 err = -EOPNOTSUPP;
1308 goto out;
1309 }
1310
1311 ctx = subflow_create_ctx(sk, GFP_KERNEL);
1312 if (!ctx) {
1313 err = -ENOMEM;
1314 goto out;
1315 }
1316
1317 pr_debug("subflow=%p, family=%d", ctx, sk->sk_family);
1318
1319 tp->is_mptcp = 1;
1320 ctx->icsk_af_ops = icsk->icsk_af_ops;
1321 icsk->icsk_af_ops = subflow_default_af_ops(sk);
1322 ctx->tcp_data_ready = sk->sk_data_ready;
1323 ctx->tcp_state_change = sk->sk_state_change;
1324 ctx->tcp_write_space = sk->sk_write_space;
1325 sk->sk_data_ready = subflow_data_ready;
1326 sk->sk_write_space = subflow_write_space;
1327 sk->sk_state_change = subflow_state_change;
1328 out:
1329 return err;
1330 }
1331
subflow_ulp_release(struct sock * sk)1332 static void subflow_ulp_release(struct sock *sk)
1333 {
1334 struct mptcp_subflow_context *ctx = mptcp_subflow_ctx(sk);
1335
1336 if (!ctx)
1337 return;
1338
1339 if (ctx->conn)
1340 sock_put(ctx->conn);
1341
1342 kfree_rcu(ctx, rcu);
1343 }
1344
subflow_ulp_clone(const struct request_sock * req,struct sock * newsk,const gfp_t priority)1345 static void subflow_ulp_clone(const struct request_sock *req,
1346 struct sock *newsk,
1347 const gfp_t priority)
1348 {
1349 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req);
1350 struct mptcp_subflow_context *old_ctx = mptcp_subflow_ctx(newsk);
1351 struct mptcp_subflow_context *new_ctx;
1352
1353 if (!tcp_rsk(req)->is_mptcp ||
1354 (!subflow_req->mp_capable && !subflow_req->mp_join)) {
1355 subflow_ulp_fallback(newsk, old_ctx);
1356 return;
1357 }
1358
1359 new_ctx = subflow_create_ctx(newsk, priority);
1360 if (!new_ctx) {
1361 subflow_ulp_fallback(newsk, old_ctx);
1362 return;
1363 }
1364
1365 new_ctx->conn_finished = 1;
1366 new_ctx->icsk_af_ops = old_ctx->icsk_af_ops;
1367 new_ctx->tcp_data_ready = old_ctx->tcp_data_ready;
1368 new_ctx->tcp_state_change = old_ctx->tcp_state_change;
1369 new_ctx->tcp_write_space = old_ctx->tcp_write_space;
1370 new_ctx->rel_write_seq = 1;
1371 new_ctx->tcp_sock = newsk;
1372
1373 if (subflow_req->mp_capable) {
1374 /* see comments in subflow_syn_recv_sock(), MPTCP connection
1375 * is fully established only after we receive the remote key
1376 */
1377 new_ctx->mp_capable = 1;
1378 new_ctx->local_key = subflow_req->local_key;
1379 new_ctx->token = subflow_req->token;
1380 new_ctx->ssn_offset = subflow_req->ssn_offset;
1381 new_ctx->idsn = subflow_req->idsn;
1382 } else if (subflow_req->mp_join) {
1383 new_ctx->ssn_offset = subflow_req->ssn_offset;
1384 new_ctx->mp_join = 1;
1385 new_ctx->fully_established = 1;
1386 new_ctx->backup = subflow_req->backup;
1387 new_ctx->local_id = subflow_req->local_id;
1388 new_ctx->remote_id = subflow_req->remote_id;
1389 new_ctx->token = subflow_req->token;
1390 new_ctx->thmac = subflow_req->thmac;
1391 }
1392 }
1393
1394 static struct tcp_ulp_ops subflow_ulp_ops __read_mostly = {
1395 .name = "mptcp",
1396 .owner = THIS_MODULE,
1397 .init = subflow_ulp_init,
1398 .release = subflow_ulp_release,
1399 .clone = subflow_ulp_clone,
1400 };
1401
subflow_ops_init(struct request_sock_ops * subflow_ops)1402 static int subflow_ops_init(struct request_sock_ops *subflow_ops)
1403 {
1404 subflow_ops->obj_size = sizeof(struct mptcp_subflow_request_sock);
1405
1406 subflow_ops->slab = kmem_cache_create(subflow_ops->slab_name,
1407 subflow_ops->obj_size, 0,
1408 SLAB_ACCOUNT |
1409 SLAB_TYPESAFE_BY_RCU,
1410 NULL);
1411 if (!subflow_ops->slab)
1412 return -ENOMEM;
1413
1414 return 0;
1415 }
1416
mptcp_subflow_init(void)1417 void __init mptcp_subflow_init(void)
1418 {
1419 mptcp_subflow_v4_request_sock_ops = tcp_request_sock_ops;
1420 mptcp_subflow_v4_request_sock_ops.slab_name = "request_sock_subflow_v4";
1421 mptcp_subflow_v4_request_sock_ops.destructor = subflow_v4_req_destructor;
1422
1423 if (subflow_ops_init(&mptcp_subflow_v4_request_sock_ops) != 0)
1424 panic("MPTCP: failed to init subflow v4 request sock ops\n");
1425
1426 subflow_request_sock_ipv4_ops = tcp_request_sock_ipv4_ops;
1427 subflow_request_sock_ipv4_ops.init_req = subflow_v4_init_req;
1428
1429 subflow_specific = ipv4_specific;
1430 subflow_specific.conn_request = subflow_v4_conn_request;
1431 subflow_specific.syn_recv_sock = subflow_syn_recv_sock;
1432 subflow_specific.sk_rx_dst_set = subflow_finish_connect;
1433
1434 #if IS_ENABLED(CONFIG_MPTCP_IPV6)
1435 /* In struct mptcp_subflow_request_sock, we assume the TCP request sock
1436 * structures for v4 and v6 have the same size. It should not changed in
1437 * the future but better to make sure to be warned if it is no longer
1438 * the case.
1439 */
1440 BUILD_BUG_ON(sizeof(struct tcp_request_sock) != sizeof(struct tcp6_request_sock));
1441
1442 mptcp_subflow_v6_request_sock_ops = tcp6_request_sock_ops;
1443 mptcp_subflow_v6_request_sock_ops.slab_name = "request_sock_subflow_v6";
1444 mptcp_subflow_v6_request_sock_ops.destructor = subflow_v6_req_destructor;
1445
1446 if (subflow_ops_init(&mptcp_subflow_v6_request_sock_ops) != 0)
1447 panic("MPTCP: failed to init subflow v6 request sock ops\n");
1448
1449 subflow_request_sock_ipv6_ops = tcp_request_sock_ipv6_ops;
1450 subflow_request_sock_ipv6_ops.init_req = subflow_v6_init_req;
1451
1452 subflow_v6_specific = ipv6_specific;
1453 subflow_v6_specific.conn_request = subflow_v6_conn_request;
1454 subflow_v6_specific.syn_recv_sock = subflow_syn_recv_sock;
1455 subflow_v6_specific.sk_rx_dst_set = subflow_finish_connect;
1456
1457 subflow_v6m_specific = subflow_v6_specific;
1458 subflow_v6m_specific.queue_xmit = ipv4_specific.queue_xmit;
1459 subflow_v6m_specific.send_check = ipv4_specific.send_check;
1460 subflow_v6m_specific.net_header_len = ipv4_specific.net_header_len;
1461 subflow_v6m_specific.mtu_reduced = ipv4_specific.mtu_reduced;
1462 subflow_v6m_specific.net_frag_header_len = 0;
1463 #endif
1464
1465 mptcp_diag_subflow_init(&subflow_ulp_ops);
1466
1467 if (tcp_register_ulp(&subflow_ulp_ops) != 0)
1468 panic("MPTCP: failed to register subflows to ULP\n");
1469 }
1470