• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 // SPDX-License-Identifier: GPL-2.0
2 /* Multipath TCP
3  *
4  * Copyright (c) 2017 - 2019, Intel Corporation.
5  */
6 
7 #define pr_fmt(fmt) "MPTCP: " fmt
8 
9 #include <linux/kernel.h>
10 #include <linux/module.h>
11 #include <linux/netdevice.h>
12 #include <crypto/algapi.h>
13 #include <crypto/sha.h>
14 #include <net/sock.h>
15 #include <net/inet_common.h>
16 #include <net/inet_hashtables.h>
17 #include <net/protocol.h>
18 #include <net/tcp.h>
19 #if IS_ENABLED(CONFIG_MPTCP_IPV6)
20 #include <net/ip6_route.h>
21 #endif
22 #include <net/mptcp.h>
23 #include <uapi/linux/mptcp.h>
24 #include "protocol.h"
25 #include "mib.h"
26 
SUBFLOW_REQ_INC_STATS(struct request_sock * req,enum linux_mptcp_mib_field field)27 static void SUBFLOW_REQ_INC_STATS(struct request_sock *req,
28 				  enum linux_mptcp_mib_field field)
29 {
30 	MPTCP_INC_STATS(sock_net(req_to_sk(req)), field);
31 }
32 
subflow_req_destructor(struct request_sock * req)33 static void subflow_req_destructor(struct request_sock *req)
34 {
35 	struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req);
36 
37 	pr_debug("subflow_req=%p", subflow_req);
38 
39 	if (subflow_req->msk)
40 		sock_put((struct sock *)subflow_req->msk);
41 
42 	mptcp_token_destroy_request(req);
43 }
44 
subflow_generate_hmac(u64 key1,u64 key2,u32 nonce1,u32 nonce2,void * hmac)45 static void subflow_generate_hmac(u64 key1, u64 key2, u32 nonce1, u32 nonce2,
46 				  void *hmac)
47 {
48 	u8 msg[8];
49 
50 	put_unaligned_be32(nonce1, &msg[0]);
51 	put_unaligned_be32(nonce2, &msg[4]);
52 
53 	mptcp_crypto_hmac_sha(key1, key2, msg, 8, hmac);
54 }
55 
mptcp_can_accept_new_subflow(const struct mptcp_sock * msk)56 static bool mptcp_can_accept_new_subflow(const struct mptcp_sock *msk)
57 {
58 	return mptcp_is_fully_established((void *)msk) &&
59 	       READ_ONCE(msk->pm.accept_subflow);
60 }
61 
62 /* validate received token and create truncated hmac and nonce for SYN-ACK */
subflow_token_join_request(struct request_sock * req,const struct sk_buff * skb)63 static struct mptcp_sock *subflow_token_join_request(struct request_sock *req,
64 						     const struct sk_buff *skb)
65 {
66 	struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req);
67 	u8 hmac[SHA256_DIGEST_SIZE];
68 	struct mptcp_sock *msk;
69 	int local_id;
70 
71 	msk = mptcp_token_get_sock(sock_net(req_to_sk(req)), subflow_req->token);
72 	if (!msk) {
73 		SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINNOTOKEN);
74 		return NULL;
75 	}
76 
77 	local_id = mptcp_pm_get_local_id(msk, (struct sock_common *)req);
78 	if (local_id < 0) {
79 		sock_put((struct sock *)msk);
80 		return NULL;
81 	}
82 	subflow_req->local_id = local_id;
83 
84 	get_random_bytes(&subflow_req->local_nonce, sizeof(u32));
85 
86 	subflow_generate_hmac(msk->local_key, msk->remote_key,
87 			      subflow_req->local_nonce,
88 			      subflow_req->remote_nonce, hmac);
89 
90 	subflow_req->thmac = get_unaligned_be64(hmac);
91 	return msk;
92 }
93 
__subflow_init_req(struct request_sock * req,const struct sock * sk_listener)94 static int __subflow_init_req(struct request_sock *req, const struct sock *sk_listener)
95 {
96 	struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req);
97 
98 	subflow_req->mp_capable = 0;
99 	subflow_req->mp_join = 0;
100 	subflow_req->msk = NULL;
101 	mptcp_token_init_request(req);
102 
103 #ifdef CONFIG_TCP_MD5SIG
104 	/* no MPTCP if MD5SIG is enabled on this socket or we may run out of
105 	 * TCP option space.
106 	 */
107 	if (rcu_access_pointer(tcp_sk(sk_listener)->md5sig_info))
108 		return -EINVAL;
109 #endif
110 
111 	return 0;
112 }
113 
subflow_init_req(struct request_sock * req,const struct sock * sk_listener,struct sk_buff * skb)114 static void subflow_init_req(struct request_sock *req,
115 			     const struct sock *sk_listener,
116 			     struct sk_buff *skb)
117 {
118 	struct mptcp_subflow_context *listener = mptcp_subflow_ctx(sk_listener);
119 	struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req);
120 	struct mptcp_options_received mp_opt;
121 	int ret;
122 
123 	pr_debug("subflow_req=%p, listener=%p", subflow_req, listener);
124 
125 	ret = __subflow_init_req(req, sk_listener);
126 	if (ret)
127 		return;
128 
129 	mptcp_get_options(skb, &mp_opt);
130 
131 	if (mp_opt.mp_capable) {
132 		SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_MPCAPABLEPASSIVE);
133 
134 		if (mp_opt.mp_join)
135 			return;
136 	} else if (mp_opt.mp_join) {
137 		SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINSYNRX);
138 	}
139 
140 	if (mp_opt.mp_capable && listener->request_mptcp) {
141 		int err, retries = 4;
142 
143 		subflow_req->ssn_offset = TCP_SKB_CB(skb)->seq;
144 again:
145 		do {
146 			get_random_bytes(&subflow_req->local_key, sizeof(subflow_req->local_key));
147 		} while (subflow_req->local_key == 0);
148 
149 		if (unlikely(req->syncookie)) {
150 			mptcp_crypto_key_sha(subflow_req->local_key,
151 					     &subflow_req->token,
152 					     &subflow_req->idsn);
153 			if (mptcp_token_exists(subflow_req->token)) {
154 				if (retries-- > 0)
155 					goto again;
156 			} else {
157 				subflow_req->mp_capable = 1;
158 			}
159 			return;
160 		}
161 
162 		err = mptcp_token_new_request(req);
163 		if (err == 0)
164 			subflow_req->mp_capable = 1;
165 		else if (retries-- > 0)
166 			goto again;
167 
168 	} else if (mp_opt.mp_join && listener->request_mptcp) {
169 		subflow_req->ssn_offset = TCP_SKB_CB(skb)->seq;
170 		subflow_req->mp_join = 1;
171 		subflow_req->backup = mp_opt.backup;
172 		subflow_req->remote_id = mp_opt.join_id;
173 		subflow_req->token = mp_opt.token;
174 		subflow_req->remote_nonce = mp_opt.nonce;
175 		subflow_req->msk = subflow_token_join_request(req, skb);
176 
177 		if (unlikely(req->syncookie) && subflow_req->msk) {
178 			if (mptcp_can_accept_new_subflow(subflow_req->msk))
179 				subflow_init_req_cookie_join_save(subflow_req, skb);
180 		}
181 
182 		pr_debug("token=%u, remote_nonce=%u msk=%p", subflow_req->token,
183 			 subflow_req->remote_nonce, subflow_req->msk);
184 	}
185 }
186 
mptcp_subflow_init_cookie_req(struct request_sock * req,const struct sock * sk_listener,struct sk_buff * skb)187 int mptcp_subflow_init_cookie_req(struct request_sock *req,
188 				  const struct sock *sk_listener,
189 				  struct sk_buff *skb)
190 {
191 	struct mptcp_subflow_context *listener = mptcp_subflow_ctx(sk_listener);
192 	struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req);
193 	struct mptcp_options_received mp_opt;
194 	int err;
195 
196 	err = __subflow_init_req(req, sk_listener);
197 	if (err)
198 		return err;
199 
200 	mptcp_get_options(skb, &mp_opt);
201 
202 	if (mp_opt.mp_capable && mp_opt.mp_join)
203 		return -EINVAL;
204 
205 	if (mp_opt.mp_capable && listener->request_mptcp) {
206 		if (mp_opt.sndr_key == 0)
207 			return -EINVAL;
208 
209 		subflow_req->local_key = mp_opt.rcvr_key;
210 		err = mptcp_token_new_request(req);
211 		if (err)
212 			return err;
213 
214 		subflow_req->mp_capable = 1;
215 		subflow_req->ssn_offset = TCP_SKB_CB(skb)->seq - 1;
216 	} else if (mp_opt.mp_join && listener->request_mptcp) {
217 		if (!mptcp_token_join_cookie_init_state(subflow_req, skb))
218 			return -EINVAL;
219 
220 		if (mptcp_can_accept_new_subflow(subflow_req->msk))
221 			subflow_req->mp_join = 1;
222 
223 		subflow_req->ssn_offset = TCP_SKB_CB(skb)->seq - 1;
224 	}
225 
226 	return 0;
227 }
228 EXPORT_SYMBOL_GPL(mptcp_subflow_init_cookie_req);
229 
subflow_v4_init_req(struct request_sock * req,const struct sock * sk_listener,struct sk_buff * skb)230 static void subflow_v4_init_req(struct request_sock *req,
231 				const struct sock *sk_listener,
232 				struct sk_buff *skb)
233 {
234 	tcp_rsk(req)->is_mptcp = 1;
235 
236 	tcp_request_sock_ipv4_ops.init_req(req, sk_listener, skb);
237 
238 	subflow_init_req(req, sk_listener, skb);
239 }
240 
241 #if IS_ENABLED(CONFIG_MPTCP_IPV6)
subflow_v6_init_req(struct request_sock * req,const struct sock * sk_listener,struct sk_buff * skb)242 static void subflow_v6_init_req(struct request_sock *req,
243 				const struct sock *sk_listener,
244 				struct sk_buff *skb)
245 {
246 	tcp_rsk(req)->is_mptcp = 1;
247 
248 	tcp_request_sock_ipv6_ops.init_req(req, sk_listener, skb);
249 
250 	subflow_init_req(req, sk_listener, skb);
251 }
252 #endif
253 
254 /* validate received truncated hmac and create hmac for third ACK */
subflow_thmac_valid(struct mptcp_subflow_context * subflow)255 static bool subflow_thmac_valid(struct mptcp_subflow_context *subflow)
256 {
257 	u8 hmac[SHA256_DIGEST_SIZE];
258 	u64 thmac;
259 
260 	subflow_generate_hmac(subflow->remote_key, subflow->local_key,
261 			      subflow->remote_nonce, subflow->local_nonce,
262 			      hmac);
263 
264 	thmac = get_unaligned_be64(hmac);
265 	pr_debug("subflow=%p, token=%u, thmac=%llu, subflow->thmac=%llu\n",
266 		 subflow, subflow->token,
267 		 (unsigned long long)thmac,
268 		 (unsigned long long)subflow->thmac);
269 
270 	return thmac == subflow->thmac;
271 }
272 
mptcp_subflow_reset(struct sock * ssk)273 void mptcp_subflow_reset(struct sock *ssk)
274 {
275 	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
276 	struct sock *sk = subflow->conn;
277 
278 	tcp_set_state(ssk, TCP_CLOSE);
279 	tcp_send_active_reset(ssk, GFP_ATOMIC);
280 	tcp_done(ssk);
281 	if (!test_and_set_bit(MPTCP_WORK_CLOSE_SUBFLOW, &mptcp_sk(sk)->flags) &&
282 	    schedule_work(&mptcp_sk(sk)->work))
283 		sock_hold(sk);
284 }
285 
subflow_finish_connect(struct sock * sk,const struct sk_buff * skb)286 static void subflow_finish_connect(struct sock *sk, const struct sk_buff *skb)
287 {
288 	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
289 	struct mptcp_options_received mp_opt;
290 	struct sock *parent = subflow->conn;
291 
292 	subflow->icsk_af_ops->sk_rx_dst_set(sk, skb);
293 
294 	if (inet_sk_state_load(parent) == TCP_SYN_SENT) {
295 		inet_sk_state_store(parent, TCP_ESTABLISHED);
296 		parent->sk_state_change(parent);
297 	}
298 
299 	/* be sure no special action on any packet other than syn-ack */
300 	if (subflow->conn_finished)
301 		return;
302 
303 	subflow->rel_write_seq = 1;
304 	subflow->conn_finished = 1;
305 	subflow->ssn_offset = TCP_SKB_CB(skb)->seq;
306 	pr_debug("subflow=%p synack seq=%x", subflow, subflow->ssn_offset);
307 
308 	mptcp_get_options(skb, &mp_opt);
309 	if (subflow->request_mptcp) {
310 		if (!mp_opt.mp_capable) {
311 			MPTCP_INC_STATS(sock_net(sk),
312 					MPTCP_MIB_MPCAPABLEACTIVEFALLBACK);
313 			mptcp_do_fallback(sk);
314 			pr_fallback(mptcp_sk(subflow->conn));
315 			goto fallback;
316 		}
317 
318 		subflow->mp_capable = 1;
319 		subflow->can_ack = 1;
320 		subflow->remote_key = mp_opt.sndr_key;
321 		pr_debug("subflow=%p, remote_key=%llu", subflow,
322 			 subflow->remote_key);
323 		mptcp_finish_connect(sk);
324 	} else if (subflow->request_join) {
325 		u8 hmac[SHA256_DIGEST_SIZE];
326 
327 		if (!mp_opt.mp_join)
328 			goto do_reset;
329 
330 		subflow->thmac = mp_opt.thmac;
331 		subflow->remote_nonce = mp_opt.nonce;
332 		pr_debug("subflow=%p, thmac=%llu, remote_nonce=%u", subflow,
333 			 subflow->thmac, subflow->remote_nonce);
334 
335 		if (!subflow_thmac_valid(subflow)) {
336 			MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_JOINACKMAC);
337 			goto do_reset;
338 		}
339 
340 		if (!mptcp_finish_join(sk))
341 			goto do_reset;
342 
343 		subflow_generate_hmac(subflow->local_key, subflow->remote_key,
344 				      subflow->local_nonce,
345 				      subflow->remote_nonce,
346 				      hmac);
347 		memcpy(subflow->hmac, hmac, MPTCPOPT_HMAC_LEN);
348 
349 		subflow->mp_join = 1;
350 		MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_JOINSYNACKRX);
351 	} else if (mptcp_check_fallback(sk)) {
352 fallback:
353 		mptcp_rcv_space_init(mptcp_sk(parent), sk);
354 	}
355 	return;
356 
357 do_reset:
358 	mptcp_subflow_reset(sk);
359 }
360 
361 static struct request_sock_ops mptcp_subflow_v4_request_sock_ops __ro_after_init;
362 static struct tcp_request_sock_ops subflow_request_sock_ipv4_ops __ro_after_init;
363 
subflow_v4_conn_request(struct sock * sk,struct sk_buff * skb)364 static int subflow_v4_conn_request(struct sock *sk, struct sk_buff *skb)
365 {
366 	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
367 
368 	pr_debug("subflow=%p", subflow);
369 
370 	/* Never answer to SYNs sent to broadcast or multicast */
371 	if (skb_rtable(skb)->rt_flags & (RTCF_BROADCAST | RTCF_MULTICAST))
372 		goto drop;
373 
374 	return tcp_conn_request(&mptcp_subflow_v4_request_sock_ops,
375 				&subflow_request_sock_ipv4_ops,
376 				sk, skb);
377 drop:
378 	tcp_listendrop(sk);
379 	return 0;
380 }
381 
subflow_v4_req_destructor(struct request_sock * req)382 static void subflow_v4_req_destructor(struct request_sock *req)
383 {
384 	subflow_req_destructor(req);
385 	tcp_request_sock_ops.destructor(req);
386 }
387 
388 #if IS_ENABLED(CONFIG_MPTCP_IPV6)
389 static struct request_sock_ops mptcp_subflow_v6_request_sock_ops __ro_after_init;
390 static struct tcp_request_sock_ops subflow_request_sock_ipv6_ops __ro_after_init;
391 static struct inet_connection_sock_af_ops subflow_v6_specific __ro_after_init;
392 static struct inet_connection_sock_af_ops subflow_v6m_specific __ro_after_init;
393 
subflow_v6_conn_request(struct sock * sk,struct sk_buff * skb)394 static int subflow_v6_conn_request(struct sock *sk, struct sk_buff *skb)
395 {
396 	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
397 
398 	pr_debug("subflow=%p", subflow);
399 
400 	if (skb->protocol == htons(ETH_P_IP))
401 		return subflow_v4_conn_request(sk, skb);
402 
403 	if (!ipv6_unicast_destination(skb))
404 		goto drop;
405 
406 	if (ipv6_addr_v4mapped(&ipv6_hdr(skb)->saddr)) {
407 		__IP6_INC_STATS(sock_net(sk), NULL, IPSTATS_MIB_INHDRERRORS);
408 		return 0;
409 	}
410 
411 	return tcp_conn_request(&mptcp_subflow_v6_request_sock_ops,
412 				&subflow_request_sock_ipv6_ops, sk, skb);
413 
414 drop:
415 	tcp_listendrop(sk);
416 	return 0; /* don't send reset */
417 }
418 
subflow_v6_req_destructor(struct request_sock * req)419 static void subflow_v6_req_destructor(struct request_sock *req)
420 {
421 	subflow_req_destructor(req);
422 	tcp6_request_sock_ops.destructor(req);
423 }
424 #endif
425 
mptcp_subflow_reqsk_alloc(const struct request_sock_ops * ops,struct sock * sk_listener,bool attach_listener)426 struct request_sock *mptcp_subflow_reqsk_alloc(const struct request_sock_ops *ops,
427 					       struct sock *sk_listener,
428 					       bool attach_listener)
429 {
430 	if (ops->family == AF_INET)
431 		ops = &mptcp_subflow_v4_request_sock_ops;
432 #if IS_ENABLED(CONFIG_MPTCP_IPV6)
433 	else if (ops->family == AF_INET6)
434 		ops = &mptcp_subflow_v6_request_sock_ops;
435 #endif
436 
437 	return inet_reqsk_alloc(ops, sk_listener, attach_listener);
438 }
439 EXPORT_SYMBOL(mptcp_subflow_reqsk_alloc);
440 
441 /* validate hmac received in third ACK */
subflow_hmac_valid(const struct request_sock * req,const struct mptcp_options_received * mp_opt)442 static bool subflow_hmac_valid(const struct request_sock *req,
443 			       const struct mptcp_options_received *mp_opt)
444 {
445 	const struct mptcp_subflow_request_sock *subflow_req;
446 	u8 hmac[SHA256_DIGEST_SIZE];
447 	struct mptcp_sock *msk;
448 
449 	subflow_req = mptcp_subflow_rsk(req);
450 	msk = subflow_req->msk;
451 	if (!msk)
452 		return false;
453 
454 	subflow_generate_hmac(msk->remote_key, msk->local_key,
455 			      subflow_req->remote_nonce,
456 			      subflow_req->local_nonce, hmac);
457 
458 	return !crypto_memneq(hmac, mp_opt->hmac, MPTCPOPT_HMAC_LEN);
459 }
460 
mptcp_sock_destruct(struct sock * sk)461 static void mptcp_sock_destruct(struct sock *sk)
462 {
463 	/* if new mptcp socket isn't accepted, it is free'd
464 	 * from the tcp listener sockets request queue, linked
465 	 * from req->sk.  The tcp socket is released.
466 	 * This calls the ULP release function which will
467 	 * also remove the mptcp socket, via
468 	 * sock_put(ctx->conn).
469 	 *
470 	 * Problem is that the mptcp socket will be in
471 	 * ESTABLISHED state and will not have the SOCK_DEAD flag.
472 	 * Both result in warnings from inet_sock_destruct.
473 	 */
474 	if ((1 << sk->sk_state) & (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)) {
475 		sk->sk_state = TCP_CLOSE;
476 		WARN_ON_ONCE(sk->sk_socket);
477 		sock_orphan(sk);
478 	}
479 
480 	mptcp_destroy_common(mptcp_sk(sk));
481 	inet_sock_destruct(sk);
482 }
483 
mptcp_force_close(struct sock * sk)484 static void mptcp_force_close(struct sock *sk)
485 {
486 	inet_sk_state_store(sk, TCP_CLOSE);
487 	sk_common_release(sk);
488 }
489 
subflow_ulp_fallback(struct sock * sk,struct mptcp_subflow_context * old_ctx)490 static void subflow_ulp_fallback(struct sock *sk,
491 				 struct mptcp_subflow_context *old_ctx)
492 {
493 	struct inet_connection_sock *icsk = inet_csk(sk);
494 
495 	mptcp_subflow_tcp_fallback(sk, old_ctx);
496 	icsk->icsk_ulp_ops = NULL;
497 	rcu_assign_pointer(icsk->icsk_ulp_data, NULL);
498 	tcp_sk(sk)->is_mptcp = 0;
499 }
500 
subflow_drop_ctx(struct sock * ssk)501 static void subflow_drop_ctx(struct sock *ssk)
502 {
503 	struct mptcp_subflow_context *ctx = mptcp_subflow_ctx(ssk);
504 
505 	if (!ctx)
506 		return;
507 
508 	subflow_ulp_fallback(ssk, ctx);
509 	if (ctx->conn)
510 		sock_put(ctx->conn);
511 
512 	kfree_rcu(ctx, rcu);
513 }
514 
mptcp_subflow_fully_established(struct mptcp_subflow_context * subflow,struct mptcp_options_received * mp_opt)515 void mptcp_subflow_fully_established(struct mptcp_subflow_context *subflow,
516 				     struct mptcp_options_received *mp_opt)
517 {
518 	struct mptcp_sock *msk = mptcp_sk(subflow->conn);
519 
520 	subflow->remote_key = mp_opt->sndr_key;
521 	subflow->fully_established = 1;
522 	subflow->can_ack = 1;
523 	WRITE_ONCE(msk->fully_established, true);
524 }
525 
subflow_syn_recv_sock(const struct sock * sk,struct sk_buff * skb,struct request_sock * req,struct dst_entry * dst,struct request_sock * req_unhash,bool * own_req)526 static struct sock *subflow_syn_recv_sock(const struct sock *sk,
527 					  struct sk_buff *skb,
528 					  struct request_sock *req,
529 					  struct dst_entry *dst,
530 					  struct request_sock *req_unhash,
531 					  bool *own_req)
532 {
533 	struct mptcp_subflow_context *listener = mptcp_subflow_ctx(sk);
534 	struct mptcp_subflow_request_sock *subflow_req;
535 	struct mptcp_options_received mp_opt;
536 	bool fallback, fallback_is_fatal;
537 	struct sock *new_msk = NULL;
538 	struct sock *child;
539 
540 	pr_debug("listener=%p, req=%p, conn=%p", listener, req, listener->conn);
541 
542 	/* After child creation we must look for 'mp_capable' even when options
543 	 * are not parsed
544 	 */
545 	mp_opt.mp_capable = 0;
546 
547 	/* hopefully temporary handling for MP_JOIN+syncookie */
548 	subflow_req = mptcp_subflow_rsk(req);
549 	fallback_is_fatal = tcp_rsk(req)->is_mptcp && subflow_req->mp_join;
550 	fallback = !tcp_rsk(req)->is_mptcp;
551 	if (fallback)
552 		goto create_child;
553 
554 	/* if the sk is MP_CAPABLE, we try to fetch the client key */
555 	if (subflow_req->mp_capable) {
556 		/* we can receive and accept an in-window, out-of-order pkt,
557 		 * which may not carry the MP_CAPABLE opt even on mptcp enabled
558 		 * paths: always try to extract the peer key, and fallback
559 		 * for packets missing it.
560 		 * Even OoO DSS packets coming legitly after dropped or
561 		 * reordered MPC will cause fallback, but we don't have other
562 		 * options.
563 		 */
564 		mptcp_get_options(skb, &mp_opt);
565 		if (!mp_opt.mp_capable) {
566 			fallback = true;
567 			goto create_child;
568 		}
569 
570 		new_msk = mptcp_sk_clone(listener->conn, &mp_opt, req);
571 		if (!new_msk)
572 			fallback = true;
573 	} else if (subflow_req->mp_join) {
574 		mptcp_get_options(skb, &mp_opt);
575 		if (!mp_opt.mp_join || !subflow_hmac_valid(req, &mp_opt) ||
576 		    !mptcp_can_accept_new_subflow(subflow_req->msk)) {
577 			SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINACKMAC);
578 			fallback = true;
579 		}
580 	}
581 
582 create_child:
583 	child = listener->icsk_af_ops->syn_recv_sock(sk, skb, req, dst,
584 						     req_unhash, own_req);
585 
586 	if (child && *own_req) {
587 		struct mptcp_subflow_context *ctx = mptcp_subflow_ctx(child);
588 
589 		tcp_rsk(req)->drop_req = false;
590 
591 		/* we need to fallback on ctx allocation failure and on pre-reqs
592 		 * checking above. In the latter scenario we additionally need
593 		 * to reset the context to non MPTCP status.
594 		 */
595 		if (!ctx || fallback) {
596 			if (fallback_is_fatal)
597 				goto dispose_child;
598 
599 			subflow_drop_ctx(child);
600 			goto out;
601 		}
602 
603 		if (ctx->mp_capable) {
604 			/* this can't race with mptcp_close(), as the msk is
605 			 * not yet exposted to user-space
606 			 */
607 			inet_sk_state_store((void *)new_msk, TCP_ESTABLISHED);
608 
609 			/* new mpc subflow takes ownership of the newly
610 			 * created mptcp socket
611 			 */
612 			new_msk->sk_destruct = mptcp_sock_destruct;
613 			mptcp_pm_new_connection(mptcp_sk(new_msk), 1);
614 			mptcp_token_accept(subflow_req, mptcp_sk(new_msk));
615 			ctx->conn = new_msk;
616 			new_msk = NULL;
617 
618 			/* with OoO packets we can reach here without ingress
619 			 * mpc option
620 			 */
621 			if (mp_opt.mp_capable)
622 				mptcp_subflow_fully_established(ctx, &mp_opt);
623 		} else if (ctx->mp_join) {
624 			struct mptcp_sock *owner;
625 
626 			owner = subflow_req->msk;
627 			if (!owner)
628 				goto dispose_child;
629 
630 			/* move the msk reference ownership to the subflow */
631 			subflow_req->msk = NULL;
632 			ctx->conn = (struct sock *)owner;
633 			if (!mptcp_finish_join(child))
634 				goto dispose_child;
635 
636 			SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINACKRX);
637 			tcp_rsk(req)->drop_req = true;
638 		}
639 	}
640 
641 out:
642 	/* dispose of the left over mptcp master, if any */
643 	if (unlikely(new_msk))
644 		mptcp_force_close(new_msk);
645 
646 	/* check for expected invariant - should never trigger, just help
647 	 * catching eariler subtle bugs
648 	 */
649 	WARN_ON_ONCE(child && *own_req && tcp_sk(child)->is_mptcp &&
650 		     (!mptcp_subflow_ctx(child) ||
651 		      !mptcp_subflow_ctx(child)->conn));
652 	return child;
653 
654 dispose_child:
655 	subflow_drop_ctx(child);
656 	tcp_rsk(req)->drop_req = true;
657 	inet_csk_prepare_for_destroy_sock(child);
658 	tcp_done(child);
659 	req->rsk_ops->send_reset(sk, skb);
660 
661 	/* The last child reference will be released by the caller */
662 	return child;
663 }
664 
665 static struct inet_connection_sock_af_ops subflow_specific __ro_after_init;
666 
667 enum mapping_status {
668 	MAPPING_OK,
669 	MAPPING_INVALID,
670 	MAPPING_EMPTY,
671 	MAPPING_DATA_FIN,
672 	MAPPING_DUMMY
673 };
674 
expand_seq(u64 old_seq,u16 old_data_len,u64 seq)675 static u64 expand_seq(u64 old_seq, u16 old_data_len, u64 seq)
676 {
677 	if ((u32)seq == (u32)old_seq)
678 		return old_seq;
679 
680 	/* Assume map covers data not mapped yet. */
681 	return seq | ((old_seq + old_data_len + 1) & GENMASK_ULL(63, 32));
682 }
683 
dbg_bad_map(struct mptcp_subflow_context * subflow,u32 ssn)684 static void dbg_bad_map(struct mptcp_subflow_context *subflow, u32 ssn)
685 {
686 	pr_debug("Bad mapping: ssn=%d map_seq=%d map_data_len=%d",
687 		 ssn, subflow->map_subflow_seq, subflow->map_data_len);
688 }
689 
skb_is_fully_mapped(struct sock * ssk,struct sk_buff * skb)690 static bool skb_is_fully_mapped(struct sock *ssk, struct sk_buff *skb)
691 {
692 	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
693 	unsigned int skb_consumed;
694 
695 	skb_consumed = tcp_sk(ssk)->copied_seq - TCP_SKB_CB(skb)->seq;
696 	if (WARN_ON_ONCE(skb_consumed >= skb->len))
697 		return true;
698 
699 	return skb->len - skb_consumed <= subflow->map_data_len -
700 					  mptcp_subflow_get_map_offset(subflow);
701 }
702 
validate_mapping(struct sock * ssk,struct sk_buff * skb)703 static bool validate_mapping(struct sock *ssk, struct sk_buff *skb)
704 {
705 	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
706 	u32 ssn = tcp_sk(ssk)->copied_seq - subflow->ssn_offset;
707 
708 	if (unlikely(before(ssn, subflow->map_subflow_seq))) {
709 		/* Mapping covers data later in the subflow stream,
710 		 * currently unsupported.
711 		 */
712 		dbg_bad_map(subflow, ssn);
713 		return false;
714 	}
715 	if (unlikely(!before(ssn, subflow->map_subflow_seq +
716 				  subflow->map_data_len))) {
717 		/* Mapping does covers past subflow data, invalid */
718 		dbg_bad_map(subflow, ssn);
719 		return false;
720 	}
721 	return true;
722 }
723 
get_mapping_status(struct sock * ssk,struct mptcp_sock * msk)724 static enum mapping_status get_mapping_status(struct sock *ssk,
725 					      struct mptcp_sock *msk)
726 {
727 	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
728 	struct mptcp_ext *mpext;
729 	struct sk_buff *skb;
730 	u16 data_len;
731 	u64 map_seq;
732 
733 	skb = skb_peek(&ssk->sk_receive_queue);
734 	if (!skb)
735 		return MAPPING_EMPTY;
736 
737 	if (mptcp_check_fallback(ssk))
738 		return MAPPING_DUMMY;
739 
740 	mpext = mptcp_get_ext(skb);
741 	if (!mpext || !mpext->use_map) {
742 		if (!subflow->map_valid && !skb->len) {
743 			/* the TCP stack deliver 0 len FIN pkt to the receive
744 			 * queue, that is the only 0len pkts ever expected here,
745 			 * and we can admit no mapping only for 0 len pkts
746 			 */
747 			if (!(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN))
748 				WARN_ONCE(1, "0len seq %d:%d flags %x",
749 					  TCP_SKB_CB(skb)->seq,
750 					  TCP_SKB_CB(skb)->end_seq,
751 					  TCP_SKB_CB(skb)->tcp_flags);
752 			sk_eat_skb(ssk, skb);
753 			return MAPPING_EMPTY;
754 		}
755 
756 		if (!subflow->map_valid)
757 			return MAPPING_INVALID;
758 
759 		goto validate_seq;
760 	}
761 
762 	pr_debug("seq=%llu is64=%d ssn=%u data_len=%u data_fin=%d",
763 		 mpext->data_seq, mpext->dsn64, mpext->subflow_seq,
764 		 mpext->data_len, mpext->data_fin);
765 
766 	data_len = mpext->data_len;
767 	if (data_len == 0) {
768 		MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_INFINITEMAPRX);
769 		return MAPPING_INVALID;
770 	}
771 
772 	if (mpext->data_fin == 1) {
773 		if (data_len == 1) {
774 			bool updated = mptcp_update_rcv_data_fin(msk, mpext->data_seq,
775 								 mpext->dsn64);
776 			pr_debug("DATA_FIN with no payload seq=%llu", mpext->data_seq);
777 			if (subflow->map_valid) {
778 				/* A DATA_FIN might arrive in a DSS
779 				 * option before the previous mapping
780 				 * has been fully consumed. Continue
781 				 * handling the existing mapping.
782 				 */
783 				skb_ext_del(skb, SKB_EXT_MPTCP);
784 				return MAPPING_OK;
785 			} else {
786 				if (updated && schedule_work(&msk->work))
787 					sock_hold((struct sock *)msk);
788 
789 				return MAPPING_DATA_FIN;
790 			}
791 		} else {
792 			u64 data_fin_seq = mpext->data_seq + data_len - 1;
793 
794 			/* If mpext->data_seq is a 32-bit value, data_fin_seq
795 			 * must also be limited to 32 bits.
796 			 */
797 			if (!mpext->dsn64)
798 				data_fin_seq &= GENMASK_ULL(31, 0);
799 
800 			mptcp_update_rcv_data_fin(msk, data_fin_seq, mpext->dsn64);
801 			pr_debug("DATA_FIN with mapping seq=%llu dsn64=%d",
802 				 data_fin_seq, mpext->dsn64);
803 		}
804 
805 		/* Adjust for DATA_FIN using 1 byte of sequence space */
806 		data_len--;
807 	}
808 
809 	if (!mpext->dsn64) {
810 		map_seq = expand_seq(subflow->map_seq, subflow->map_data_len,
811 				     mpext->data_seq);
812 		pr_debug("expanded seq=%llu", subflow->map_seq);
813 	} else {
814 		map_seq = mpext->data_seq;
815 	}
816 	WRITE_ONCE(mptcp_sk(subflow->conn)->use_64bit_ack, !!mpext->dsn64);
817 
818 	if (subflow->map_valid) {
819 		/* Allow replacing only with an identical map */
820 		if (subflow->map_seq == map_seq &&
821 		    subflow->map_subflow_seq == mpext->subflow_seq &&
822 		    subflow->map_data_len == data_len) {
823 			skb_ext_del(skb, SKB_EXT_MPTCP);
824 			return MAPPING_OK;
825 		}
826 
827 		/* If this skb data are fully covered by the current mapping,
828 		 * the new map would need caching, which is not supported
829 		 */
830 		if (skb_is_fully_mapped(ssk, skb)) {
831 			MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_DSSNOMATCH);
832 			return MAPPING_INVALID;
833 		}
834 
835 		/* will validate the next map after consuming the current one */
836 		return MAPPING_OK;
837 	}
838 
839 	subflow->map_seq = map_seq;
840 	subflow->map_subflow_seq = mpext->subflow_seq;
841 	subflow->map_data_len = data_len;
842 	subflow->map_valid = 1;
843 	subflow->mpc_map = mpext->mpc_map;
844 	pr_debug("new map seq=%llu subflow_seq=%u data_len=%u",
845 		 subflow->map_seq, subflow->map_subflow_seq,
846 		 subflow->map_data_len);
847 
848 validate_seq:
849 	/* we revalidate valid mapping on new skb, because we must ensure
850 	 * the current skb is completely covered by the available mapping
851 	 */
852 	if (!validate_mapping(ssk, skb))
853 		return MAPPING_INVALID;
854 
855 	skb_ext_del(skb, SKB_EXT_MPTCP);
856 	return MAPPING_OK;
857 }
858 
mptcp_subflow_discard_data(struct sock * ssk,struct sk_buff * skb,u64 limit)859 static void mptcp_subflow_discard_data(struct sock *ssk, struct sk_buff *skb,
860 				       u64 limit)
861 {
862 	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
863 	bool fin = TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN;
864 	u32 incr;
865 
866 	incr = limit >= skb->len ? skb->len + fin : limit;
867 
868 	pr_debug("discarding=%d len=%d seq=%d", incr, skb->len,
869 		 subflow->map_subflow_seq);
870 	MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_DUPDATA);
871 	tcp_sk(ssk)->copied_seq += incr;
872 	if (!before(tcp_sk(ssk)->copied_seq, TCP_SKB_CB(skb)->end_seq))
873 		sk_eat_skb(ssk, skb);
874 	if (mptcp_subflow_get_map_offset(subflow) >= subflow->map_data_len)
875 		subflow->map_valid = 0;
876 	if (incr)
877 		tcp_cleanup_rbuf(ssk, incr);
878 }
879 
subflow_check_data_avail(struct sock * ssk)880 static bool subflow_check_data_avail(struct sock *ssk)
881 {
882 	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
883 	enum mapping_status status;
884 	struct mptcp_sock *msk;
885 	struct sk_buff *skb;
886 
887 	pr_debug("msk=%p ssk=%p data_avail=%d skb=%p", subflow->conn, ssk,
888 		 subflow->data_avail, skb_peek(&ssk->sk_receive_queue));
889 	if (!skb_peek(&ssk->sk_receive_queue))
890 		subflow->data_avail = 0;
891 	if (subflow->data_avail)
892 		return true;
893 
894 	msk = mptcp_sk(subflow->conn);
895 	for (;;) {
896 		u64 ack_seq;
897 		u64 old_ack;
898 
899 		status = get_mapping_status(ssk, msk);
900 		pr_debug("msk=%p ssk=%p status=%d", msk, ssk, status);
901 		if (status == MAPPING_INVALID) {
902 			ssk->sk_err = EBADMSG;
903 			goto fatal;
904 		}
905 		if (status == MAPPING_DUMMY) {
906 			__mptcp_do_fallback(msk);
907 			skb = skb_peek(&ssk->sk_receive_queue);
908 			subflow->map_valid = 1;
909 			subflow->map_seq = READ_ONCE(msk->ack_seq);
910 			subflow->map_data_len = skb->len;
911 			subflow->map_subflow_seq = tcp_sk(ssk)->copied_seq -
912 						   subflow->ssn_offset;
913 			subflow->data_avail = MPTCP_SUBFLOW_DATA_AVAIL;
914 			return true;
915 		}
916 
917 		if (status != MAPPING_OK)
918 			return false;
919 
920 		skb = skb_peek(&ssk->sk_receive_queue);
921 		if (WARN_ON_ONCE(!skb))
922 			return false;
923 
924 		/* if msk lacks the remote key, this subflow must provide an
925 		 * MP_CAPABLE-based mapping
926 		 */
927 		if (unlikely(!READ_ONCE(msk->can_ack))) {
928 			if (!subflow->mpc_map) {
929 				ssk->sk_err = EBADMSG;
930 				goto fatal;
931 			}
932 			WRITE_ONCE(msk->remote_key, subflow->remote_key);
933 			WRITE_ONCE(msk->ack_seq, subflow->map_seq);
934 			WRITE_ONCE(msk->can_ack, true);
935 		}
936 
937 		old_ack = READ_ONCE(msk->ack_seq);
938 		ack_seq = mptcp_subflow_get_mapped_dsn(subflow);
939 		pr_debug("msk ack_seq=%llx subflow ack_seq=%llx", old_ack,
940 			 ack_seq);
941 		if (ack_seq == old_ack) {
942 			subflow->data_avail = MPTCP_SUBFLOW_DATA_AVAIL;
943 			break;
944 		} else if (after64(ack_seq, old_ack)) {
945 			subflow->data_avail = MPTCP_SUBFLOW_OOO_DATA;
946 			break;
947 		}
948 
949 		/* only accept in-sequence mapping. Old values are spurious
950 		 * retransmission
951 		 */
952 		mptcp_subflow_discard_data(ssk, skb, old_ack - ack_seq);
953 	}
954 	return true;
955 
956 fatal:
957 	/* fatal protocol error, close the socket */
958 	/* This barrier is coupled with smp_rmb() in tcp_poll() */
959 	smp_wmb();
960 	ssk->sk_error_report(ssk);
961 	tcp_set_state(ssk, TCP_CLOSE);
962 	tcp_send_active_reset(ssk, GFP_ATOMIC);
963 	subflow->data_avail = 0;
964 	return false;
965 }
966 
mptcp_subflow_data_available(struct sock * sk)967 bool mptcp_subflow_data_available(struct sock *sk)
968 {
969 	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
970 
971 	/* check if current mapping is still valid */
972 	if (subflow->map_valid &&
973 	    mptcp_subflow_get_map_offset(subflow) >= subflow->map_data_len) {
974 		subflow->map_valid = 0;
975 		subflow->data_avail = 0;
976 
977 		pr_debug("Done with mapping: seq=%u data_len=%u",
978 			 subflow->map_subflow_seq,
979 			 subflow->map_data_len);
980 	}
981 
982 	return subflow_check_data_avail(sk);
983 }
984 
985 /* If ssk has an mptcp parent socket, use the mptcp rcvbuf occupancy,
986  * not the ssk one.
987  *
988  * In mptcp, rwin is about the mptcp-level connection data.
989  *
990  * Data that is still on the ssk rx queue can thus be ignored,
991  * as far as mptcp peer is concerened that data is still inflight.
992  * DSS ACK is updated when skb is moved to the mptcp rx queue.
993  */
mptcp_space(const struct sock * ssk,int * space,int * full_space)994 void mptcp_space(const struct sock *ssk, int *space, int *full_space)
995 {
996 	const struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
997 	const struct sock *sk = subflow->conn;
998 
999 	*space = tcp_space(sk);
1000 	*full_space = tcp_full_space(sk);
1001 }
1002 
subflow_data_ready(struct sock * sk)1003 static void subflow_data_ready(struct sock *sk)
1004 {
1005 	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
1006 	u16 state = 1 << inet_sk_state_load(sk);
1007 	struct sock *parent = subflow->conn;
1008 	struct mptcp_sock *msk;
1009 
1010 	msk = mptcp_sk(parent);
1011 	if (state & TCPF_LISTEN) {
1012 		/* MPJ subflow are removed from accept queue before reaching here,
1013 		 * avoid stray wakeups
1014 		 */
1015 		if (reqsk_queue_empty(&inet_csk(sk)->icsk_accept_queue))
1016 			return;
1017 
1018 		set_bit(MPTCP_DATA_READY, &msk->flags);
1019 		parent->sk_data_ready(parent);
1020 		return;
1021 	}
1022 
1023 	WARN_ON_ONCE(!__mptcp_check_fallback(msk) && !subflow->mp_capable &&
1024 		     !subflow->mp_join && !(state & TCPF_CLOSE));
1025 
1026 	if (mptcp_subflow_data_available(sk))
1027 		mptcp_data_ready(parent, sk);
1028 }
1029 
subflow_write_space(struct sock * sk)1030 static void subflow_write_space(struct sock *sk)
1031 {
1032 	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
1033 	struct sock *parent = subflow->conn;
1034 
1035 	if (!sk_stream_is_writeable(sk))
1036 		return;
1037 
1038 	if (sk_stream_is_writeable(parent)) {
1039 		set_bit(MPTCP_SEND_SPACE, &mptcp_sk(parent)->flags);
1040 		smp_mb__after_atomic();
1041 		/* set SEND_SPACE before sk_stream_write_space clears NOSPACE */
1042 		sk_stream_write_space(parent);
1043 	}
1044 }
1045 
1046 static const struct inet_connection_sock_af_ops *
subflow_default_af_ops(struct sock * sk)1047 subflow_default_af_ops(struct sock *sk)
1048 {
1049 #if IS_ENABLED(CONFIG_MPTCP_IPV6)
1050 	if (sk->sk_family == AF_INET6)
1051 		return &subflow_v6_specific;
1052 #endif
1053 	return &subflow_specific;
1054 }
1055 
1056 #if IS_ENABLED(CONFIG_MPTCP_IPV6)
mptcpv6_handle_mapped(struct sock * sk,bool mapped)1057 void mptcpv6_handle_mapped(struct sock *sk, bool mapped)
1058 {
1059 	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
1060 	struct inet_connection_sock *icsk = inet_csk(sk);
1061 	const struct inet_connection_sock_af_ops *target;
1062 
1063 	target = mapped ? &subflow_v6m_specific : subflow_default_af_ops(sk);
1064 
1065 	pr_debug("subflow=%p family=%d ops=%p target=%p mapped=%d",
1066 		 subflow, sk->sk_family, icsk->icsk_af_ops, target, mapped);
1067 
1068 	if (likely(icsk->icsk_af_ops == target))
1069 		return;
1070 
1071 	subflow->icsk_af_ops = icsk->icsk_af_ops;
1072 	icsk->icsk_af_ops = target;
1073 }
1074 #endif
1075 
mptcp_info2sockaddr(const struct mptcp_addr_info * info,struct sockaddr_storage * addr)1076 static void mptcp_info2sockaddr(const struct mptcp_addr_info *info,
1077 				struct sockaddr_storage *addr)
1078 {
1079 	memset(addr, 0, sizeof(*addr));
1080 	addr->ss_family = info->family;
1081 	if (addr->ss_family == AF_INET) {
1082 		struct sockaddr_in *in_addr = (struct sockaddr_in *)addr;
1083 
1084 		in_addr->sin_addr = info->addr;
1085 		in_addr->sin_port = info->port;
1086 	}
1087 #if IS_ENABLED(CONFIG_MPTCP_IPV6)
1088 	else if (addr->ss_family == AF_INET6) {
1089 		struct sockaddr_in6 *in6_addr = (struct sockaddr_in6 *)addr;
1090 
1091 		in6_addr->sin6_addr = info->addr6;
1092 		in6_addr->sin6_port = info->port;
1093 	}
1094 #endif
1095 }
1096 
__mptcp_subflow_connect(struct sock * sk,const struct mptcp_addr_info * loc,const struct mptcp_addr_info * remote)1097 int __mptcp_subflow_connect(struct sock *sk, const struct mptcp_addr_info *loc,
1098 			    const struct mptcp_addr_info *remote)
1099 {
1100 	struct mptcp_sock *msk = mptcp_sk(sk);
1101 	struct mptcp_subflow_context *subflow;
1102 	struct sockaddr_storage addr;
1103 	int remote_id = remote->id;
1104 	int local_id = loc->id;
1105 	struct socket *sf;
1106 	struct sock *ssk;
1107 	u32 remote_token;
1108 	int addrlen;
1109 	int err;
1110 
1111 	if (!mptcp_is_fully_established(sk))
1112 		return -ENOTCONN;
1113 
1114 	err = mptcp_subflow_create_socket(sk, &sf);
1115 	if (err)
1116 		return err;
1117 
1118 	ssk = sf->sk;
1119 	subflow = mptcp_subflow_ctx(ssk);
1120 	do {
1121 		get_random_bytes(&subflow->local_nonce, sizeof(u32));
1122 	} while (!subflow->local_nonce);
1123 
1124 	if (!local_id) {
1125 		err = mptcp_pm_get_local_id(msk, (struct sock_common *)ssk);
1126 		if (err < 0)
1127 			goto failed;
1128 
1129 		local_id = err;
1130 	}
1131 
1132 	subflow->remote_key = msk->remote_key;
1133 	subflow->local_key = msk->local_key;
1134 	subflow->token = msk->token;
1135 	mptcp_info2sockaddr(loc, &addr);
1136 
1137 	addrlen = sizeof(struct sockaddr_in);
1138 #if IS_ENABLED(CONFIG_MPTCP_IPV6)
1139 	if (loc->family == AF_INET6)
1140 		addrlen = sizeof(struct sockaddr_in6);
1141 #endif
1142 	ssk->sk_bound_dev_if = loc->ifindex;
1143 	err = kernel_bind(sf, (struct sockaddr *)&addr, addrlen);
1144 	if (err)
1145 		goto failed;
1146 
1147 	mptcp_crypto_key_sha(subflow->remote_key, &remote_token, NULL);
1148 	pr_debug("msk=%p remote_token=%u local_id=%d remote_id=%d", msk,
1149 		 remote_token, local_id, remote_id);
1150 	subflow->remote_token = remote_token;
1151 	subflow->local_id = local_id;
1152 	subflow->remote_id = remote_id;
1153 	subflow->request_join = 1;
1154 	subflow->request_bkup = !!(loc->flags & MPTCP_PM_ADDR_FLAG_BACKUP);
1155 	mptcp_info2sockaddr(remote, &addr);
1156 
1157 	err = kernel_connect(sf, (struct sockaddr *)&addr, addrlen, O_NONBLOCK);
1158 	if (err && err != -EINPROGRESS)
1159 		goto failed;
1160 
1161 	spin_lock_bh(&msk->join_list_lock);
1162 	list_add_tail(&subflow->node, &msk->join_list);
1163 	spin_unlock_bh(&msk->join_list_lock);
1164 
1165 	return err;
1166 
1167 failed:
1168 	sock_release(sf);
1169 	return err;
1170 }
1171 
mptcp_subflow_create_socket(struct sock * sk,struct socket ** new_sock)1172 int mptcp_subflow_create_socket(struct sock *sk, struct socket **new_sock)
1173 {
1174 	struct mptcp_subflow_context *subflow;
1175 	struct net *net = sock_net(sk);
1176 	struct socket *sf;
1177 	int err;
1178 
1179 	/* un-accepted server sockets can reach here - on bad configuration
1180 	 * bail early to avoid greater trouble later
1181 	 */
1182 	if (unlikely(!sk->sk_socket))
1183 		return -EINVAL;
1184 
1185 	err = sock_create_kern(net, sk->sk_family, SOCK_STREAM, IPPROTO_TCP,
1186 			       &sf);
1187 	if (err)
1188 		return err;
1189 
1190 	lock_sock(sf->sk);
1191 
1192 	/* kernel sockets do not by default acquire net ref, but TCP timer
1193 	 * needs it.
1194 	 */
1195 	sf->sk->sk_net_refcnt = 1;
1196 	get_net(net);
1197 #ifdef CONFIG_PROC_FS
1198 	this_cpu_add(*net->core.sock_inuse, 1);
1199 #endif
1200 	err = tcp_set_ulp(sf->sk, "mptcp");
1201 	release_sock(sf->sk);
1202 
1203 	if (err) {
1204 		sock_release(sf);
1205 		return err;
1206 	}
1207 
1208 	/* the newly created socket really belongs to the owning MPTCP master
1209 	 * socket, even if for additional subflows the allocation is performed
1210 	 * by a kernel workqueue. Adjust inode references, so that the
1211 	 * procfs/diag interaces really show this one belonging to the correct
1212 	 * user.
1213 	 */
1214 	SOCK_INODE(sf)->i_ino = SOCK_INODE(sk->sk_socket)->i_ino;
1215 	SOCK_INODE(sf)->i_uid = SOCK_INODE(sk->sk_socket)->i_uid;
1216 	SOCK_INODE(sf)->i_gid = SOCK_INODE(sk->sk_socket)->i_gid;
1217 
1218 	subflow = mptcp_subflow_ctx(sf->sk);
1219 	pr_debug("subflow=%p", subflow);
1220 
1221 	*new_sock = sf;
1222 	sock_hold(sk);
1223 	subflow->conn = sk;
1224 
1225 	return 0;
1226 }
1227 
subflow_create_ctx(struct sock * sk,gfp_t priority)1228 static struct mptcp_subflow_context *subflow_create_ctx(struct sock *sk,
1229 							gfp_t priority)
1230 {
1231 	struct inet_connection_sock *icsk = inet_csk(sk);
1232 	struct mptcp_subflow_context *ctx;
1233 
1234 	ctx = kzalloc(sizeof(*ctx), priority);
1235 	if (!ctx)
1236 		return NULL;
1237 
1238 	rcu_assign_pointer(icsk->icsk_ulp_data, ctx);
1239 	INIT_LIST_HEAD(&ctx->node);
1240 
1241 	pr_debug("subflow=%p", ctx);
1242 
1243 	ctx->tcp_sock = sk;
1244 
1245 	return ctx;
1246 }
1247 
__subflow_state_change(struct sock * sk)1248 static void __subflow_state_change(struct sock *sk)
1249 {
1250 	struct socket_wq *wq;
1251 
1252 	rcu_read_lock();
1253 	wq = rcu_dereference(sk->sk_wq);
1254 	if (skwq_has_sleeper(wq))
1255 		wake_up_interruptible_all(&wq->wait);
1256 	rcu_read_unlock();
1257 }
1258 
subflow_is_done(const struct sock * sk)1259 static bool subflow_is_done(const struct sock *sk)
1260 {
1261 	return sk->sk_shutdown & RCV_SHUTDOWN || sk->sk_state == TCP_CLOSE;
1262 }
1263 
subflow_state_change(struct sock * sk)1264 static void subflow_state_change(struct sock *sk)
1265 {
1266 	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
1267 	struct sock *parent = subflow->conn;
1268 
1269 	__subflow_state_change(sk);
1270 
1271 	if (subflow_simultaneous_connect(sk)) {
1272 		mptcp_do_fallback(sk);
1273 		mptcp_rcv_space_init(mptcp_sk(parent), sk);
1274 		pr_fallback(mptcp_sk(parent));
1275 		subflow->conn_finished = 1;
1276 		if (inet_sk_state_load(parent) == TCP_SYN_SENT) {
1277 			inet_sk_state_store(parent, TCP_ESTABLISHED);
1278 			parent->sk_state_change(parent);
1279 		}
1280 	}
1281 
1282 	/* as recvmsg() does not acquire the subflow socket for ssk selection
1283 	 * a fin packet carrying a DSS can be unnoticed if we don't trigger
1284 	 * the data available machinery here.
1285 	 */
1286 	if (mptcp_subflow_data_available(sk))
1287 		mptcp_data_ready(parent, sk);
1288 
1289 	if (__mptcp_check_fallback(mptcp_sk(parent)) &&
1290 	    !(parent->sk_shutdown & RCV_SHUTDOWN) &&
1291 	    !subflow->rx_eof && subflow_is_done(sk)) {
1292 		subflow->rx_eof = 1;
1293 		mptcp_subflow_eof(parent);
1294 	}
1295 }
1296 
subflow_ulp_init(struct sock * sk)1297 static int subflow_ulp_init(struct sock *sk)
1298 {
1299 	struct inet_connection_sock *icsk = inet_csk(sk);
1300 	struct mptcp_subflow_context *ctx;
1301 	struct tcp_sock *tp = tcp_sk(sk);
1302 	int err = 0;
1303 
1304 	/* disallow attaching ULP to a socket unless it has been
1305 	 * created with sock_create_kern()
1306 	 */
1307 	if (!sk->sk_kern_sock) {
1308 		err = -EOPNOTSUPP;
1309 		goto out;
1310 	}
1311 
1312 	ctx = subflow_create_ctx(sk, GFP_KERNEL);
1313 	if (!ctx) {
1314 		err = -ENOMEM;
1315 		goto out;
1316 	}
1317 
1318 	pr_debug("subflow=%p, family=%d", ctx, sk->sk_family);
1319 
1320 	tp->is_mptcp = 1;
1321 	ctx->icsk_af_ops = icsk->icsk_af_ops;
1322 	icsk->icsk_af_ops = subflow_default_af_ops(sk);
1323 	ctx->tcp_data_ready = sk->sk_data_ready;
1324 	ctx->tcp_state_change = sk->sk_state_change;
1325 	ctx->tcp_write_space = sk->sk_write_space;
1326 	sk->sk_data_ready = subflow_data_ready;
1327 	sk->sk_write_space = subflow_write_space;
1328 	sk->sk_state_change = subflow_state_change;
1329 out:
1330 	return err;
1331 }
1332 
subflow_ulp_release(struct sock * sk)1333 static void subflow_ulp_release(struct sock *sk)
1334 {
1335 	struct mptcp_subflow_context *ctx = mptcp_subflow_ctx(sk);
1336 
1337 	if (!ctx)
1338 		return;
1339 
1340 	if (ctx->conn)
1341 		sock_put(ctx->conn);
1342 
1343 	kfree_rcu(ctx, rcu);
1344 }
1345 
subflow_ulp_clone(const struct request_sock * req,struct sock * newsk,const gfp_t priority)1346 static void subflow_ulp_clone(const struct request_sock *req,
1347 			      struct sock *newsk,
1348 			      const gfp_t priority)
1349 {
1350 	struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req);
1351 	struct mptcp_subflow_context *old_ctx = mptcp_subflow_ctx(newsk);
1352 	struct mptcp_subflow_context *new_ctx;
1353 
1354 	if (!tcp_rsk(req)->is_mptcp ||
1355 	    (!subflow_req->mp_capable && !subflow_req->mp_join)) {
1356 		subflow_ulp_fallback(newsk, old_ctx);
1357 		return;
1358 	}
1359 
1360 	new_ctx = subflow_create_ctx(newsk, priority);
1361 	if (!new_ctx) {
1362 		subflow_ulp_fallback(newsk, old_ctx);
1363 		return;
1364 	}
1365 
1366 	new_ctx->conn_finished = 1;
1367 	new_ctx->icsk_af_ops = old_ctx->icsk_af_ops;
1368 	new_ctx->tcp_data_ready = old_ctx->tcp_data_ready;
1369 	new_ctx->tcp_state_change = old_ctx->tcp_state_change;
1370 	new_ctx->tcp_write_space = old_ctx->tcp_write_space;
1371 	new_ctx->rel_write_seq = 1;
1372 	new_ctx->tcp_sock = newsk;
1373 
1374 	if (subflow_req->mp_capable) {
1375 		/* see comments in subflow_syn_recv_sock(), MPTCP connection
1376 		 * is fully established only after we receive the remote key
1377 		 */
1378 		new_ctx->mp_capable = 1;
1379 		new_ctx->local_key = subflow_req->local_key;
1380 		new_ctx->token = subflow_req->token;
1381 		new_ctx->ssn_offset = subflow_req->ssn_offset;
1382 		new_ctx->idsn = subflow_req->idsn;
1383 	} else if (subflow_req->mp_join) {
1384 		new_ctx->ssn_offset = subflow_req->ssn_offset;
1385 		new_ctx->mp_join = 1;
1386 		new_ctx->fully_established = 1;
1387 		new_ctx->backup = subflow_req->backup;
1388 		new_ctx->local_id = subflow_req->local_id;
1389 		new_ctx->remote_id = subflow_req->remote_id;
1390 		new_ctx->token = subflow_req->token;
1391 		new_ctx->thmac = subflow_req->thmac;
1392 	}
1393 }
1394 
1395 static struct tcp_ulp_ops subflow_ulp_ops __read_mostly = {
1396 	.name		= "mptcp",
1397 	.owner		= THIS_MODULE,
1398 	.init		= subflow_ulp_init,
1399 	.release	= subflow_ulp_release,
1400 	.clone		= subflow_ulp_clone,
1401 };
1402 
subflow_ops_init(struct request_sock_ops * subflow_ops)1403 static int subflow_ops_init(struct request_sock_ops *subflow_ops)
1404 {
1405 	subflow_ops->obj_size = sizeof(struct mptcp_subflow_request_sock);
1406 
1407 	subflow_ops->slab = kmem_cache_create(subflow_ops->slab_name,
1408 					      subflow_ops->obj_size, 0,
1409 					      SLAB_ACCOUNT |
1410 					      SLAB_TYPESAFE_BY_RCU,
1411 					      NULL);
1412 	if (!subflow_ops->slab)
1413 		return -ENOMEM;
1414 
1415 	return 0;
1416 }
1417 
mptcp_subflow_init(void)1418 void __init mptcp_subflow_init(void)
1419 {
1420 	mptcp_subflow_v4_request_sock_ops = tcp_request_sock_ops;
1421 	mptcp_subflow_v4_request_sock_ops.slab_name = "request_sock_subflow_v4";
1422 	mptcp_subflow_v4_request_sock_ops.destructor = subflow_v4_req_destructor;
1423 
1424 	if (subflow_ops_init(&mptcp_subflow_v4_request_sock_ops) != 0)
1425 		panic("MPTCP: failed to init subflow v4 request sock ops\n");
1426 
1427 	subflow_request_sock_ipv4_ops = tcp_request_sock_ipv4_ops;
1428 	subflow_request_sock_ipv4_ops.init_req = subflow_v4_init_req;
1429 
1430 	subflow_specific = ipv4_specific;
1431 	subflow_specific.conn_request = subflow_v4_conn_request;
1432 	subflow_specific.syn_recv_sock = subflow_syn_recv_sock;
1433 	subflow_specific.sk_rx_dst_set = subflow_finish_connect;
1434 
1435 #if IS_ENABLED(CONFIG_MPTCP_IPV6)
1436 	/* In struct mptcp_subflow_request_sock, we assume the TCP request sock
1437 	 * structures for v4 and v6 have the same size. It should not changed in
1438 	 * the future but better to make sure to be warned if it is no longer
1439 	 * the case.
1440 	 */
1441 	BUILD_BUG_ON(sizeof(struct tcp_request_sock) != sizeof(struct tcp6_request_sock));
1442 
1443 	mptcp_subflow_v6_request_sock_ops = tcp6_request_sock_ops;
1444 	mptcp_subflow_v6_request_sock_ops.slab_name = "request_sock_subflow_v6";
1445 	mptcp_subflow_v6_request_sock_ops.destructor = subflow_v6_req_destructor;
1446 
1447 	if (subflow_ops_init(&mptcp_subflow_v6_request_sock_ops) != 0)
1448 		panic("MPTCP: failed to init subflow v6 request sock ops\n");
1449 
1450 	subflow_request_sock_ipv6_ops = tcp_request_sock_ipv6_ops;
1451 	subflow_request_sock_ipv6_ops.init_req = subflow_v6_init_req;
1452 
1453 	subflow_v6_specific = ipv6_specific;
1454 	subflow_v6_specific.conn_request = subflow_v6_conn_request;
1455 	subflow_v6_specific.syn_recv_sock = subflow_syn_recv_sock;
1456 	subflow_v6_specific.sk_rx_dst_set = subflow_finish_connect;
1457 
1458 	subflow_v6m_specific = subflow_v6_specific;
1459 	subflow_v6m_specific.queue_xmit = ipv4_specific.queue_xmit;
1460 	subflow_v6m_specific.send_check = ipv4_specific.send_check;
1461 	subflow_v6m_specific.net_header_len = ipv4_specific.net_header_len;
1462 	subflow_v6m_specific.mtu_reduced = ipv4_specific.mtu_reduced;
1463 	subflow_v6m_specific.net_frag_header_len = 0;
1464 #endif
1465 
1466 	mptcp_diag_subflow_init(&subflow_ulp_ops);
1467 
1468 	if (tcp_register_ulp(&subflow_ulp_ops) != 0)
1469 		panic("MPTCP: failed to register subflows to ULP\n");
1470 }
1471