1 /*
2 * Copyright (c) 2016-2017, Mellanox Technologies. All rights reserved.
3 * Copyright (c) 2016-2017, Dave Watson <davejwatson@fb.com>. All rights reserved.
4 * Copyright (c) 2016-2017, Lance Chao <lancerchao@fb.com>. All rights reserved.
5 * Copyright (c) 2016, Fridolin Pokorny <fridolin.pokorny@gmail.com>. All rights reserved.
6 * Copyright (c) 2016, Nikos Mavrogiannopoulos <nmav@gnutls.org>. All rights reserved.
7 * Copyright (c) 2018, Covalent IO, Inc. http://covalent.io
8 *
9 * This software is available to you under a choice of one of two
10 * licenses. You may choose to be licensed under the terms of the GNU
11 * General Public License (GPL) Version 2, available from the file
12 * COPYING in the main directory of this source tree, or the
13 * OpenIB.org BSD license below:
14 *
15 * Redistribution and use in source and binary forms, with or
16 * without modification, are permitted provided that the following
17 * conditions are met:
18 *
19 * - Redistributions of source code must retain the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer.
22 *
23 * - Redistributions in binary form must reproduce the above
24 * copyright notice, this list of conditions and the following
25 * disclaimer in the documentation and/or other materials
26 * provided with the distribution.
27 *
28 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
29 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
30 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
31 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
32 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
33 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
34 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
35 * SOFTWARE.
36 */
37
38 #include <linux/bug.h>
39 #include <linux/sched/signal.h>
40 #include <linux/module.h>
41 #include <linux/splice.h>
42 #include <crypto/aead.h>
43
44 #include <net/strparser.h>
45 #include <net/tls.h>
46
47 struct tls_decrypt_arg {
48 bool zc;
49 bool async;
50 };
51
tls_err_abort(struct sock * sk,int err)52 noinline void tls_err_abort(struct sock *sk, int err)
53 {
54 WARN_ON_ONCE(err >= 0);
55 /* sk->sk_err should contain a positive error code. */
56 sk->sk_err = -err;
57 sk_error_report(sk);
58 }
59
__skb_nsg(struct sk_buff * skb,int offset,int len,unsigned int recursion_level)60 static int __skb_nsg(struct sk_buff *skb, int offset, int len,
61 unsigned int recursion_level)
62 {
63 int start = skb_headlen(skb);
64 int i, chunk = start - offset;
65 struct sk_buff *frag_iter;
66 int elt = 0;
67
68 if (unlikely(recursion_level >= 24))
69 return -EMSGSIZE;
70
71 if (chunk > 0) {
72 if (chunk > len)
73 chunk = len;
74 elt++;
75 len -= chunk;
76 if (len == 0)
77 return elt;
78 offset += chunk;
79 }
80
81 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
82 int end;
83
84 WARN_ON(start > offset + len);
85
86 end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]);
87 chunk = end - offset;
88 if (chunk > 0) {
89 if (chunk > len)
90 chunk = len;
91 elt++;
92 len -= chunk;
93 if (len == 0)
94 return elt;
95 offset += chunk;
96 }
97 start = end;
98 }
99
100 if (unlikely(skb_has_frag_list(skb))) {
101 skb_walk_frags(skb, frag_iter) {
102 int end, ret;
103
104 WARN_ON(start > offset + len);
105
106 end = start + frag_iter->len;
107 chunk = end - offset;
108 if (chunk > 0) {
109 if (chunk > len)
110 chunk = len;
111 ret = __skb_nsg(frag_iter, offset - start, chunk,
112 recursion_level + 1);
113 if (unlikely(ret < 0))
114 return ret;
115 elt += ret;
116 len -= chunk;
117 if (len == 0)
118 return elt;
119 offset += chunk;
120 }
121 start = end;
122 }
123 }
124 BUG_ON(len);
125 return elt;
126 }
127
128 /* Return the number of scatterlist elements required to completely map the
129 * skb, or -EMSGSIZE if the recursion depth is exceeded.
130 */
skb_nsg(struct sk_buff * skb,int offset,int len)131 static int skb_nsg(struct sk_buff *skb, int offset, int len)
132 {
133 return __skb_nsg(skb, offset, len, 0);
134 }
135
padding_length(struct tls_prot_info * prot,struct sk_buff * skb)136 static int padding_length(struct tls_prot_info *prot, struct sk_buff *skb)
137 {
138 struct strp_msg *rxm = strp_msg(skb);
139 struct tls_msg *tlm = tls_msg(skb);
140 int sub = 0;
141
142 /* Determine zero-padding length */
143 if (prot->version == TLS_1_3_VERSION) {
144 char content_type = 0;
145 int err;
146 int back = 17;
147
148 while (content_type == 0) {
149 if (back > rxm->full_len - prot->prepend_size)
150 return -EBADMSG;
151 err = skb_copy_bits(skb,
152 rxm->offset + rxm->full_len - back,
153 &content_type, 1);
154 if (err)
155 return err;
156 if (content_type)
157 break;
158 sub++;
159 back++;
160 }
161 tlm->control = content_type;
162 }
163 return sub;
164 }
165
tls_decrypt_done(struct crypto_async_request * req,int err)166 static void tls_decrypt_done(struct crypto_async_request *req, int err)
167 {
168 struct aead_request *aead_req = (struct aead_request *)req;
169 struct scatterlist *sgout = aead_req->dst;
170 struct scatterlist *sgin = aead_req->src;
171 struct tls_sw_context_rx *ctx;
172 struct tls_context *tls_ctx;
173 struct tls_prot_info *prot;
174 struct scatterlist *sg;
175 struct sk_buff *skb;
176 unsigned int pages;
177 int pending;
178
179 skb = (struct sk_buff *)req->data;
180 tls_ctx = tls_get_ctx(skb->sk);
181 ctx = tls_sw_ctx_rx(tls_ctx);
182 prot = &tls_ctx->prot_info;
183
184 /* Propagate if there was an err */
185 if (err) {
186 if (err == -EBADMSG)
187 TLS_INC_STATS(sock_net(skb->sk),
188 LINUX_MIB_TLSDECRYPTERROR);
189 ctx->async_wait.err = err;
190 tls_err_abort(skb->sk, err);
191 } else {
192 struct strp_msg *rxm = strp_msg(skb);
193 int pad;
194
195 pad = padding_length(prot, skb);
196 if (pad < 0) {
197 ctx->async_wait.err = pad;
198 tls_err_abort(skb->sk, pad);
199 } else {
200 rxm->full_len -= pad;
201 rxm->offset += prot->prepend_size;
202 rxm->full_len -= prot->overhead_size;
203 }
204 }
205
206 /* After using skb->sk to propagate sk through crypto async callback
207 * we need to NULL it again.
208 */
209 skb->sk = NULL;
210
211
212 /* Free the destination pages if skb was not decrypted inplace */
213 if (sgout != sgin) {
214 /* Skip the first S/G entry as it points to AAD */
215 for_each_sg(sg_next(sgout), sg, UINT_MAX, pages) {
216 if (!sg)
217 break;
218 put_page(sg_page(sg));
219 }
220 }
221
222 kfree(aead_req);
223
224 spin_lock_bh(&ctx->decrypt_compl_lock);
225 pending = atomic_dec_return(&ctx->decrypt_pending);
226
227 if (!pending && ctx->async_notify)
228 complete(&ctx->async_wait.completion);
229 spin_unlock_bh(&ctx->decrypt_compl_lock);
230 }
231
tls_do_decryption(struct sock * sk,struct sk_buff * skb,struct scatterlist * sgin,struct scatterlist * sgout,char * iv_recv,size_t data_len,struct aead_request * aead_req,struct tls_decrypt_arg * darg)232 static int tls_do_decryption(struct sock *sk,
233 struct sk_buff *skb,
234 struct scatterlist *sgin,
235 struct scatterlist *sgout,
236 char *iv_recv,
237 size_t data_len,
238 struct aead_request *aead_req,
239 struct tls_decrypt_arg *darg)
240 {
241 struct tls_context *tls_ctx = tls_get_ctx(sk);
242 struct tls_prot_info *prot = &tls_ctx->prot_info;
243 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
244 int ret;
245
246 aead_request_set_tfm(aead_req, ctx->aead_recv);
247 aead_request_set_ad(aead_req, prot->aad_size);
248 aead_request_set_crypt(aead_req, sgin, sgout,
249 data_len + prot->tag_size,
250 (u8 *)iv_recv);
251
252 if (darg->async) {
253 /* Using skb->sk to push sk through to crypto async callback
254 * handler. This allows propagating errors up to the socket
255 * if needed. It _must_ be cleared in the async handler
256 * before consume_skb is called. We _know_ skb->sk is NULL
257 * because it is a clone from strparser.
258 */
259 skb->sk = sk;
260 aead_request_set_callback(aead_req,
261 CRYPTO_TFM_REQ_MAY_BACKLOG,
262 tls_decrypt_done, skb);
263 atomic_inc(&ctx->decrypt_pending);
264 } else {
265 aead_request_set_callback(aead_req,
266 CRYPTO_TFM_REQ_MAY_BACKLOG,
267 crypto_req_done, &ctx->async_wait);
268 }
269
270 ret = crypto_aead_decrypt(aead_req);
271 if (ret == -EINPROGRESS) {
272 if (darg->async)
273 return 0;
274
275 ret = crypto_wait_req(ret, &ctx->async_wait);
276 } else if (darg->async) {
277 atomic_dec(&ctx->decrypt_pending);
278 }
279 darg->async = false;
280
281 return ret;
282 }
283
tls_trim_both_msgs(struct sock * sk,int target_size)284 static void tls_trim_both_msgs(struct sock *sk, int target_size)
285 {
286 struct tls_context *tls_ctx = tls_get_ctx(sk);
287 struct tls_prot_info *prot = &tls_ctx->prot_info;
288 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
289 struct tls_rec *rec = ctx->open_rec;
290
291 sk_msg_trim(sk, &rec->msg_plaintext, target_size);
292 if (target_size > 0)
293 target_size += prot->overhead_size;
294 sk_msg_trim(sk, &rec->msg_encrypted, target_size);
295 }
296
tls_alloc_encrypted_msg(struct sock * sk,int len)297 static int tls_alloc_encrypted_msg(struct sock *sk, int len)
298 {
299 struct tls_context *tls_ctx = tls_get_ctx(sk);
300 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
301 struct tls_rec *rec = ctx->open_rec;
302 struct sk_msg *msg_en = &rec->msg_encrypted;
303
304 return sk_msg_alloc(sk, msg_en, len, 0);
305 }
306
tls_clone_plaintext_msg(struct sock * sk,int required)307 static int tls_clone_plaintext_msg(struct sock *sk, int required)
308 {
309 struct tls_context *tls_ctx = tls_get_ctx(sk);
310 struct tls_prot_info *prot = &tls_ctx->prot_info;
311 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
312 struct tls_rec *rec = ctx->open_rec;
313 struct sk_msg *msg_pl = &rec->msg_plaintext;
314 struct sk_msg *msg_en = &rec->msg_encrypted;
315 int skip, len;
316
317 /* We add page references worth len bytes from encrypted sg
318 * at the end of plaintext sg. It is guaranteed that msg_en
319 * has enough required room (ensured by caller).
320 */
321 len = required - msg_pl->sg.size;
322
323 /* Skip initial bytes in msg_en's data to be able to use
324 * same offset of both plain and encrypted data.
325 */
326 skip = prot->prepend_size + msg_pl->sg.size;
327
328 return sk_msg_clone(sk, msg_pl, msg_en, skip, len);
329 }
330
tls_get_rec(struct sock * sk)331 static struct tls_rec *tls_get_rec(struct sock *sk)
332 {
333 struct tls_context *tls_ctx = tls_get_ctx(sk);
334 struct tls_prot_info *prot = &tls_ctx->prot_info;
335 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
336 struct sk_msg *msg_pl, *msg_en;
337 struct tls_rec *rec;
338 int mem_size;
339
340 mem_size = sizeof(struct tls_rec) + crypto_aead_reqsize(ctx->aead_send);
341
342 rec = kzalloc(mem_size, sk->sk_allocation);
343 if (!rec)
344 return NULL;
345
346 msg_pl = &rec->msg_plaintext;
347 msg_en = &rec->msg_encrypted;
348
349 sk_msg_init(msg_pl);
350 sk_msg_init(msg_en);
351
352 sg_init_table(rec->sg_aead_in, 2);
353 sg_set_buf(&rec->sg_aead_in[0], rec->aad_space, prot->aad_size);
354 sg_unmark_end(&rec->sg_aead_in[1]);
355
356 sg_init_table(rec->sg_aead_out, 2);
357 sg_set_buf(&rec->sg_aead_out[0], rec->aad_space, prot->aad_size);
358 sg_unmark_end(&rec->sg_aead_out[1]);
359
360 return rec;
361 }
362
tls_free_rec(struct sock * sk,struct tls_rec * rec)363 static void tls_free_rec(struct sock *sk, struct tls_rec *rec)
364 {
365 sk_msg_free(sk, &rec->msg_encrypted);
366 sk_msg_free(sk, &rec->msg_plaintext);
367 kfree(rec);
368 }
369
tls_free_open_rec(struct sock * sk)370 static void tls_free_open_rec(struct sock *sk)
371 {
372 struct tls_context *tls_ctx = tls_get_ctx(sk);
373 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
374 struct tls_rec *rec = ctx->open_rec;
375
376 if (rec) {
377 tls_free_rec(sk, rec);
378 ctx->open_rec = NULL;
379 }
380 }
381
tls_tx_records(struct sock * sk,int flags)382 int tls_tx_records(struct sock *sk, int flags)
383 {
384 struct tls_context *tls_ctx = tls_get_ctx(sk);
385 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
386 struct tls_rec *rec, *tmp;
387 struct sk_msg *msg_en;
388 int tx_flags, rc = 0;
389
390 if (tls_is_partially_sent_record(tls_ctx)) {
391 rec = list_first_entry(&ctx->tx_list,
392 struct tls_rec, list);
393
394 if (flags == -1)
395 tx_flags = rec->tx_flags;
396 else
397 tx_flags = flags;
398
399 rc = tls_push_partial_record(sk, tls_ctx, tx_flags);
400 if (rc)
401 goto tx_err;
402
403 /* Full record has been transmitted.
404 * Remove the head of tx_list
405 */
406 list_del(&rec->list);
407 sk_msg_free(sk, &rec->msg_plaintext);
408 kfree(rec);
409 }
410
411 /* Tx all ready records */
412 list_for_each_entry_safe(rec, tmp, &ctx->tx_list, list) {
413 if (READ_ONCE(rec->tx_ready)) {
414 if (flags == -1)
415 tx_flags = rec->tx_flags;
416 else
417 tx_flags = flags;
418
419 msg_en = &rec->msg_encrypted;
420 rc = tls_push_sg(sk, tls_ctx,
421 &msg_en->sg.data[msg_en->sg.curr],
422 0, tx_flags);
423 if (rc)
424 goto tx_err;
425
426 list_del(&rec->list);
427 sk_msg_free(sk, &rec->msg_plaintext);
428 kfree(rec);
429 } else {
430 break;
431 }
432 }
433
434 tx_err:
435 if (rc < 0 && rc != -EAGAIN)
436 tls_err_abort(sk, -EBADMSG);
437
438 return rc;
439 }
440
tls_encrypt_done(struct crypto_async_request * req,int err)441 static void tls_encrypt_done(struct crypto_async_request *req, int err)
442 {
443 struct aead_request *aead_req = (struct aead_request *)req;
444 struct sock *sk = req->data;
445 struct tls_context *tls_ctx = tls_get_ctx(sk);
446 struct tls_prot_info *prot = &tls_ctx->prot_info;
447 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
448 struct scatterlist *sge;
449 struct sk_msg *msg_en;
450 struct tls_rec *rec;
451 bool ready = false;
452 int pending;
453
454 rec = container_of(aead_req, struct tls_rec, aead_req);
455 msg_en = &rec->msg_encrypted;
456
457 sge = sk_msg_elem(msg_en, msg_en->sg.curr);
458 sge->offset -= prot->prepend_size;
459 sge->length += prot->prepend_size;
460
461 /* Check if error is previously set on socket */
462 if (err || sk->sk_err) {
463 rec = NULL;
464
465 /* If err is already set on socket, return the same code */
466 if (sk->sk_err) {
467 ctx->async_wait.err = -sk->sk_err;
468 } else {
469 ctx->async_wait.err = err;
470 tls_err_abort(sk, err);
471 }
472 }
473
474 if (rec) {
475 struct tls_rec *first_rec;
476
477 /* Mark the record as ready for transmission */
478 smp_store_mb(rec->tx_ready, true);
479
480 /* If received record is at head of tx_list, schedule tx */
481 first_rec = list_first_entry(&ctx->tx_list,
482 struct tls_rec, list);
483 if (rec == first_rec)
484 ready = true;
485 }
486
487 spin_lock_bh(&ctx->encrypt_compl_lock);
488 pending = atomic_dec_return(&ctx->encrypt_pending);
489
490 if (!pending && ctx->async_notify)
491 complete(&ctx->async_wait.completion);
492 spin_unlock_bh(&ctx->encrypt_compl_lock);
493
494 if (!ready)
495 return;
496
497 /* Schedule the transmission */
498 if (!test_and_set_bit(BIT_TX_SCHEDULED, &ctx->tx_bitmask))
499 schedule_delayed_work(&ctx->tx_work.work, 1);
500 }
501
tls_do_encryption(struct sock * sk,struct tls_context * tls_ctx,struct tls_sw_context_tx * ctx,struct aead_request * aead_req,size_t data_len,u32 start)502 static int tls_do_encryption(struct sock *sk,
503 struct tls_context *tls_ctx,
504 struct tls_sw_context_tx *ctx,
505 struct aead_request *aead_req,
506 size_t data_len, u32 start)
507 {
508 struct tls_prot_info *prot = &tls_ctx->prot_info;
509 struct tls_rec *rec = ctx->open_rec;
510 struct sk_msg *msg_en = &rec->msg_encrypted;
511 struct scatterlist *sge = sk_msg_elem(msg_en, start);
512 int rc, iv_offset = 0;
513
514 /* For CCM based ciphers, first byte of IV is a constant */
515 if (prot->cipher_type == TLS_CIPHER_AES_CCM_128) {
516 rec->iv_data[0] = TLS_AES_CCM_IV_B0_BYTE;
517 iv_offset = 1;
518 }
519
520 memcpy(&rec->iv_data[iv_offset], tls_ctx->tx.iv,
521 prot->iv_size + prot->salt_size);
522
523 xor_iv_with_seq(prot, rec->iv_data + iv_offset, tls_ctx->tx.rec_seq);
524
525 sge->offset += prot->prepend_size;
526 sge->length -= prot->prepend_size;
527
528 msg_en->sg.curr = start;
529
530 aead_request_set_tfm(aead_req, ctx->aead_send);
531 aead_request_set_ad(aead_req, prot->aad_size);
532 aead_request_set_crypt(aead_req, rec->sg_aead_in,
533 rec->sg_aead_out,
534 data_len, rec->iv_data);
535
536 aead_request_set_callback(aead_req, CRYPTO_TFM_REQ_MAY_BACKLOG,
537 tls_encrypt_done, sk);
538
539 /* Add the record in tx_list */
540 list_add_tail((struct list_head *)&rec->list, &ctx->tx_list);
541 atomic_inc(&ctx->encrypt_pending);
542
543 rc = crypto_aead_encrypt(aead_req);
544 if (!rc || rc != -EINPROGRESS) {
545 atomic_dec(&ctx->encrypt_pending);
546 sge->offset -= prot->prepend_size;
547 sge->length += prot->prepend_size;
548 }
549
550 if (!rc) {
551 WRITE_ONCE(rec->tx_ready, true);
552 } else if (rc != -EINPROGRESS) {
553 list_del(&rec->list);
554 return rc;
555 }
556
557 /* Unhook the record from context if encryption is not failure */
558 ctx->open_rec = NULL;
559 tls_advance_record_sn(sk, prot, &tls_ctx->tx);
560 return rc;
561 }
562
tls_split_open_record(struct sock * sk,struct tls_rec * from,struct tls_rec ** to,struct sk_msg * msg_opl,struct sk_msg * msg_oen,u32 split_point,u32 tx_overhead_size,u32 * orig_end)563 static int tls_split_open_record(struct sock *sk, struct tls_rec *from,
564 struct tls_rec **to, struct sk_msg *msg_opl,
565 struct sk_msg *msg_oen, u32 split_point,
566 u32 tx_overhead_size, u32 *orig_end)
567 {
568 u32 i, j, bytes = 0, apply = msg_opl->apply_bytes;
569 struct scatterlist *sge, *osge, *nsge;
570 u32 orig_size = msg_opl->sg.size;
571 struct scatterlist tmp = { };
572 struct sk_msg *msg_npl;
573 struct tls_rec *new;
574 int ret;
575
576 new = tls_get_rec(sk);
577 if (!new)
578 return -ENOMEM;
579 ret = sk_msg_alloc(sk, &new->msg_encrypted, msg_opl->sg.size +
580 tx_overhead_size, 0);
581 if (ret < 0) {
582 tls_free_rec(sk, new);
583 return ret;
584 }
585
586 *orig_end = msg_opl->sg.end;
587 i = msg_opl->sg.start;
588 sge = sk_msg_elem(msg_opl, i);
589 while (apply && sge->length) {
590 if (sge->length > apply) {
591 u32 len = sge->length - apply;
592
593 get_page(sg_page(sge));
594 sg_set_page(&tmp, sg_page(sge), len,
595 sge->offset + apply);
596 sge->length = apply;
597 bytes += apply;
598 apply = 0;
599 } else {
600 apply -= sge->length;
601 bytes += sge->length;
602 }
603
604 sk_msg_iter_var_next(i);
605 if (i == msg_opl->sg.end)
606 break;
607 sge = sk_msg_elem(msg_opl, i);
608 }
609
610 msg_opl->sg.end = i;
611 msg_opl->sg.curr = i;
612 msg_opl->sg.copybreak = 0;
613 msg_opl->apply_bytes = 0;
614 msg_opl->sg.size = bytes;
615
616 msg_npl = &new->msg_plaintext;
617 msg_npl->apply_bytes = apply;
618 msg_npl->sg.size = orig_size - bytes;
619
620 j = msg_npl->sg.start;
621 nsge = sk_msg_elem(msg_npl, j);
622 if (tmp.length) {
623 memcpy(nsge, &tmp, sizeof(*nsge));
624 sk_msg_iter_var_next(j);
625 nsge = sk_msg_elem(msg_npl, j);
626 }
627
628 osge = sk_msg_elem(msg_opl, i);
629 while (osge->length) {
630 memcpy(nsge, osge, sizeof(*nsge));
631 sg_unmark_end(nsge);
632 sk_msg_iter_var_next(i);
633 sk_msg_iter_var_next(j);
634 if (i == *orig_end)
635 break;
636 osge = sk_msg_elem(msg_opl, i);
637 nsge = sk_msg_elem(msg_npl, j);
638 }
639
640 msg_npl->sg.end = j;
641 msg_npl->sg.curr = j;
642 msg_npl->sg.copybreak = 0;
643
644 *to = new;
645 return 0;
646 }
647
tls_merge_open_record(struct sock * sk,struct tls_rec * to,struct tls_rec * from,u32 orig_end)648 static void tls_merge_open_record(struct sock *sk, struct tls_rec *to,
649 struct tls_rec *from, u32 orig_end)
650 {
651 struct sk_msg *msg_npl = &from->msg_plaintext;
652 struct sk_msg *msg_opl = &to->msg_plaintext;
653 struct scatterlist *osge, *nsge;
654 u32 i, j;
655
656 i = msg_opl->sg.end;
657 sk_msg_iter_var_prev(i);
658 j = msg_npl->sg.start;
659
660 osge = sk_msg_elem(msg_opl, i);
661 nsge = sk_msg_elem(msg_npl, j);
662
663 if (sg_page(osge) == sg_page(nsge) &&
664 osge->offset + osge->length == nsge->offset) {
665 osge->length += nsge->length;
666 put_page(sg_page(nsge));
667 }
668
669 msg_opl->sg.end = orig_end;
670 msg_opl->sg.curr = orig_end;
671 msg_opl->sg.copybreak = 0;
672 msg_opl->apply_bytes = msg_opl->sg.size + msg_npl->sg.size;
673 msg_opl->sg.size += msg_npl->sg.size;
674
675 sk_msg_free(sk, &to->msg_encrypted);
676 sk_msg_xfer_full(&to->msg_encrypted, &from->msg_encrypted);
677
678 kfree(from);
679 }
680
tls_push_record(struct sock * sk,int flags,unsigned char record_type)681 static int tls_push_record(struct sock *sk, int flags,
682 unsigned char record_type)
683 {
684 struct tls_context *tls_ctx = tls_get_ctx(sk);
685 struct tls_prot_info *prot = &tls_ctx->prot_info;
686 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
687 struct tls_rec *rec = ctx->open_rec, *tmp = NULL;
688 u32 i, split_point, orig_end;
689 struct sk_msg *msg_pl, *msg_en;
690 struct aead_request *req;
691 bool split;
692 int rc;
693
694 if (!rec)
695 return 0;
696
697 msg_pl = &rec->msg_plaintext;
698 msg_en = &rec->msg_encrypted;
699
700 split_point = msg_pl->apply_bytes;
701 split = split_point && split_point < msg_pl->sg.size;
702 if (unlikely((!split &&
703 msg_pl->sg.size +
704 prot->overhead_size > msg_en->sg.size) ||
705 (split &&
706 split_point +
707 prot->overhead_size > msg_en->sg.size))) {
708 split = true;
709 split_point = msg_en->sg.size;
710 }
711 if (split) {
712 rc = tls_split_open_record(sk, rec, &tmp, msg_pl, msg_en,
713 split_point, prot->overhead_size,
714 &orig_end);
715 if (rc < 0)
716 return rc;
717 /* This can happen if above tls_split_open_record allocates
718 * a single large encryption buffer instead of two smaller
719 * ones. In this case adjust pointers and continue without
720 * split.
721 */
722 if (!msg_pl->sg.size) {
723 tls_merge_open_record(sk, rec, tmp, orig_end);
724 msg_pl = &rec->msg_plaintext;
725 msg_en = &rec->msg_encrypted;
726 split = false;
727 }
728 sk_msg_trim(sk, msg_en, msg_pl->sg.size +
729 prot->overhead_size);
730 }
731
732 rec->tx_flags = flags;
733 req = &rec->aead_req;
734
735 i = msg_pl->sg.end;
736 sk_msg_iter_var_prev(i);
737
738 rec->content_type = record_type;
739 if (prot->version == TLS_1_3_VERSION) {
740 /* Add content type to end of message. No padding added */
741 sg_set_buf(&rec->sg_content_type, &rec->content_type, 1);
742 sg_mark_end(&rec->sg_content_type);
743 sg_chain(msg_pl->sg.data, msg_pl->sg.end + 1,
744 &rec->sg_content_type);
745 } else {
746 sg_mark_end(sk_msg_elem(msg_pl, i));
747 }
748
749 if (msg_pl->sg.end < msg_pl->sg.start) {
750 sg_chain(&msg_pl->sg.data[msg_pl->sg.start],
751 MAX_SKB_FRAGS - msg_pl->sg.start + 1,
752 msg_pl->sg.data);
753 }
754
755 i = msg_pl->sg.start;
756 sg_chain(rec->sg_aead_in, 2, &msg_pl->sg.data[i]);
757
758 i = msg_en->sg.end;
759 sk_msg_iter_var_prev(i);
760 sg_mark_end(sk_msg_elem(msg_en, i));
761
762 i = msg_en->sg.start;
763 sg_chain(rec->sg_aead_out, 2, &msg_en->sg.data[i]);
764
765 tls_make_aad(rec->aad_space, msg_pl->sg.size + prot->tail_size,
766 tls_ctx->tx.rec_seq, record_type, prot);
767
768 tls_fill_prepend(tls_ctx,
769 page_address(sg_page(&msg_en->sg.data[i])) +
770 msg_en->sg.data[i].offset,
771 msg_pl->sg.size + prot->tail_size,
772 record_type);
773
774 tls_ctx->pending_open_record_frags = false;
775
776 rc = tls_do_encryption(sk, tls_ctx, ctx, req,
777 msg_pl->sg.size + prot->tail_size, i);
778 if (rc < 0) {
779 if (rc != -EINPROGRESS) {
780 tls_err_abort(sk, -EBADMSG);
781 if (split) {
782 tls_ctx->pending_open_record_frags = true;
783 tls_merge_open_record(sk, rec, tmp, orig_end);
784 }
785 }
786 ctx->async_capable = 1;
787 return rc;
788 } else if (split) {
789 msg_pl = &tmp->msg_plaintext;
790 msg_en = &tmp->msg_encrypted;
791 sk_msg_trim(sk, msg_en, msg_pl->sg.size + prot->overhead_size);
792 tls_ctx->pending_open_record_frags = true;
793 ctx->open_rec = tmp;
794 }
795
796 return tls_tx_records(sk, flags);
797 }
798
bpf_exec_tx_verdict(struct sk_msg * msg,struct sock * sk,bool full_record,u8 record_type,ssize_t * copied,int flags)799 static int bpf_exec_tx_verdict(struct sk_msg *msg, struct sock *sk,
800 bool full_record, u8 record_type,
801 ssize_t *copied, int flags)
802 {
803 struct tls_context *tls_ctx = tls_get_ctx(sk);
804 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
805 struct sk_msg msg_redir = { };
806 struct sk_psock *psock;
807 struct sock *sk_redir;
808 struct tls_rec *rec;
809 bool enospc, policy, redir_ingress;
810 int err = 0, send;
811 u32 delta = 0;
812
813 policy = !(flags & MSG_SENDPAGE_NOPOLICY);
814 psock = sk_psock_get(sk);
815 if (!psock || !policy) {
816 err = tls_push_record(sk, flags, record_type);
817 if (err && err != -EINPROGRESS && sk->sk_err == EBADMSG) {
818 *copied -= sk_msg_free(sk, msg);
819 tls_free_open_rec(sk);
820 err = -sk->sk_err;
821 }
822 if (psock)
823 sk_psock_put(sk, psock);
824 return err;
825 }
826 more_data:
827 enospc = sk_msg_full(msg);
828 if (psock->eval == __SK_NONE) {
829 delta = msg->sg.size;
830 psock->eval = sk_psock_msg_verdict(sk, psock, msg);
831 delta -= msg->sg.size;
832 }
833 if (msg->cork_bytes && msg->cork_bytes > msg->sg.size &&
834 !enospc && !full_record) {
835 err = -ENOSPC;
836 goto out_err;
837 }
838 msg->cork_bytes = 0;
839 send = msg->sg.size;
840 if (msg->apply_bytes && msg->apply_bytes < send)
841 send = msg->apply_bytes;
842
843 switch (psock->eval) {
844 case __SK_PASS:
845 err = tls_push_record(sk, flags, record_type);
846 if (err && err != -EINPROGRESS && sk->sk_err == EBADMSG) {
847 *copied -= sk_msg_free(sk, msg);
848 tls_free_open_rec(sk);
849 err = -sk->sk_err;
850 goto out_err;
851 }
852 break;
853 case __SK_REDIRECT:
854 redir_ingress = psock->redir_ingress;
855 sk_redir = psock->sk_redir;
856 memcpy(&msg_redir, msg, sizeof(*msg));
857 if (msg->apply_bytes < send)
858 msg->apply_bytes = 0;
859 else
860 msg->apply_bytes -= send;
861 sk_msg_return_zero(sk, msg, send);
862 msg->sg.size -= send;
863 release_sock(sk);
864 err = tcp_bpf_sendmsg_redir(sk_redir, redir_ingress,
865 &msg_redir, send, flags);
866 lock_sock(sk);
867 if (err < 0) {
868 *copied -= sk_msg_free_nocharge(sk, &msg_redir);
869 msg->sg.size = 0;
870 }
871 if (msg->sg.size == 0)
872 tls_free_open_rec(sk);
873 break;
874 case __SK_DROP:
875 default:
876 sk_msg_free_partial(sk, msg, send);
877 if (msg->apply_bytes < send)
878 msg->apply_bytes = 0;
879 else
880 msg->apply_bytes -= send;
881 if (msg->sg.size == 0)
882 tls_free_open_rec(sk);
883 *copied -= (send + delta);
884 err = -EACCES;
885 }
886
887 if (likely(!err)) {
888 bool reset_eval = !ctx->open_rec;
889
890 rec = ctx->open_rec;
891 if (rec) {
892 msg = &rec->msg_plaintext;
893 if (!msg->apply_bytes)
894 reset_eval = true;
895 }
896 if (reset_eval) {
897 psock->eval = __SK_NONE;
898 if (psock->sk_redir) {
899 sock_put(psock->sk_redir);
900 psock->sk_redir = NULL;
901 }
902 }
903 if (rec)
904 goto more_data;
905 }
906 out_err:
907 sk_psock_put(sk, psock);
908 return err;
909 }
910
tls_sw_push_pending_record(struct sock * sk,int flags)911 static int tls_sw_push_pending_record(struct sock *sk, int flags)
912 {
913 struct tls_context *tls_ctx = tls_get_ctx(sk);
914 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
915 struct tls_rec *rec = ctx->open_rec;
916 struct sk_msg *msg_pl;
917 size_t copied;
918
919 if (!rec)
920 return 0;
921
922 msg_pl = &rec->msg_plaintext;
923 copied = msg_pl->sg.size;
924 if (!copied)
925 return 0;
926
927 return bpf_exec_tx_verdict(msg_pl, sk, true, TLS_RECORD_TYPE_DATA,
928 &copied, flags);
929 }
930
tls_sw_sendmsg(struct sock * sk,struct msghdr * msg,size_t size)931 int tls_sw_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)
932 {
933 long timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT);
934 struct tls_context *tls_ctx = tls_get_ctx(sk);
935 struct tls_prot_info *prot = &tls_ctx->prot_info;
936 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
937 bool async_capable = ctx->async_capable;
938 unsigned char record_type = TLS_RECORD_TYPE_DATA;
939 bool is_kvec = iov_iter_is_kvec(&msg->msg_iter);
940 bool eor = !(msg->msg_flags & MSG_MORE);
941 size_t try_to_copy;
942 ssize_t copied = 0;
943 struct sk_msg *msg_pl, *msg_en;
944 struct tls_rec *rec;
945 int required_size;
946 int num_async = 0;
947 bool full_record;
948 int record_room;
949 int num_zc = 0;
950 int orig_size;
951 int ret = 0;
952 int pending;
953
954 if (msg->msg_flags & ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL |
955 MSG_CMSG_COMPAT))
956 return -EOPNOTSUPP;
957
958 ret = mutex_lock_interruptible(&tls_ctx->tx_lock);
959 if (ret)
960 return ret;
961 lock_sock(sk);
962
963 if (unlikely(msg->msg_controllen)) {
964 ret = tls_proccess_cmsg(sk, msg, &record_type);
965 if (ret) {
966 if (ret == -EINPROGRESS)
967 num_async++;
968 else if (ret != -EAGAIN)
969 goto send_end;
970 }
971 }
972
973 while (msg_data_left(msg)) {
974 if (sk->sk_err) {
975 ret = -sk->sk_err;
976 goto send_end;
977 }
978
979 if (ctx->open_rec)
980 rec = ctx->open_rec;
981 else
982 rec = ctx->open_rec = tls_get_rec(sk);
983 if (!rec) {
984 ret = -ENOMEM;
985 goto send_end;
986 }
987
988 msg_pl = &rec->msg_plaintext;
989 msg_en = &rec->msg_encrypted;
990
991 orig_size = msg_pl->sg.size;
992 full_record = false;
993 try_to_copy = msg_data_left(msg);
994 record_room = TLS_MAX_PAYLOAD_SIZE - msg_pl->sg.size;
995 if (try_to_copy >= record_room) {
996 try_to_copy = record_room;
997 full_record = true;
998 }
999
1000 required_size = msg_pl->sg.size + try_to_copy +
1001 prot->overhead_size;
1002
1003 if (!sk_stream_memory_free(sk))
1004 goto wait_for_sndbuf;
1005
1006 alloc_encrypted:
1007 ret = tls_alloc_encrypted_msg(sk, required_size);
1008 if (ret) {
1009 if (ret != -ENOSPC)
1010 goto wait_for_memory;
1011
1012 /* Adjust try_to_copy according to the amount that was
1013 * actually allocated. The difference is due
1014 * to max sg elements limit
1015 */
1016 try_to_copy -= required_size - msg_en->sg.size;
1017 full_record = true;
1018 }
1019
1020 if (!is_kvec && (full_record || eor) && !async_capable) {
1021 u32 first = msg_pl->sg.end;
1022
1023 ret = sk_msg_zerocopy_from_iter(sk, &msg->msg_iter,
1024 msg_pl, try_to_copy);
1025 if (ret)
1026 goto fallback_to_reg_send;
1027
1028 num_zc++;
1029 copied += try_to_copy;
1030
1031 sk_msg_sg_copy_set(msg_pl, first);
1032 ret = bpf_exec_tx_verdict(msg_pl, sk, full_record,
1033 record_type, &copied,
1034 msg->msg_flags);
1035 if (ret) {
1036 if (ret == -EINPROGRESS)
1037 num_async++;
1038 else if (ret == -ENOMEM)
1039 goto wait_for_memory;
1040 else if (ctx->open_rec && ret == -ENOSPC)
1041 goto rollback_iter;
1042 else if (ret != -EAGAIN)
1043 goto send_end;
1044 }
1045 continue;
1046 rollback_iter:
1047 copied -= try_to_copy;
1048 sk_msg_sg_copy_clear(msg_pl, first);
1049 iov_iter_revert(&msg->msg_iter,
1050 msg_pl->sg.size - orig_size);
1051 fallback_to_reg_send:
1052 sk_msg_trim(sk, msg_pl, orig_size);
1053 }
1054
1055 required_size = msg_pl->sg.size + try_to_copy;
1056
1057 ret = tls_clone_plaintext_msg(sk, required_size);
1058 if (ret) {
1059 if (ret != -ENOSPC)
1060 goto send_end;
1061
1062 /* Adjust try_to_copy according to the amount that was
1063 * actually allocated. The difference is due
1064 * to max sg elements limit
1065 */
1066 try_to_copy -= required_size - msg_pl->sg.size;
1067 full_record = true;
1068 sk_msg_trim(sk, msg_en,
1069 msg_pl->sg.size + prot->overhead_size);
1070 }
1071
1072 if (try_to_copy) {
1073 ret = sk_msg_memcopy_from_iter(sk, &msg->msg_iter,
1074 msg_pl, try_to_copy);
1075 if (ret < 0)
1076 goto trim_sgl;
1077 }
1078
1079 /* Open records defined only if successfully copied, otherwise
1080 * we would trim the sg but not reset the open record frags.
1081 */
1082 tls_ctx->pending_open_record_frags = true;
1083 copied += try_to_copy;
1084 if (full_record || eor) {
1085 ret = bpf_exec_tx_verdict(msg_pl, sk, full_record,
1086 record_type, &copied,
1087 msg->msg_flags);
1088 if (ret) {
1089 if (ret == -EINPROGRESS)
1090 num_async++;
1091 else if (ret == -ENOMEM)
1092 goto wait_for_memory;
1093 else if (ret != -EAGAIN) {
1094 if (ret == -ENOSPC)
1095 ret = 0;
1096 goto send_end;
1097 }
1098 }
1099 }
1100
1101 continue;
1102
1103 wait_for_sndbuf:
1104 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
1105 wait_for_memory:
1106 ret = sk_stream_wait_memory(sk, &timeo);
1107 if (ret) {
1108 trim_sgl:
1109 if (ctx->open_rec)
1110 tls_trim_both_msgs(sk, orig_size);
1111 goto send_end;
1112 }
1113
1114 if (ctx->open_rec && msg_en->sg.size < required_size)
1115 goto alloc_encrypted;
1116 }
1117
1118 if (!num_async) {
1119 goto send_end;
1120 } else if (num_zc) {
1121 /* Wait for pending encryptions to get completed */
1122 spin_lock_bh(&ctx->encrypt_compl_lock);
1123 ctx->async_notify = true;
1124
1125 pending = atomic_read(&ctx->encrypt_pending);
1126 spin_unlock_bh(&ctx->encrypt_compl_lock);
1127 if (pending)
1128 crypto_wait_req(-EINPROGRESS, &ctx->async_wait);
1129 else
1130 reinit_completion(&ctx->async_wait.completion);
1131
1132 /* There can be no concurrent accesses, since we have no
1133 * pending encrypt operations
1134 */
1135 WRITE_ONCE(ctx->async_notify, false);
1136
1137 if (ctx->async_wait.err) {
1138 ret = ctx->async_wait.err;
1139 copied = 0;
1140 }
1141 }
1142
1143 /* Transmit if any encryptions have completed */
1144 if (test_and_clear_bit(BIT_TX_SCHEDULED, &ctx->tx_bitmask)) {
1145 cancel_delayed_work(&ctx->tx_work.work);
1146 tls_tx_records(sk, msg->msg_flags);
1147 }
1148
1149 send_end:
1150 ret = sk_stream_error(sk, msg->msg_flags, ret);
1151
1152 release_sock(sk);
1153 mutex_unlock(&tls_ctx->tx_lock);
1154 return copied > 0 ? copied : ret;
1155 }
1156
tls_sw_do_sendpage(struct sock * sk,struct page * page,int offset,size_t size,int flags)1157 static int tls_sw_do_sendpage(struct sock *sk, struct page *page,
1158 int offset, size_t size, int flags)
1159 {
1160 long timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT);
1161 struct tls_context *tls_ctx = tls_get_ctx(sk);
1162 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
1163 struct tls_prot_info *prot = &tls_ctx->prot_info;
1164 unsigned char record_type = TLS_RECORD_TYPE_DATA;
1165 struct sk_msg *msg_pl;
1166 struct tls_rec *rec;
1167 int num_async = 0;
1168 ssize_t copied = 0;
1169 bool full_record;
1170 int record_room;
1171 int ret = 0;
1172 bool eor;
1173
1174 eor = !(flags & MSG_SENDPAGE_NOTLAST);
1175 sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk);
1176
1177 /* Call the sk_stream functions to manage the sndbuf mem. */
1178 while (size > 0) {
1179 size_t copy, required_size;
1180
1181 if (sk->sk_err) {
1182 ret = -sk->sk_err;
1183 goto sendpage_end;
1184 }
1185
1186 if (ctx->open_rec)
1187 rec = ctx->open_rec;
1188 else
1189 rec = ctx->open_rec = tls_get_rec(sk);
1190 if (!rec) {
1191 ret = -ENOMEM;
1192 goto sendpage_end;
1193 }
1194
1195 msg_pl = &rec->msg_plaintext;
1196
1197 full_record = false;
1198 record_room = TLS_MAX_PAYLOAD_SIZE - msg_pl->sg.size;
1199 copy = size;
1200 if (copy >= record_room) {
1201 copy = record_room;
1202 full_record = true;
1203 }
1204
1205 required_size = msg_pl->sg.size + copy + prot->overhead_size;
1206
1207 if (!sk_stream_memory_free(sk))
1208 goto wait_for_sndbuf;
1209 alloc_payload:
1210 ret = tls_alloc_encrypted_msg(sk, required_size);
1211 if (ret) {
1212 if (ret != -ENOSPC)
1213 goto wait_for_memory;
1214
1215 /* Adjust copy according to the amount that was
1216 * actually allocated. The difference is due
1217 * to max sg elements limit
1218 */
1219 copy -= required_size - msg_pl->sg.size;
1220 full_record = true;
1221 }
1222
1223 sk_msg_page_add(msg_pl, page, copy, offset);
1224 msg_pl->sg.copybreak = 0;
1225 msg_pl->sg.curr = msg_pl->sg.end;
1226 sk_mem_charge(sk, copy);
1227
1228 offset += copy;
1229 size -= copy;
1230 copied += copy;
1231
1232 tls_ctx->pending_open_record_frags = true;
1233 if (full_record || eor || sk_msg_full(msg_pl)) {
1234 ret = bpf_exec_tx_verdict(msg_pl, sk, full_record,
1235 record_type, &copied, flags);
1236 if (ret) {
1237 if (ret == -EINPROGRESS)
1238 num_async++;
1239 else if (ret == -ENOMEM)
1240 goto wait_for_memory;
1241 else if (ret != -EAGAIN) {
1242 if (ret == -ENOSPC)
1243 ret = 0;
1244 goto sendpage_end;
1245 }
1246 }
1247 }
1248 continue;
1249 wait_for_sndbuf:
1250 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
1251 wait_for_memory:
1252 ret = sk_stream_wait_memory(sk, &timeo);
1253 if (ret) {
1254 if (ctx->open_rec)
1255 tls_trim_both_msgs(sk, msg_pl->sg.size);
1256 goto sendpage_end;
1257 }
1258
1259 if (ctx->open_rec)
1260 goto alloc_payload;
1261 }
1262
1263 if (num_async) {
1264 /* Transmit if any encryptions have completed */
1265 if (test_and_clear_bit(BIT_TX_SCHEDULED, &ctx->tx_bitmask)) {
1266 cancel_delayed_work(&ctx->tx_work.work);
1267 tls_tx_records(sk, flags);
1268 }
1269 }
1270 sendpage_end:
1271 ret = sk_stream_error(sk, flags, ret);
1272 return copied > 0 ? copied : ret;
1273 }
1274
tls_sw_sendpage_locked(struct sock * sk,struct page * page,int offset,size_t size,int flags)1275 int tls_sw_sendpage_locked(struct sock *sk, struct page *page,
1276 int offset, size_t size, int flags)
1277 {
1278 if (flags & ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL |
1279 MSG_SENDPAGE_NOTLAST | MSG_SENDPAGE_NOPOLICY |
1280 MSG_NO_SHARED_FRAGS))
1281 return -EOPNOTSUPP;
1282
1283 return tls_sw_do_sendpage(sk, page, offset, size, flags);
1284 }
1285
tls_sw_sendpage(struct sock * sk,struct page * page,int offset,size_t size,int flags)1286 int tls_sw_sendpage(struct sock *sk, struct page *page,
1287 int offset, size_t size, int flags)
1288 {
1289 struct tls_context *tls_ctx = tls_get_ctx(sk);
1290 int ret;
1291
1292 if (flags & ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL |
1293 MSG_SENDPAGE_NOTLAST | MSG_SENDPAGE_NOPOLICY))
1294 return -EOPNOTSUPP;
1295
1296 ret = mutex_lock_interruptible(&tls_ctx->tx_lock);
1297 if (ret)
1298 return ret;
1299 lock_sock(sk);
1300 ret = tls_sw_do_sendpage(sk, page, offset, size, flags);
1301 release_sock(sk);
1302 mutex_unlock(&tls_ctx->tx_lock);
1303 return ret;
1304 }
1305
tls_wait_data(struct sock * sk,struct sk_psock * psock,bool nonblock,long timeo,int * err)1306 static struct sk_buff *tls_wait_data(struct sock *sk, struct sk_psock *psock,
1307 bool nonblock, long timeo, int *err)
1308 {
1309 struct tls_context *tls_ctx = tls_get_ctx(sk);
1310 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
1311 struct sk_buff *skb;
1312 DEFINE_WAIT_FUNC(wait, woken_wake_function);
1313
1314 while (!(skb = ctx->recv_pkt) && sk_psock_queue_empty(psock)) {
1315 if (sk->sk_err) {
1316 *err = sock_error(sk);
1317 return NULL;
1318 }
1319
1320 if (!skb_queue_empty(&sk->sk_receive_queue)) {
1321 __strp_unpause(&ctx->strp);
1322 if (ctx->recv_pkt)
1323 return ctx->recv_pkt;
1324 }
1325
1326 if (sk->sk_shutdown & RCV_SHUTDOWN)
1327 return NULL;
1328
1329 if (sock_flag(sk, SOCK_DONE))
1330 return NULL;
1331
1332 if (nonblock || !timeo) {
1333 *err = -EAGAIN;
1334 return NULL;
1335 }
1336
1337 add_wait_queue(sk_sleep(sk), &wait);
1338 sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk);
1339 sk_wait_event(sk, &timeo,
1340 ctx->recv_pkt != skb ||
1341 !sk_psock_queue_empty(psock),
1342 &wait);
1343 sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk);
1344 remove_wait_queue(sk_sleep(sk), &wait);
1345
1346 /* Handle signals */
1347 if (signal_pending(current)) {
1348 *err = sock_intr_errno(timeo);
1349 return NULL;
1350 }
1351 }
1352
1353 return skb;
1354 }
1355
tls_setup_from_iter(struct iov_iter * from,int length,int * pages_used,struct scatterlist * to,int to_max_pages)1356 static int tls_setup_from_iter(struct iov_iter *from,
1357 int length, int *pages_used,
1358 struct scatterlist *to,
1359 int to_max_pages)
1360 {
1361 int rc = 0, i = 0, num_elem = *pages_used, maxpages;
1362 struct page *pages[MAX_SKB_FRAGS];
1363 unsigned int size = 0;
1364 ssize_t copied, use;
1365 size_t offset;
1366
1367 while (length > 0) {
1368 i = 0;
1369 maxpages = to_max_pages - num_elem;
1370 if (maxpages == 0) {
1371 rc = -EFAULT;
1372 goto out;
1373 }
1374 copied = iov_iter_get_pages(from, pages,
1375 length,
1376 maxpages, &offset);
1377 if (copied <= 0) {
1378 rc = -EFAULT;
1379 goto out;
1380 }
1381
1382 iov_iter_advance(from, copied);
1383
1384 length -= copied;
1385 size += copied;
1386 while (copied) {
1387 use = min_t(int, copied, PAGE_SIZE - offset);
1388
1389 sg_set_page(&to[num_elem],
1390 pages[i], use, offset);
1391 sg_unmark_end(&to[num_elem]);
1392 /* We do not uncharge memory from this API */
1393
1394 offset = 0;
1395 copied -= use;
1396
1397 i++;
1398 num_elem++;
1399 }
1400 }
1401 /* Mark the end in the last sg entry if newly added */
1402 if (num_elem > *pages_used)
1403 sg_mark_end(&to[num_elem - 1]);
1404 out:
1405 if (rc)
1406 iov_iter_revert(from, size);
1407 *pages_used = num_elem;
1408
1409 return rc;
1410 }
1411
1412 /* This function decrypts the input skb into either out_iov or in out_sg
1413 * or in skb buffers itself. The input parameter 'zc' indicates if
1414 * zero-copy mode needs to be tried or not. With zero-copy mode, either
1415 * out_iov or out_sg must be non-NULL. In case both out_iov and out_sg are
1416 * NULL, then the decryption happens inside skb buffers itself, i.e.
1417 * zero-copy gets disabled and 'zc' is updated.
1418 */
1419
decrypt_internal(struct sock * sk,struct sk_buff * skb,struct iov_iter * out_iov,struct scatterlist * out_sg,struct tls_decrypt_arg * darg)1420 static int decrypt_internal(struct sock *sk, struct sk_buff *skb,
1421 struct iov_iter *out_iov,
1422 struct scatterlist *out_sg,
1423 struct tls_decrypt_arg *darg)
1424 {
1425 struct tls_context *tls_ctx = tls_get_ctx(sk);
1426 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
1427 struct tls_prot_info *prot = &tls_ctx->prot_info;
1428 struct strp_msg *rxm = strp_msg(skb);
1429 struct tls_msg *tlm = tls_msg(skb);
1430 int n_sgin, n_sgout, nsg, mem_size, aead_size, err, pages = 0;
1431 struct aead_request *aead_req;
1432 struct sk_buff *unused;
1433 u8 *aad, *iv, *mem = NULL;
1434 struct scatterlist *sgin = NULL;
1435 struct scatterlist *sgout = NULL;
1436 const int data_len = rxm->full_len - prot->overhead_size +
1437 prot->tail_size;
1438 int iv_offset = 0;
1439
1440 if (darg->zc && (out_iov || out_sg)) {
1441 if (out_iov)
1442 n_sgout = iov_iter_npages(out_iov, INT_MAX) + 1;
1443 else
1444 n_sgout = sg_nents(out_sg);
1445 n_sgin = skb_nsg(skb, rxm->offset + prot->prepend_size,
1446 rxm->full_len - prot->prepend_size);
1447 } else {
1448 n_sgout = 0;
1449 darg->zc = false;
1450 n_sgin = skb_cow_data(skb, 0, &unused);
1451 }
1452
1453 if (n_sgin < 1)
1454 return -EBADMSG;
1455
1456 /* Increment to accommodate AAD */
1457 n_sgin = n_sgin + 1;
1458
1459 nsg = n_sgin + n_sgout;
1460
1461 aead_size = sizeof(*aead_req) + crypto_aead_reqsize(ctx->aead_recv);
1462 mem_size = aead_size + (nsg * sizeof(struct scatterlist));
1463 mem_size = mem_size + prot->aad_size;
1464 mem_size = mem_size + crypto_aead_ivsize(ctx->aead_recv);
1465
1466 /* Allocate a single block of memory which contains
1467 * aead_req || sgin[] || sgout[] || aad || iv.
1468 * This order achieves correct alignment for aead_req, sgin, sgout.
1469 */
1470 mem = kmalloc(mem_size, sk->sk_allocation);
1471 if (!mem)
1472 return -ENOMEM;
1473
1474 /* Segment the allocated memory */
1475 aead_req = (struct aead_request *)mem;
1476 sgin = (struct scatterlist *)(mem + aead_size);
1477 sgout = sgin + n_sgin;
1478 aad = (u8 *)(sgout + n_sgout);
1479 iv = aad + prot->aad_size;
1480
1481 /* For CCM based ciphers, first byte of nonce+iv is always '2' */
1482 if (prot->cipher_type == TLS_CIPHER_AES_CCM_128) {
1483 iv[0] = 2;
1484 iv_offset = 1;
1485 }
1486
1487 /* Prepare IV */
1488 err = skb_copy_bits(skb, rxm->offset + TLS_HEADER_SIZE,
1489 iv + iv_offset + prot->salt_size,
1490 prot->iv_size);
1491 if (err < 0) {
1492 kfree(mem);
1493 return err;
1494 }
1495 if (prot->version == TLS_1_3_VERSION ||
1496 prot->cipher_type == TLS_CIPHER_CHACHA20_POLY1305)
1497 memcpy(iv + iv_offset, tls_ctx->rx.iv,
1498 prot->iv_size + prot->salt_size);
1499 else
1500 memcpy(iv + iv_offset, tls_ctx->rx.iv, prot->salt_size);
1501
1502 xor_iv_with_seq(prot, iv + iv_offset, tls_ctx->rx.rec_seq);
1503
1504 /* Prepare AAD */
1505 tls_make_aad(aad, rxm->full_len - prot->overhead_size +
1506 prot->tail_size,
1507 tls_ctx->rx.rec_seq, tlm->control, prot);
1508
1509 /* Prepare sgin */
1510 sg_init_table(sgin, n_sgin);
1511 sg_set_buf(&sgin[0], aad, prot->aad_size);
1512 err = skb_to_sgvec(skb, &sgin[1],
1513 rxm->offset + prot->prepend_size,
1514 rxm->full_len - prot->prepend_size);
1515 if (err < 0) {
1516 kfree(mem);
1517 return err;
1518 }
1519
1520 if (n_sgout) {
1521 if (out_iov) {
1522 sg_init_table(sgout, n_sgout);
1523 sg_set_buf(&sgout[0], aad, prot->aad_size);
1524
1525 err = tls_setup_from_iter(out_iov, data_len,
1526 &pages, &sgout[1],
1527 (n_sgout - 1));
1528 if (err < 0)
1529 goto fallback_to_reg_recv;
1530 } else if (out_sg) {
1531 memcpy(sgout, out_sg, n_sgout * sizeof(*sgout));
1532 } else {
1533 goto fallback_to_reg_recv;
1534 }
1535 } else {
1536 fallback_to_reg_recv:
1537 sgout = sgin;
1538 pages = 0;
1539 darg->zc = false;
1540 }
1541
1542 /* Prepare and submit AEAD request */
1543 err = tls_do_decryption(sk, skb, sgin, sgout, iv,
1544 data_len, aead_req, darg);
1545 if (darg->async)
1546 return 0;
1547
1548 /* Release the pages in case iov was mapped to pages */
1549 for (; pages > 0; pages--)
1550 put_page(sg_page(&sgout[pages]));
1551
1552 kfree(mem);
1553 return err;
1554 }
1555
decrypt_skb_update(struct sock * sk,struct sk_buff * skb,struct iov_iter * dest,struct tls_decrypt_arg * darg)1556 static int decrypt_skb_update(struct sock *sk, struct sk_buff *skb,
1557 struct iov_iter *dest,
1558 struct tls_decrypt_arg *darg)
1559 {
1560 struct tls_context *tls_ctx = tls_get_ctx(sk);
1561 struct tls_prot_info *prot = &tls_ctx->prot_info;
1562 struct strp_msg *rxm = strp_msg(skb);
1563 struct tls_msg *tlm = tls_msg(skb);
1564 int pad, err;
1565
1566 if (tlm->decrypted) {
1567 darg->zc = false;
1568 darg->async = false;
1569 return 0;
1570 }
1571
1572 if (tls_ctx->rx_conf == TLS_HW) {
1573 err = tls_device_decrypted(sk, tls_ctx, skb, rxm);
1574 if (err < 0)
1575 return err;
1576 if (err > 0) {
1577 tlm->decrypted = 1;
1578 darg->zc = false;
1579 darg->async = false;
1580 goto decrypt_done;
1581 }
1582 }
1583
1584 err = decrypt_internal(sk, skb, dest, NULL, darg);
1585 if (err < 0) {
1586 if (err == -EBADMSG)
1587 TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSDECRYPTERROR);
1588 return err;
1589 }
1590 if (darg->async)
1591 goto decrypt_next;
1592
1593 decrypt_done:
1594 pad = padding_length(prot, skb);
1595 if (pad < 0)
1596 return pad;
1597
1598 rxm->full_len -= pad;
1599 rxm->offset += prot->prepend_size;
1600 rxm->full_len -= prot->overhead_size;
1601 tlm->decrypted = 1;
1602 decrypt_next:
1603 tls_advance_record_sn(sk, prot, &tls_ctx->rx);
1604
1605 return 0;
1606 }
1607
decrypt_skb(struct sock * sk,struct sk_buff * skb,struct scatterlist * sgout)1608 int decrypt_skb(struct sock *sk, struct sk_buff *skb,
1609 struct scatterlist *sgout)
1610 {
1611 struct tls_decrypt_arg darg = { .zc = true, };
1612
1613 return decrypt_internal(sk, skb, NULL, sgout, &darg);
1614 }
1615
tls_sw_advance_skb(struct sock * sk,struct sk_buff * skb,unsigned int len)1616 static bool tls_sw_advance_skb(struct sock *sk, struct sk_buff *skb,
1617 unsigned int len)
1618 {
1619 struct tls_context *tls_ctx = tls_get_ctx(sk);
1620 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
1621
1622 if (skb) {
1623 struct strp_msg *rxm = strp_msg(skb);
1624
1625 if (len < rxm->full_len) {
1626 rxm->offset += len;
1627 rxm->full_len -= len;
1628 return false;
1629 }
1630 consume_skb(skb);
1631 }
1632
1633 /* Finished with message */
1634 ctx->recv_pkt = NULL;
1635 __strp_unpause(&ctx->strp);
1636
1637 return true;
1638 }
1639
tls_record_content_type(struct msghdr * msg,struct tls_msg * tlm,u8 * control)1640 static int tls_record_content_type(struct msghdr *msg, struct tls_msg *tlm,
1641 u8 *control)
1642 {
1643 int err;
1644
1645 if (!*control) {
1646 *control = tlm->control;
1647 if (!*control)
1648 return -EBADMSG;
1649
1650 err = put_cmsg(msg, SOL_TLS, TLS_GET_RECORD_TYPE,
1651 sizeof(*control), control);
1652 if (*control != TLS_RECORD_TYPE_DATA) {
1653 if (err || msg->msg_flags & MSG_CTRUNC)
1654 return -EIO;
1655 }
1656 } else if (*control != tlm->control) {
1657 return 0;
1658 }
1659
1660 return 1;
1661 }
1662
1663 /* This function traverses the rx_list in tls receive context to copies the
1664 * decrypted records into the buffer provided by caller zero copy is not
1665 * true. Further, the records are removed from the rx_list if it is not a peek
1666 * case and the record has been consumed completely.
1667 */
process_rx_list(struct tls_sw_context_rx * ctx,struct msghdr * msg,u8 * control,size_t skip,size_t len,bool zc,bool is_peek)1668 static int process_rx_list(struct tls_sw_context_rx *ctx,
1669 struct msghdr *msg,
1670 u8 *control,
1671 size_t skip,
1672 size_t len,
1673 bool zc,
1674 bool is_peek)
1675 {
1676 struct sk_buff *skb = skb_peek(&ctx->rx_list);
1677 struct tls_msg *tlm;
1678 ssize_t copied = 0;
1679 int err;
1680
1681 while (skip && skb) {
1682 struct strp_msg *rxm = strp_msg(skb);
1683 tlm = tls_msg(skb);
1684
1685 err = tls_record_content_type(msg, tlm, control);
1686 if (err <= 0)
1687 return err;
1688
1689 if (skip < rxm->full_len)
1690 break;
1691
1692 skip = skip - rxm->full_len;
1693 skb = skb_peek_next(skb, &ctx->rx_list);
1694 }
1695
1696 while (len && skb) {
1697 struct sk_buff *next_skb;
1698 struct strp_msg *rxm = strp_msg(skb);
1699 int chunk = min_t(unsigned int, rxm->full_len - skip, len);
1700
1701 tlm = tls_msg(skb);
1702
1703 err = tls_record_content_type(msg, tlm, control);
1704 if (err <= 0)
1705 return err;
1706
1707 if (!zc || (rxm->full_len - skip) > len) {
1708 err = skb_copy_datagram_msg(skb, rxm->offset + skip,
1709 msg, chunk);
1710 if (err < 0)
1711 return err;
1712 }
1713
1714 len = len - chunk;
1715 copied = copied + chunk;
1716
1717 /* Consume the data from record if it is non-peek case*/
1718 if (!is_peek) {
1719 rxm->offset = rxm->offset + chunk;
1720 rxm->full_len = rxm->full_len - chunk;
1721
1722 /* Return if there is unconsumed data in the record */
1723 if (rxm->full_len - skip)
1724 break;
1725 }
1726
1727 /* The remaining skip-bytes must lie in 1st record in rx_list.
1728 * So from the 2nd record, 'skip' should be 0.
1729 */
1730 skip = 0;
1731
1732 if (msg)
1733 msg->msg_flags |= MSG_EOR;
1734
1735 next_skb = skb_peek_next(skb, &ctx->rx_list);
1736
1737 if (!is_peek) {
1738 skb_unlink(skb, &ctx->rx_list);
1739 consume_skb(skb);
1740 }
1741
1742 skb = next_skb;
1743 }
1744
1745 return copied;
1746 }
1747
tls_sw_recvmsg(struct sock * sk,struct msghdr * msg,size_t len,int nonblock,int flags,int * addr_len)1748 int tls_sw_recvmsg(struct sock *sk,
1749 struct msghdr *msg,
1750 size_t len,
1751 int nonblock,
1752 int flags,
1753 int *addr_len)
1754 {
1755 struct tls_context *tls_ctx = tls_get_ctx(sk);
1756 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
1757 struct tls_prot_info *prot = &tls_ctx->prot_info;
1758 struct sk_psock *psock;
1759 unsigned char control = 0;
1760 ssize_t decrypted = 0;
1761 struct strp_msg *rxm;
1762 struct tls_msg *tlm;
1763 struct sk_buff *skb;
1764 ssize_t copied = 0;
1765 bool async = false;
1766 int target, err = 0;
1767 long timeo;
1768 bool is_kvec = iov_iter_is_kvec(&msg->msg_iter);
1769 bool is_peek = flags & MSG_PEEK;
1770 bool bpf_strp_enabled;
1771
1772 flags |= nonblock;
1773
1774 if (unlikely(flags & MSG_ERRQUEUE))
1775 return sock_recv_errqueue(sk, msg, len, SOL_IP, IP_RECVERR);
1776
1777 psock = sk_psock_get(sk);
1778 lock_sock(sk);
1779 bpf_strp_enabled = sk_psock_strp_enabled(psock);
1780
1781 /* Process pending decrypted records. It must be non-zero-copy */
1782 err = process_rx_list(ctx, msg, &control, 0, len, false, is_peek);
1783 if (err < 0) {
1784 tls_err_abort(sk, err);
1785 goto end;
1786 }
1787
1788 copied = err;
1789 if (len <= copied || (copied && control != TLS_RECORD_TYPE_DATA))
1790 goto end;
1791
1792 target = sock_rcvlowat(sk, flags & MSG_WAITALL, len);
1793 len = len - copied;
1794 timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT);
1795
1796 decrypted = 0;
1797 while (len && (decrypted + copied < target || ctx->recv_pkt)) {
1798 struct tls_decrypt_arg darg = {};
1799 bool retain_skb = false;
1800 int to_decrypt, chunk;
1801
1802 skb = tls_wait_data(sk, psock, flags & MSG_DONTWAIT, timeo, &err);
1803 if (!skb) {
1804 if (psock) {
1805 int ret = sk_msg_recvmsg(sk, psock, msg, len,
1806 flags);
1807
1808 if (ret > 0) {
1809 decrypted += ret;
1810 len -= ret;
1811 continue;
1812 }
1813 }
1814 goto recv_end;
1815 }
1816
1817 rxm = strp_msg(skb);
1818 tlm = tls_msg(skb);
1819
1820 to_decrypt = rxm->full_len - prot->overhead_size;
1821
1822 if (to_decrypt <= len && !is_kvec && !is_peek &&
1823 tlm->control == TLS_RECORD_TYPE_DATA &&
1824 prot->version != TLS_1_3_VERSION &&
1825 !bpf_strp_enabled)
1826 darg.zc = true;
1827
1828 /* Do not use async mode if record is non-data */
1829 if (tlm->control == TLS_RECORD_TYPE_DATA && !bpf_strp_enabled)
1830 darg.async = ctx->async_capable;
1831 else
1832 darg.async = false;
1833
1834 err = decrypt_skb_update(sk, skb, &msg->msg_iter, &darg);
1835 if (err < 0) {
1836 tls_err_abort(sk, -EBADMSG);
1837 goto recv_end;
1838 }
1839
1840 async |= darg.async;
1841
1842 /* If the type of records being processed is not known yet,
1843 * set it to record type just dequeued. If it is already known,
1844 * but does not match the record type just dequeued, go to end.
1845 * We always get record type here since for tls1.2, record type
1846 * is known just after record is dequeued from stream parser.
1847 * For tls1.3, we disable async.
1848 */
1849 err = tls_record_content_type(msg, tlm, &control);
1850 if (err <= 0)
1851 goto recv_end;
1852
1853 if (async) {
1854 /* TLS 1.2-only, to_decrypt must be text length */
1855 chunk = min_t(int, to_decrypt, len);
1856 goto pick_next_record;
1857 }
1858 /* TLS 1.3 may have updated the length by more than overhead */
1859 chunk = rxm->full_len;
1860
1861 if (!darg.zc) {
1862 if (bpf_strp_enabled) {
1863 err = sk_psock_tls_strp_read(psock, skb);
1864 if (err != __SK_PASS) {
1865 rxm->offset = rxm->offset + rxm->full_len;
1866 rxm->full_len = 0;
1867 if (err == __SK_DROP)
1868 consume_skb(skb);
1869 ctx->recv_pkt = NULL;
1870 __strp_unpause(&ctx->strp);
1871 continue;
1872 }
1873 }
1874
1875 if (chunk > len) {
1876 retain_skb = true;
1877 chunk = len;
1878 }
1879
1880 err = skb_copy_datagram_msg(skb, rxm->offset,
1881 msg, chunk);
1882 if (err < 0)
1883 goto recv_end;
1884
1885 if (!is_peek) {
1886 rxm->offset = rxm->offset + chunk;
1887 rxm->full_len = rxm->full_len - chunk;
1888 }
1889 }
1890
1891 pick_next_record:
1892 decrypted += chunk;
1893 len -= chunk;
1894
1895 /* For async or peek case, queue the current skb */
1896 if (async || is_peek || retain_skb) {
1897 skb_queue_tail(&ctx->rx_list, skb);
1898 skb = NULL;
1899 }
1900
1901 if (tls_sw_advance_skb(sk, skb, chunk)) {
1902 /* Return full control message to
1903 * userspace before trying to parse
1904 * another message type
1905 */
1906 msg->msg_flags |= MSG_EOR;
1907 if (control != TLS_RECORD_TYPE_DATA)
1908 goto recv_end;
1909 } else {
1910 break;
1911 }
1912 }
1913
1914 recv_end:
1915 if (async) {
1916 int pending;
1917
1918 /* Wait for all previously submitted records to be decrypted */
1919 spin_lock_bh(&ctx->decrypt_compl_lock);
1920 ctx->async_notify = true;
1921 pending = atomic_read(&ctx->decrypt_pending);
1922 spin_unlock_bh(&ctx->decrypt_compl_lock);
1923 if (pending) {
1924 err = crypto_wait_req(-EINPROGRESS, &ctx->async_wait);
1925 if (err) {
1926 /* one of async decrypt failed */
1927 tls_err_abort(sk, err);
1928 copied = 0;
1929 decrypted = 0;
1930 goto end;
1931 }
1932 } else {
1933 reinit_completion(&ctx->async_wait.completion);
1934 }
1935
1936 /* There can be no concurrent accesses, since we have no
1937 * pending decrypt operations
1938 */
1939 WRITE_ONCE(ctx->async_notify, false);
1940
1941 /* Drain records from the rx_list & copy if required */
1942 if (is_peek || is_kvec)
1943 err = process_rx_list(ctx, msg, &control, copied,
1944 decrypted, false, is_peek);
1945 else
1946 err = process_rx_list(ctx, msg, &control, 0,
1947 decrypted, true, is_peek);
1948 if (err < 0) {
1949 tls_err_abort(sk, err);
1950 copied = 0;
1951 goto end;
1952 }
1953 }
1954
1955 copied += decrypted;
1956
1957 end:
1958 release_sock(sk);
1959 if (psock)
1960 sk_psock_put(sk, psock);
1961 return copied ? : err;
1962 }
1963
tls_sw_splice_read(struct socket * sock,loff_t * ppos,struct pipe_inode_info * pipe,size_t len,unsigned int flags)1964 ssize_t tls_sw_splice_read(struct socket *sock, loff_t *ppos,
1965 struct pipe_inode_info *pipe,
1966 size_t len, unsigned int flags)
1967 {
1968 struct tls_context *tls_ctx = tls_get_ctx(sock->sk);
1969 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
1970 struct strp_msg *rxm = NULL;
1971 struct sock *sk = sock->sk;
1972 struct tls_msg *tlm;
1973 struct sk_buff *skb;
1974 ssize_t copied = 0;
1975 bool from_queue;
1976 int err = 0;
1977 long timeo;
1978 int chunk;
1979
1980 lock_sock(sk);
1981
1982 timeo = sock_rcvtimeo(sk, flags & SPLICE_F_NONBLOCK);
1983
1984 from_queue = !skb_queue_empty(&ctx->rx_list);
1985 if (from_queue) {
1986 skb = __skb_dequeue(&ctx->rx_list);
1987 } else {
1988 struct tls_decrypt_arg darg = {};
1989
1990 skb = tls_wait_data(sk, NULL, flags & SPLICE_F_NONBLOCK, timeo,
1991 &err);
1992 if (!skb)
1993 goto splice_read_end;
1994
1995 err = decrypt_skb_update(sk, skb, NULL, &darg);
1996 if (err < 0) {
1997 tls_err_abort(sk, -EBADMSG);
1998 goto splice_read_end;
1999 }
2000 }
2001
2002 rxm = strp_msg(skb);
2003 tlm = tls_msg(skb);
2004
2005 /* splice does not support reading control messages */
2006 if (tlm->control != TLS_RECORD_TYPE_DATA) {
2007 err = -EINVAL;
2008 goto splice_read_end;
2009 }
2010
2011 chunk = min_t(unsigned int, rxm->full_len, len);
2012 copied = skb_splice_bits(skb, sk, rxm->offset, pipe, chunk, flags);
2013 if (copied < 0)
2014 goto splice_read_end;
2015
2016 if (!from_queue) {
2017 ctx->recv_pkt = NULL;
2018 __strp_unpause(&ctx->strp);
2019 }
2020 if (chunk < rxm->full_len) {
2021 __skb_queue_head(&ctx->rx_list, skb);
2022 rxm->offset += len;
2023 rxm->full_len -= len;
2024 } else {
2025 consume_skb(skb);
2026 }
2027
2028 splice_read_end:
2029 release_sock(sk);
2030 return copied ? : err;
2031 }
2032
tls_sw_sock_is_readable(struct sock * sk)2033 bool tls_sw_sock_is_readable(struct sock *sk)
2034 {
2035 struct tls_context *tls_ctx = tls_get_ctx(sk);
2036 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
2037 bool ingress_empty = true;
2038 struct sk_psock *psock;
2039
2040 rcu_read_lock();
2041 psock = sk_psock(sk);
2042 if (psock)
2043 ingress_empty = list_empty(&psock->ingress_msg);
2044 rcu_read_unlock();
2045
2046 return !ingress_empty || ctx->recv_pkt ||
2047 !skb_queue_empty(&ctx->rx_list);
2048 }
2049
tls_read_size(struct strparser * strp,struct sk_buff * skb)2050 static int tls_read_size(struct strparser *strp, struct sk_buff *skb)
2051 {
2052 struct tls_context *tls_ctx = tls_get_ctx(strp->sk);
2053 struct tls_prot_info *prot = &tls_ctx->prot_info;
2054 char header[TLS_HEADER_SIZE + MAX_IV_SIZE];
2055 struct strp_msg *rxm = strp_msg(skb);
2056 struct tls_msg *tlm = tls_msg(skb);
2057 size_t cipher_overhead;
2058 size_t data_len = 0;
2059 int ret;
2060
2061 /* Verify that we have a full TLS header, or wait for more data */
2062 if (rxm->offset + prot->prepend_size > skb->len)
2063 return 0;
2064
2065 /* Sanity-check size of on-stack buffer. */
2066 if (WARN_ON(prot->prepend_size > sizeof(header))) {
2067 ret = -EINVAL;
2068 goto read_failure;
2069 }
2070
2071 /* Linearize header to local buffer */
2072 ret = skb_copy_bits(skb, rxm->offset, header, prot->prepend_size);
2073
2074 if (ret < 0)
2075 goto read_failure;
2076
2077 tlm->control = header[0];
2078
2079 data_len = ((header[4] & 0xFF) | (header[3] << 8));
2080
2081 cipher_overhead = prot->tag_size;
2082 if (prot->version != TLS_1_3_VERSION &&
2083 prot->cipher_type != TLS_CIPHER_CHACHA20_POLY1305)
2084 cipher_overhead += prot->iv_size;
2085
2086 if (data_len > TLS_MAX_PAYLOAD_SIZE + cipher_overhead +
2087 prot->tail_size) {
2088 ret = -EMSGSIZE;
2089 goto read_failure;
2090 }
2091 if (data_len < cipher_overhead) {
2092 ret = -EBADMSG;
2093 goto read_failure;
2094 }
2095
2096 /* Note that both TLS1.3 and TLS1.2 use TLS_1_2 version here */
2097 if (header[1] != TLS_1_2_VERSION_MINOR ||
2098 header[2] != TLS_1_2_VERSION_MAJOR) {
2099 ret = -EINVAL;
2100 goto read_failure;
2101 }
2102
2103 tls_device_rx_resync_new_rec(strp->sk, data_len + TLS_HEADER_SIZE,
2104 TCP_SKB_CB(skb)->seq + rxm->offset);
2105 return data_len + TLS_HEADER_SIZE;
2106
2107 read_failure:
2108 tls_err_abort(strp->sk, ret);
2109
2110 return ret;
2111 }
2112
tls_queue(struct strparser * strp,struct sk_buff * skb)2113 static void tls_queue(struct strparser *strp, struct sk_buff *skb)
2114 {
2115 struct tls_context *tls_ctx = tls_get_ctx(strp->sk);
2116 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
2117 struct tls_msg *tlm = tls_msg(skb);
2118
2119 tlm->decrypted = 0;
2120
2121 ctx->recv_pkt = skb;
2122 strp_pause(strp);
2123
2124 ctx->saved_data_ready(strp->sk);
2125 }
2126
tls_data_ready(struct sock * sk)2127 static void tls_data_ready(struct sock *sk)
2128 {
2129 struct tls_context *tls_ctx = tls_get_ctx(sk);
2130 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
2131 struct sk_psock *psock;
2132
2133 strp_data_ready(&ctx->strp);
2134
2135 psock = sk_psock_get(sk);
2136 if (psock) {
2137 if (!list_empty(&psock->ingress_msg))
2138 ctx->saved_data_ready(sk);
2139 sk_psock_put(sk, psock);
2140 }
2141 }
2142
tls_sw_cancel_work_tx(struct tls_context * tls_ctx)2143 void tls_sw_cancel_work_tx(struct tls_context *tls_ctx)
2144 {
2145 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
2146
2147 set_bit(BIT_TX_CLOSING, &ctx->tx_bitmask);
2148 set_bit(BIT_TX_SCHEDULED, &ctx->tx_bitmask);
2149 cancel_delayed_work_sync(&ctx->tx_work.work);
2150 }
2151
tls_sw_release_resources_tx(struct sock * sk)2152 void tls_sw_release_resources_tx(struct sock *sk)
2153 {
2154 struct tls_context *tls_ctx = tls_get_ctx(sk);
2155 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
2156 struct tls_rec *rec, *tmp;
2157 int pending;
2158
2159 /* Wait for any pending async encryptions to complete */
2160 spin_lock_bh(&ctx->encrypt_compl_lock);
2161 ctx->async_notify = true;
2162 pending = atomic_read(&ctx->encrypt_pending);
2163 spin_unlock_bh(&ctx->encrypt_compl_lock);
2164
2165 if (pending)
2166 crypto_wait_req(-EINPROGRESS, &ctx->async_wait);
2167
2168 tls_tx_records(sk, -1);
2169
2170 /* Free up un-sent records in tx_list. First, free
2171 * the partially sent record if any at head of tx_list.
2172 */
2173 if (tls_ctx->partially_sent_record) {
2174 tls_free_partial_record(sk, tls_ctx);
2175 rec = list_first_entry(&ctx->tx_list,
2176 struct tls_rec, list);
2177 list_del(&rec->list);
2178 sk_msg_free(sk, &rec->msg_plaintext);
2179 kfree(rec);
2180 }
2181
2182 list_for_each_entry_safe(rec, tmp, &ctx->tx_list, list) {
2183 list_del(&rec->list);
2184 sk_msg_free(sk, &rec->msg_encrypted);
2185 sk_msg_free(sk, &rec->msg_plaintext);
2186 kfree(rec);
2187 }
2188
2189 crypto_free_aead(ctx->aead_send);
2190 tls_free_open_rec(sk);
2191 }
2192
tls_sw_free_ctx_tx(struct tls_context * tls_ctx)2193 void tls_sw_free_ctx_tx(struct tls_context *tls_ctx)
2194 {
2195 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
2196
2197 kfree(ctx);
2198 }
2199
tls_sw_release_resources_rx(struct sock * sk)2200 void tls_sw_release_resources_rx(struct sock *sk)
2201 {
2202 struct tls_context *tls_ctx = tls_get_ctx(sk);
2203 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
2204
2205 kfree(tls_ctx->rx.rec_seq);
2206 kfree(tls_ctx->rx.iv);
2207
2208 if (ctx->aead_recv) {
2209 kfree_skb(ctx->recv_pkt);
2210 ctx->recv_pkt = NULL;
2211 skb_queue_purge(&ctx->rx_list);
2212 crypto_free_aead(ctx->aead_recv);
2213 strp_stop(&ctx->strp);
2214 /* If tls_sw_strparser_arm() was not called (cleanup paths)
2215 * we still want to strp_stop(), but sk->sk_data_ready was
2216 * never swapped.
2217 */
2218 if (ctx->saved_data_ready) {
2219 write_lock_bh(&sk->sk_callback_lock);
2220 sk->sk_data_ready = ctx->saved_data_ready;
2221 write_unlock_bh(&sk->sk_callback_lock);
2222 }
2223 }
2224 }
2225
tls_sw_strparser_done(struct tls_context * tls_ctx)2226 void tls_sw_strparser_done(struct tls_context *tls_ctx)
2227 {
2228 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
2229
2230 strp_done(&ctx->strp);
2231 }
2232
tls_sw_free_ctx_rx(struct tls_context * tls_ctx)2233 void tls_sw_free_ctx_rx(struct tls_context *tls_ctx)
2234 {
2235 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
2236
2237 kfree(ctx);
2238 }
2239
tls_sw_free_resources_rx(struct sock * sk)2240 void tls_sw_free_resources_rx(struct sock *sk)
2241 {
2242 struct tls_context *tls_ctx = tls_get_ctx(sk);
2243
2244 tls_sw_release_resources_rx(sk);
2245 tls_sw_free_ctx_rx(tls_ctx);
2246 }
2247
2248 /* The work handler to transmitt the encrypted records in tx_list */
tx_work_handler(struct work_struct * work)2249 static void tx_work_handler(struct work_struct *work)
2250 {
2251 struct delayed_work *delayed_work = to_delayed_work(work);
2252 struct tx_work *tx_work = container_of(delayed_work,
2253 struct tx_work, work);
2254 struct sock *sk = tx_work->sk;
2255 struct tls_context *tls_ctx = tls_get_ctx(sk);
2256 struct tls_sw_context_tx *ctx;
2257
2258 if (unlikely(!tls_ctx))
2259 return;
2260
2261 ctx = tls_sw_ctx_tx(tls_ctx);
2262 if (test_bit(BIT_TX_CLOSING, &ctx->tx_bitmask))
2263 return;
2264
2265 if (!test_and_clear_bit(BIT_TX_SCHEDULED, &ctx->tx_bitmask))
2266 return;
2267
2268 if (mutex_trylock(&tls_ctx->tx_lock)) {
2269 lock_sock(sk);
2270 tls_tx_records(sk, -1);
2271 release_sock(sk);
2272 mutex_unlock(&tls_ctx->tx_lock);
2273 } else if (!test_and_set_bit(BIT_TX_SCHEDULED, &ctx->tx_bitmask)) {
2274 /* Someone is holding the tx_lock, they will likely run Tx
2275 * and cancel the work on their way out of the lock section.
2276 * Schedule a long delay just in case.
2277 */
2278 schedule_delayed_work(&ctx->tx_work.work, msecs_to_jiffies(10));
2279 }
2280 }
2281
tls_sw_write_space(struct sock * sk,struct tls_context * ctx)2282 void tls_sw_write_space(struct sock *sk, struct tls_context *ctx)
2283 {
2284 struct tls_sw_context_tx *tx_ctx = tls_sw_ctx_tx(ctx);
2285
2286 /* Schedule the transmission if tx list is ready */
2287 if (is_tx_ready(tx_ctx) &&
2288 !test_and_set_bit(BIT_TX_SCHEDULED, &tx_ctx->tx_bitmask))
2289 schedule_delayed_work(&tx_ctx->tx_work.work, 0);
2290 }
2291
tls_sw_strparser_arm(struct sock * sk,struct tls_context * tls_ctx)2292 void tls_sw_strparser_arm(struct sock *sk, struct tls_context *tls_ctx)
2293 {
2294 struct tls_sw_context_rx *rx_ctx = tls_sw_ctx_rx(tls_ctx);
2295
2296 write_lock_bh(&sk->sk_callback_lock);
2297 rx_ctx->saved_data_ready = sk->sk_data_ready;
2298 sk->sk_data_ready = tls_data_ready;
2299 write_unlock_bh(&sk->sk_callback_lock);
2300
2301 strp_check_rcv(&rx_ctx->strp);
2302 }
2303
tls_set_sw_offload(struct sock * sk,struct tls_context * ctx,int tx)2304 int tls_set_sw_offload(struct sock *sk, struct tls_context *ctx, int tx)
2305 {
2306 struct tls_context *tls_ctx = tls_get_ctx(sk);
2307 struct tls_prot_info *prot = &tls_ctx->prot_info;
2308 struct tls_crypto_info *crypto_info;
2309 struct tls12_crypto_info_aes_gcm_128 *gcm_128_info;
2310 struct tls12_crypto_info_aes_gcm_256 *gcm_256_info;
2311 struct tls12_crypto_info_aes_ccm_128 *ccm_128_info;
2312 struct tls12_crypto_info_chacha20_poly1305 *chacha20_poly1305_info;
2313 struct tls_sw_context_tx *sw_ctx_tx = NULL;
2314 struct tls_sw_context_rx *sw_ctx_rx = NULL;
2315 struct cipher_context *cctx;
2316 struct crypto_aead **aead;
2317 struct strp_callbacks cb;
2318 u16 nonce_size, tag_size, iv_size, rec_seq_size, salt_size;
2319 struct crypto_tfm *tfm;
2320 char *iv, *rec_seq, *key, *salt, *cipher_name;
2321 size_t keysize;
2322 int rc = 0;
2323
2324 if (!ctx) {
2325 rc = -EINVAL;
2326 goto out;
2327 }
2328
2329 if (tx) {
2330 if (!ctx->priv_ctx_tx) {
2331 sw_ctx_tx = kzalloc(sizeof(*sw_ctx_tx), GFP_KERNEL);
2332 if (!sw_ctx_tx) {
2333 rc = -ENOMEM;
2334 goto out;
2335 }
2336 ctx->priv_ctx_tx = sw_ctx_tx;
2337 } else {
2338 sw_ctx_tx =
2339 (struct tls_sw_context_tx *)ctx->priv_ctx_tx;
2340 }
2341 } else {
2342 if (!ctx->priv_ctx_rx) {
2343 sw_ctx_rx = kzalloc(sizeof(*sw_ctx_rx), GFP_KERNEL);
2344 if (!sw_ctx_rx) {
2345 rc = -ENOMEM;
2346 goto out;
2347 }
2348 ctx->priv_ctx_rx = sw_ctx_rx;
2349 } else {
2350 sw_ctx_rx =
2351 (struct tls_sw_context_rx *)ctx->priv_ctx_rx;
2352 }
2353 }
2354
2355 if (tx) {
2356 crypto_init_wait(&sw_ctx_tx->async_wait);
2357 spin_lock_init(&sw_ctx_tx->encrypt_compl_lock);
2358 crypto_info = &ctx->crypto_send.info;
2359 cctx = &ctx->tx;
2360 aead = &sw_ctx_tx->aead_send;
2361 INIT_LIST_HEAD(&sw_ctx_tx->tx_list);
2362 INIT_DELAYED_WORK(&sw_ctx_tx->tx_work.work, tx_work_handler);
2363 sw_ctx_tx->tx_work.sk = sk;
2364 } else {
2365 crypto_init_wait(&sw_ctx_rx->async_wait);
2366 spin_lock_init(&sw_ctx_rx->decrypt_compl_lock);
2367 crypto_info = &ctx->crypto_recv.info;
2368 cctx = &ctx->rx;
2369 skb_queue_head_init(&sw_ctx_rx->rx_list);
2370 aead = &sw_ctx_rx->aead_recv;
2371 }
2372
2373 switch (crypto_info->cipher_type) {
2374 case TLS_CIPHER_AES_GCM_128: {
2375 nonce_size = TLS_CIPHER_AES_GCM_128_IV_SIZE;
2376 tag_size = TLS_CIPHER_AES_GCM_128_TAG_SIZE;
2377 iv_size = TLS_CIPHER_AES_GCM_128_IV_SIZE;
2378 iv = ((struct tls12_crypto_info_aes_gcm_128 *)crypto_info)->iv;
2379 rec_seq_size = TLS_CIPHER_AES_GCM_128_REC_SEQ_SIZE;
2380 rec_seq =
2381 ((struct tls12_crypto_info_aes_gcm_128 *)crypto_info)->rec_seq;
2382 gcm_128_info =
2383 (struct tls12_crypto_info_aes_gcm_128 *)crypto_info;
2384 keysize = TLS_CIPHER_AES_GCM_128_KEY_SIZE;
2385 key = gcm_128_info->key;
2386 salt = gcm_128_info->salt;
2387 salt_size = TLS_CIPHER_AES_GCM_128_SALT_SIZE;
2388 cipher_name = "gcm(aes)";
2389 break;
2390 }
2391 case TLS_CIPHER_AES_GCM_256: {
2392 nonce_size = TLS_CIPHER_AES_GCM_256_IV_SIZE;
2393 tag_size = TLS_CIPHER_AES_GCM_256_TAG_SIZE;
2394 iv_size = TLS_CIPHER_AES_GCM_256_IV_SIZE;
2395 iv = ((struct tls12_crypto_info_aes_gcm_256 *)crypto_info)->iv;
2396 rec_seq_size = TLS_CIPHER_AES_GCM_256_REC_SEQ_SIZE;
2397 rec_seq =
2398 ((struct tls12_crypto_info_aes_gcm_256 *)crypto_info)->rec_seq;
2399 gcm_256_info =
2400 (struct tls12_crypto_info_aes_gcm_256 *)crypto_info;
2401 keysize = TLS_CIPHER_AES_GCM_256_KEY_SIZE;
2402 key = gcm_256_info->key;
2403 salt = gcm_256_info->salt;
2404 salt_size = TLS_CIPHER_AES_GCM_256_SALT_SIZE;
2405 cipher_name = "gcm(aes)";
2406 break;
2407 }
2408 case TLS_CIPHER_AES_CCM_128: {
2409 nonce_size = TLS_CIPHER_AES_CCM_128_IV_SIZE;
2410 tag_size = TLS_CIPHER_AES_CCM_128_TAG_SIZE;
2411 iv_size = TLS_CIPHER_AES_CCM_128_IV_SIZE;
2412 iv = ((struct tls12_crypto_info_aes_ccm_128 *)crypto_info)->iv;
2413 rec_seq_size = TLS_CIPHER_AES_CCM_128_REC_SEQ_SIZE;
2414 rec_seq =
2415 ((struct tls12_crypto_info_aes_ccm_128 *)crypto_info)->rec_seq;
2416 ccm_128_info =
2417 (struct tls12_crypto_info_aes_ccm_128 *)crypto_info;
2418 keysize = TLS_CIPHER_AES_CCM_128_KEY_SIZE;
2419 key = ccm_128_info->key;
2420 salt = ccm_128_info->salt;
2421 salt_size = TLS_CIPHER_AES_CCM_128_SALT_SIZE;
2422 cipher_name = "ccm(aes)";
2423 break;
2424 }
2425 case TLS_CIPHER_CHACHA20_POLY1305: {
2426 chacha20_poly1305_info = (void *)crypto_info;
2427 nonce_size = 0;
2428 tag_size = TLS_CIPHER_CHACHA20_POLY1305_TAG_SIZE;
2429 iv_size = TLS_CIPHER_CHACHA20_POLY1305_IV_SIZE;
2430 iv = chacha20_poly1305_info->iv;
2431 rec_seq_size = TLS_CIPHER_CHACHA20_POLY1305_REC_SEQ_SIZE;
2432 rec_seq = chacha20_poly1305_info->rec_seq;
2433 keysize = TLS_CIPHER_CHACHA20_POLY1305_KEY_SIZE;
2434 key = chacha20_poly1305_info->key;
2435 salt = chacha20_poly1305_info->salt;
2436 salt_size = TLS_CIPHER_CHACHA20_POLY1305_SALT_SIZE;
2437 cipher_name = "rfc7539(chacha20,poly1305)";
2438 break;
2439 }
2440 default:
2441 rc = -EINVAL;
2442 goto free_priv;
2443 }
2444
2445 /* Sanity-check the sizes for stack allocations. */
2446 if (iv_size > MAX_IV_SIZE || nonce_size > MAX_IV_SIZE ||
2447 rec_seq_size > TLS_MAX_REC_SEQ_SIZE) {
2448 rc = -EINVAL;
2449 goto free_priv;
2450 }
2451
2452 if (crypto_info->version == TLS_1_3_VERSION) {
2453 nonce_size = 0;
2454 prot->aad_size = TLS_HEADER_SIZE;
2455 prot->tail_size = 1;
2456 } else {
2457 prot->aad_size = TLS_AAD_SPACE_SIZE;
2458 prot->tail_size = 0;
2459 }
2460
2461 prot->version = crypto_info->version;
2462 prot->cipher_type = crypto_info->cipher_type;
2463 prot->prepend_size = TLS_HEADER_SIZE + nonce_size;
2464 prot->tag_size = tag_size;
2465 prot->overhead_size = prot->prepend_size +
2466 prot->tag_size + prot->tail_size;
2467 prot->iv_size = iv_size;
2468 prot->salt_size = salt_size;
2469 cctx->iv = kmalloc(iv_size + salt_size, GFP_KERNEL);
2470 if (!cctx->iv) {
2471 rc = -ENOMEM;
2472 goto free_priv;
2473 }
2474 /* Note: 128 & 256 bit salt are the same size */
2475 prot->rec_seq_size = rec_seq_size;
2476 memcpy(cctx->iv, salt, salt_size);
2477 memcpy(cctx->iv + salt_size, iv, iv_size);
2478 cctx->rec_seq = kmemdup(rec_seq, rec_seq_size, GFP_KERNEL);
2479 if (!cctx->rec_seq) {
2480 rc = -ENOMEM;
2481 goto free_iv;
2482 }
2483
2484 if (!*aead) {
2485 *aead = crypto_alloc_aead(cipher_name, 0, 0);
2486 if (IS_ERR(*aead)) {
2487 rc = PTR_ERR(*aead);
2488 *aead = NULL;
2489 goto free_rec_seq;
2490 }
2491 }
2492
2493 ctx->push_pending_record = tls_sw_push_pending_record;
2494
2495 rc = crypto_aead_setkey(*aead, key, keysize);
2496
2497 if (rc)
2498 goto free_aead;
2499
2500 rc = crypto_aead_setauthsize(*aead, prot->tag_size);
2501 if (rc)
2502 goto free_aead;
2503
2504 if (sw_ctx_rx) {
2505 tfm = crypto_aead_tfm(sw_ctx_rx->aead_recv);
2506
2507 if (crypto_info->version == TLS_1_3_VERSION)
2508 sw_ctx_rx->async_capable = 0;
2509 else
2510 sw_ctx_rx->async_capable =
2511 !!(tfm->__crt_alg->cra_flags &
2512 CRYPTO_ALG_ASYNC);
2513
2514 /* Set up strparser */
2515 memset(&cb, 0, sizeof(cb));
2516 cb.rcv_msg = tls_queue;
2517 cb.parse_msg = tls_read_size;
2518
2519 strp_init(&sw_ctx_rx->strp, sk, &cb);
2520 }
2521
2522 goto out;
2523
2524 free_aead:
2525 crypto_free_aead(*aead);
2526 *aead = NULL;
2527 free_rec_seq:
2528 kfree(cctx->rec_seq);
2529 cctx->rec_seq = NULL;
2530 free_iv:
2531 kfree(cctx->iv);
2532 cctx->iv = NULL;
2533 free_priv:
2534 if (tx) {
2535 kfree(ctx->priv_ctx_tx);
2536 ctx->priv_ctx_tx = NULL;
2537 } else {
2538 kfree(ctx->priv_ctx_rx);
2539 ctx->priv_ctx_rx = NULL;
2540 }
2541 out:
2542 return rc;
2543 }
2544