61 	int (*complete)(struct aead_request *req, u32 flags);
83 static int crypto_rfc4543_copy_src_to_dst(struct aead_request *req, bool enc);
86 	struct aead_request *req)  in crypto_gcm_reqctx()  argument
88 	unsigned long align = crypto_aead_alignmask(crypto_aead_reqtfm(req));  in crypto_gcm_reqctx()
90 	return (void *)PTR_ALIGN((u8 *)aead_request_ctx(req), align + 1);  in crypto_gcm_reqctx()
106 		struct skcipher_request req;  in crypto_gcm_setkey()  member
126 	skcipher_request_set_tfm(&data->req, ctr);  in crypto_gcm_setkey()
127 	skcipher_request_set_callback(&data->req, CRYPTO_TFM_REQ_MAY_SLEEP |  in crypto_gcm_setkey()
131 	skcipher_request_set_crypt(&data->req, data->sg, data->sg,  in crypto_gcm_setkey()
134 	err = crypto_wait_req(crypto_skcipher_encrypt(&data->req),  in crypto_gcm_setkey()
158 static void crypto_gcm_init_common(struct aead_request *req)  in crypto_gcm_init_common()  argument
160 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in crypto_gcm_init_common()
165 	memcpy(pctx->iv, req->iv, GCM_AES_IV_SIZE);  in crypto_gcm_init_common()
170 	sg = scatterwalk_ffwd(pctx->src + 1, req->src, req->assoclen);  in crypto_gcm_init_common()
174 	if (req->src != req->dst) {  in crypto_gcm_init_common()
177 		sg = scatterwalk_ffwd(pctx->dst + 1, req->dst, req->assoclen);  in crypto_gcm_init_common()
183 static void crypto_gcm_init_crypt(struct aead_request *req,  in crypto_gcm_init_crypt()  argument
186 	struct crypto_aead *aead = crypto_aead_reqtfm(req);  in crypto_gcm_init_crypt()
188 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in crypto_gcm_init_crypt()
192 	dst = req->src == req->dst ? pctx->src : pctx->dst;  in crypto_gcm_init_crypt()
208 static int gcm_hash_update(struct aead_request *req,  in gcm_hash_update()  argument
213 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in gcm_hash_update()
216 	ahash_request_set_callback(ahreq, flags, compl, req);  in gcm_hash_update()
222 static int gcm_hash_remain(struct aead_request *req,  in gcm_hash_remain()  argument
226 	return gcm_hash_update(req, compl, &gcm_zeroes->sg, remain, flags);  in gcm_hash_remain()
229 static int gcm_hash_len(struct aead_request *req, u32 flags)  in gcm_hash_len()  argument
231 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in gcm_hash_len()
236 	lengths.a = cpu_to_be64(req->assoclen * 8);  in gcm_hash_len()
240 	ahash_request_set_callback(ahreq, flags, gcm_hash_len_done, req);  in gcm_hash_len()
247 static int gcm_hash_len_continue(struct aead_request *req, u32 flags)  in gcm_hash_len_continue()  argument
249 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in gcm_hash_len_continue()
252 	return gctx->complete(req, flags);  in gcm_hash_len_continue()
257 	struct aead_request *req = areq->data;  in gcm_hash_len_done()  local
262 	err = gcm_hash_len_continue(req, 0);  in gcm_hash_len_done()
267 	aead_request_complete(req, err);  in gcm_hash_len_done()
270 static int gcm_hash_crypt_remain_continue(struct aead_request *req, u32 flags)  in gcm_hash_crypt_remain_continue()  argument
272 	return gcm_hash_len(req, flags) ?:  in gcm_hash_crypt_remain_continue()
273 	       gcm_hash_len_continue(req, flags);  in gcm_hash_crypt_remain_continue()
279 	struct aead_request *req = areq->data;  in gcm_hash_crypt_remain_done()  local
284 	err = gcm_hash_crypt_remain_continue(req, 0);  in gcm_hash_crypt_remain_done()
289 	aead_request_complete(req, err);  in gcm_hash_crypt_remain_done()
292 static int gcm_hash_crypt_continue(struct aead_request *req, u32 flags)  in gcm_hash_crypt_continue()  argument
294 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in gcm_hash_crypt_continue()
300 		return gcm_hash_remain(req, remain,  in gcm_hash_crypt_continue()
302 		       gcm_hash_crypt_remain_continue(req, flags);  in gcm_hash_crypt_continue()
304 	return gcm_hash_crypt_remain_continue(req, flags);  in gcm_hash_crypt_continue()
309 	struct aead_request *req = areq->data;  in gcm_hash_crypt_done()  local
314 	err = gcm_hash_crypt_continue(req, 0);  in gcm_hash_crypt_done()
319 	aead_request_complete(req, err);  in gcm_hash_crypt_done()
322 static int gcm_hash_assoc_remain_continue(struct aead_request *req, u32 flags)  in gcm_hash_assoc_remain_continue()  argument
324 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in gcm_hash_assoc_remain_continue()
328 		return gcm_hash_update(req, gcm_hash_crypt_done,  in gcm_hash_assoc_remain_continue()
330 		       gcm_hash_crypt_continue(req, flags);  in gcm_hash_assoc_remain_continue()
332 	return gcm_hash_crypt_remain_continue(req, flags);  in gcm_hash_assoc_remain_continue()
338 	struct aead_request *req = areq->data;  in gcm_hash_assoc_remain_done()  local
343 	err = gcm_hash_assoc_remain_continue(req, 0);  in gcm_hash_assoc_remain_done()
348 	aead_request_complete(req, err);  in gcm_hash_assoc_remain_done()
351 static int gcm_hash_assoc_continue(struct aead_request *req, u32 flags)  in gcm_hash_assoc_continue()  argument
355 	remain = gcm_remain(req->assoclen);  in gcm_hash_assoc_continue()
357 		return gcm_hash_remain(req, remain,  in gcm_hash_assoc_continue()
359 		       gcm_hash_assoc_remain_continue(req, flags);  in gcm_hash_assoc_continue()
361 	return gcm_hash_assoc_remain_continue(req, flags);  in gcm_hash_assoc_continue()
366 	struct aead_request *req = areq->data;  in gcm_hash_assoc_done()  local
371 	err = gcm_hash_assoc_continue(req, 0);  in gcm_hash_assoc_done()
376 	aead_request_complete(req, err);  in gcm_hash_assoc_done()
379 static int gcm_hash_init_continue(struct aead_request *req, u32 flags)  in gcm_hash_init_continue()  argument
381 	if (req->assoclen)  in gcm_hash_init_continue()
382 		return gcm_hash_update(req, gcm_hash_assoc_done,  in gcm_hash_init_continue()
383 				       req->src, req->assoclen, flags) ?:  in gcm_hash_init_continue()
384 		       gcm_hash_assoc_continue(req, flags);  in gcm_hash_init_continue()
386 	return gcm_hash_assoc_remain_continue(req, flags);  in gcm_hash_init_continue()
391 	struct aead_request *req = areq->data;  in gcm_hash_init_done()  local
396 	err = gcm_hash_init_continue(req, 0);  in gcm_hash_init_done()
401 	aead_request_complete(req, err);  in gcm_hash_init_done()
404 static int gcm_hash(struct aead_request *req, u32 flags)  in gcm_hash()  argument
406 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in gcm_hash()
408 	struct crypto_gcm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));  in gcm_hash()
412 	ahash_request_set_callback(ahreq, flags, gcm_hash_init_done, req);  in gcm_hash()
414 	       gcm_hash_init_continue(req, flags);  in gcm_hash()
417 static int gcm_enc_copy_hash(struct aead_request *req, u32 flags)  in gcm_enc_copy_hash()  argument
419 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in gcm_enc_copy_hash()
420 	struct crypto_aead *aead = crypto_aead_reqtfm(req);  in gcm_enc_copy_hash()
424 	scatterwalk_map_and_copy(auth_tag, req->dst,  in gcm_enc_copy_hash()
425 				 req->assoclen + req->cryptlen,  in gcm_enc_copy_hash()
430 static int gcm_encrypt_continue(struct aead_request *req, u32 flags)  in gcm_encrypt_continue()  argument
432 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in gcm_encrypt_continue()
435 	gctx->src = sg_next(req->src == req->dst ? pctx->src : pctx->dst);  in gcm_encrypt_continue()
436 	gctx->cryptlen = req->cryptlen;  in gcm_encrypt_continue()
439 	return gcm_hash(req, flags);  in gcm_encrypt_continue()
444 	struct aead_request *req = areq->data;  in gcm_encrypt_done()  local
449 	err = gcm_encrypt_continue(req, 0);  in gcm_encrypt_done()
454 	aead_request_complete(req, err);  in gcm_encrypt_done()
457 static int crypto_gcm_encrypt(struct aead_request *req)  in crypto_gcm_encrypt()  argument
459 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in crypto_gcm_encrypt()
461 	u32 flags = aead_request_flags(req);  in crypto_gcm_encrypt()
463 	crypto_gcm_init_common(req);  in crypto_gcm_encrypt()
464 	crypto_gcm_init_crypt(req, req->cryptlen);  in crypto_gcm_encrypt()
465 	skcipher_request_set_callback(skreq, flags, gcm_encrypt_done, req);  in crypto_gcm_encrypt()
468 	       gcm_encrypt_continue(req, flags);  in crypto_gcm_encrypt()
471 static int crypto_gcm_verify(struct aead_request *req)  in crypto_gcm_verify()  argument
473 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in crypto_gcm_verify()
474 	struct crypto_aead *aead = crypto_aead_reqtfm(req);  in crypto_gcm_verify()
478 	unsigned int cryptlen = req->cryptlen - authsize;  in crypto_gcm_verify()
481 	scatterwalk_map_and_copy(iauth_tag, req->src,  in crypto_gcm_verify()
482 				 req->assoclen + cryptlen, authsize, 0);  in crypto_gcm_verify()
488 	struct aead_request *req = areq->data;  in gcm_decrypt_done()  local
491 		err = crypto_gcm_verify(req);  in gcm_decrypt_done()
493 	aead_request_complete(req, err);  in gcm_decrypt_done()
496 static int gcm_dec_hash_continue(struct aead_request *req, u32 flags)  in gcm_dec_hash_continue()  argument
498 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in gcm_dec_hash_continue()
502 	crypto_gcm_init_crypt(req, gctx->cryptlen);  in gcm_dec_hash_continue()
503 	skcipher_request_set_callback(skreq, flags, gcm_decrypt_done, req);  in gcm_dec_hash_continue()
504 	return crypto_skcipher_decrypt(skreq) ?: crypto_gcm_verify(req);  in gcm_dec_hash_continue()
507 static int crypto_gcm_decrypt(struct aead_request *req)  in crypto_gcm_decrypt()  argument
509 	struct crypto_aead *aead = crypto_aead_reqtfm(req);  in crypto_gcm_decrypt()
510 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in crypto_gcm_decrypt()
513 	unsigned int cryptlen = req->cryptlen;  in crypto_gcm_decrypt()
514 	u32 flags = aead_request_flags(req);  in crypto_gcm_decrypt()
518 	crypto_gcm_init_common(req);  in crypto_gcm_decrypt()
524 	return gcm_hash(req, flags);  in crypto_gcm_decrypt()
761 static struct aead_request *crypto_rfc4106_crypt(struct aead_request *req)  in crypto_rfc4106_crypt()  argument
763 	struct crypto_rfc4106_req_ctx *rctx = aead_request_ctx(req);  in crypto_rfc4106_crypt()
764 	struct crypto_aead *aead = crypto_aead_reqtfm(req);  in crypto_rfc4106_crypt()
772 	scatterwalk_map_and_copy(iv + GCM_AES_IV_SIZE, req->src, 0, req->assoclen - 8, 0);  in crypto_rfc4106_crypt()
775 	memcpy(iv + 4, req->iv, 8);  in crypto_rfc4106_crypt()
778 	sg_set_buf(rctx->src, iv + GCM_AES_IV_SIZE, req->assoclen - 8);  in crypto_rfc4106_crypt()
779 	sg = scatterwalk_ffwd(rctx->src + 1, req->src, req->assoclen);  in crypto_rfc4106_crypt()
783 	if (req->src != req->dst) {  in crypto_rfc4106_crypt()
785 		sg_set_buf(rctx->dst, iv + GCM_AES_IV_SIZE, req->assoclen - 8);  in crypto_rfc4106_crypt()
786 		sg = scatterwalk_ffwd(rctx->dst + 1, req->dst, req->assoclen);  in crypto_rfc4106_crypt()
792 	aead_request_set_callback(subreq, req->base.flags, req->base.complete,  in crypto_rfc4106_crypt()
793 				  req->base.data);  in crypto_rfc4106_crypt()
795 			       req->src == req->dst ? rctx->src : rctx->dst,  in crypto_rfc4106_crypt()
796 			       req->cryptlen, iv);  in crypto_rfc4106_crypt()
797 	aead_request_set_ad(subreq, req->assoclen - 8);  in crypto_rfc4106_crypt()
802 static int crypto_rfc4106_encrypt(struct aead_request *req)  in crypto_rfc4106_encrypt()  argument
806 	err = crypto_ipsec_check_assoclen(req->assoclen);  in crypto_rfc4106_encrypt()
810 	req = crypto_rfc4106_crypt(req);  in crypto_rfc4106_encrypt()
812 	return crypto_aead_encrypt(req);  in crypto_rfc4106_encrypt()
815 static int crypto_rfc4106_decrypt(struct aead_request *req)  in crypto_rfc4106_decrypt()  argument
819 	err = crypto_ipsec_check_assoclen(req->assoclen);  in crypto_rfc4106_decrypt()
823 	req = crypto_rfc4106_crypt(req);  in crypto_rfc4106_decrypt()
825 	return crypto_aead_decrypt(req);  in crypto_rfc4106_decrypt()
988 static int crypto_rfc4543_crypt(struct aead_request *req, bool enc)  in crypto_rfc4543_crypt()  argument
990 	struct crypto_aead *aead = crypto_aead_reqtfm(req);  in crypto_rfc4543_crypt()
992 	struct crypto_rfc4543_req_ctx *rctx = aead_request_ctx(req);  in crypto_rfc4543_crypt()
999 	if (req->src != req->dst) {  in crypto_rfc4543_crypt()
1000 		err = crypto_rfc4543_copy_src_to_dst(req, enc);  in crypto_rfc4543_crypt()
1006 	memcpy(iv + 4, req->iv, 8);  in crypto_rfc4543_crypt()
1009 	aead_request_set_callback(subreq, req->base.flags,  in crypto_rfc4543_crypt()
1010 				  req->base.complete, req->base.data);  in crypto_rfc4543_crypt()
1011 	aead_request_set_crypt(subreq, req->src, req->dst,  in crypto_rfc4543_crypt()
1013 	aead_request_set_ad(subreq, req->assoclen + req->cryptlen -  in crypto_rfc4543_crypt()
1019 static int crypto_rfc4543_copy_src_to_dst(struct aead_request *req, bool enc)  in crypto_rfc4543_copy_src_to_dst()  argument
1021 	struct crypto_aead *aead = crypto_aead_reqtfm(req);  in crypto_rfc4543_copy_src_to_dst()
1024 	unsigned int nbytes = req->assoclen + req->cryptlen -  in crypto_rfc4543_copy_src_to_dst()
1029 	skcipher_request_set_callback(nreq, req->base.flags, NULL, NULL);  in crypto_rfc4543_copy_src_to_dst()
1030 	skcipher_request_set_crypt(nreq, req->src, req->dst, nbytes, NULL);  in crypto_rfc4543_copy_src_to_dst()
1035 static int crypto_rfc4543_encrypt(struct aead_request *req)  in crypto_rfc4543_encrypt()  argument
1037 	return crypto_ipsec_check_assoclen(req->assoclen) ?:  in crypto_rfc4543_encrypt()
1038 	       crypto_rfc4543_crypt(req, true);  in crypto_rfc4543_encrypt()
1041 static int crypto_rfc4543_decrypt(struct aead_request *req)  in crypto_rfc4543_decrypt()  argument
1043 	return crypto_ipsec_check_assoclen(req->assoclen) ?:  in crypto_rfc4543_decrypt()
1044 	       crypto_rfc4543_crypt(req, false);  in crypto_rfc4543_decrypt()