63 	void (*complete)(struct aead_request *req, int err);
86 	struct aead_request *req)  in crypto_gcm_reqctx()  argument
88 	unsigned long align = crypto_aead_alignmask(crypto_aead_reqtfm(req));  in crypto_gcm_reqctx()
90 	return (void *)PTR_ALIGN((u8 *)aead_request_ctx(req), align + 1);  in crypto_gcm_reqctx()
93 static void crypto_gcm_setkey_done(struct crypto_async_request *req, int err)  in crypto_gcm_setkey_done()  argument
95 	struct crypto_gcm_setkey_result *result = req->data;  in crypto_gcm_setkey_done()
117 		struct ablkcipher_request req;  in crypto_gcm_setkey()  member
139 	ablkcipher_request_set_tfm(&data->req, ctr);  in crypto_gcm_setkey()
140 	ablkcipher_request_set_callback(&data->req, CRYPTO_TFM_REQ_MAY_SLEEP |  in crypto_gcm_setkey()
144 	ablkcipher_request_set_crypt(&data->req, data->sg, data->sg,  in crypto_gcm_setkey()
147 	err = crypto_ablkcipher_encrypt(&data->req);  in crypto_gcm_setkey()
188 				  struct aead_request *req,  in crypto_gcm_init_crypt()  argument
191 	struct crypto_aead *aead = crypto_aead_reqtfm(req);  in crypto_gcm_init_crypt()
193 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in crypto_gcm_init_crypt()
198 	memcpy(req->iv + 12, &counter, 4);  in crypto_gcm_init_crypt()
202 	scatterwalk_sg_chain(pctx->src, 2, req->src);  in crypto_gcm_init_crypt()
205 	if (req->src != req->dst) {  in crypto_gcm_init_crypt()
208 		scatterwalk_sg_chain(pctx->dst, 2, req->dst);  in crypto_gcm_init_crypt()
215 				     req->iv);  in crypto_gcm_init_crypt()
227 static int gcm_hash_update(struct aead_request *req,  in gcm_hash_update()  argument
235 	ahash_request_set_callback(ahreq, aead_request_flags(req),  in gcm_hash_update()
236 				   compl, req);  in gcm_hash_update()
242 static int gcm_hash_remain(struct aead_request *req,  in gcm_hash_remain()  argument
249 	ahash_request_set_callback(ahreq, aead_request_flags(req),  in gcm_hash_remain()
250 				   compl, req);  in gcm_hash_remain()
257 static int gcm_hash_len(struct aead_request *req,  in gcm_hash_len()  argument
264 	lengths.a = cpu_to_be64(req->assoclen * 8);  in gcm_hash_len()
268 	ahash_request_set_callback(ahreq, aead_request_flags(req),  in gcm_hash_len()
269 				   gcm_hash_len_done, req);  in gcm_hash_len()
276 static int gcm_hash_final(struct aead_request *req,  in gcm_hash_final()  argument
281 	ahash_request_set_callback(ahreq, aead_request_flags(req),  in gcm_hash_final()
282 				   gcm_hash_final_done, req);  in gcm_hash_final()
288 static void __gcm_hash_final_done(struct aead_request *req, int err)  in __gcm_hash_final_done()  argument
290 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in __gcm_hash_final_done()
296 	gctx->complete(req, err);  in __gcm_hash_final_done()
301 	struct aead_request *req = areq->data;  in gcm_hash_final_done()  local
303 	__gcm_hash_final_done(req, err);  in gcm_hash_final_done()
306 static void __gcm_hash_len_done(struct aead_request *req, int err)  in __gcm_hash_len_done()  argument
308 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in __gcm_hash_len_done()
311 		err = gcm_hash_final(req, pctx);  in __gcm_hash_len_done()
316 	__gcm_hash_final_done(req, err);  in __gcm_hash_len_done()
321 	struct aead_request *req = areq->data;  in gcm_hash_len_done()  local
323 	__gcm_hash_len_done(req, err);  in gcm_hash_len_done()
326 static void __gcm_hash_crypt_remain_done(struct aead_request *req, int err)  in __gcm_hash_crypt_remain_done()  argument
328 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in __gcm_hash_crypt_remain_done()
331 		err = gcm_hash_len(req, pctx);  in __gcm_hash_crypt_remain_done()
336 	__gcm_hash_len_done(req, err);  in __gcm_hash_crypt_remain_done()
342 	struct aead_request *req = areq->data;  in gcm_hash_crypt_remain_done()  local
344 	__gcm_hash_crypt_remain_done(req, err);  in gcm_hash_crypt_remain_done()
347 static void __gcm_hash_crypt_done(struct aead_request *req, int err)  in __gcm_hash_crypt_done()  argument
349 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in __gcm_hash_crypt_done()
356 		err = gcm_hash_remain(req, pctx, remain,  in __gcm_hash_crypt_done()
362 	__gcm_hash_crypt_remain_done(req, err);  in __gcm_hash_crypt_done()
367 	struct aead_request *req = areq->data;  in gcm_hash_crypt_done()  local
369 	__gcm_hash_crypt_done(req, err);  in gcm_hash_crypt_done()
372 static void __gcm_hash_assoc_remain_done(struct aead_request *req, int err)  in __gcm_hash_assoc_remain_done()  argument
374 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in __gcm_hash_assoc_remain_done()
383 		err = gcm_hash_update(req, pctx, compl,  in __gcm_hash_assoc_remain_done()
390 		__gcm_hash_crypt_done(req, err);  in __gcm_hash_assoc_remain_done()
392 		__gcm_hash_crypt_remain_done(req, err);  in __gcm_hash_assoc_remain_done()
398 	struct aead_request *req = areq->data;  in gcm_hash_assoc_remain_done()  local
400 	__gcm_hash_assoc_remain_done(req, err);  in gcm_hash_assoc_remain_done()
403 static void __gcm_hash_assoc_done(struct aead_request *req, int err)  in __gcm_hash_assoc_done()  argument
405 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in __gcm_hash_assoc_done()
409 		remain = gcm_remain(req->assoclen);  in __gcm_hash_assoc_done()
411 		err = gcm_hash_remain(req, pctx, remain,  in __gcm_hash_assoc_done()
417 	__gcm_hash_assoc_remain_done(req, err);  in __gcm_hash_assoc_done()
422 	struct aead_request *req = areq->data;  in gcm_hash_assoc_done()  local
424 	__gcm_hash_assoc_done(req, err);  in gcm_hash_assoc_done()
427 static void __gcm_hash_init_done(struct aead_request *req, int err)  in __gcm_hash_init_done()  argument
429 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in __gcm_hash_init_done()
433 	if (!err && req->assoclen) {  in __gcm_hash_init_done()
434 		remain = gcm_remain(req->assoclen);  in __gcm_hash_init_done()
437 		err = gcm_hash_update(req, pctx, compl,  in __gcm_hash_init_done()
438 				      req->assoc, req->assoclen);  in __gcm_hash_init_done()
444 		__gcm_hash_assoc_done(req, err);  in __gcm_hash_init_done()
446 		__gcm_hash_assoc_remain_done(req, err);  in __gcm_hash_init_done()
451 	struct aead_request *req = areq->data;  in gcm_hash_init_done()  local
453 	__gcm_hash_init_done(req, err);  in gcm_hash_init_done()
456 static int gcm_hash(struct aead_request *req,  in gcm_hash()  argument
461 	struct crypto_gcm_ctx *ctx = crypto_tfm_ctx(req->base.tfm);  in gcm_hash()
468 	ahash_request_set_callback(ahreq, aead_request_flags(req),  in gcm_hash()
469 				   gcm_hash_init_done, req);  in gcm_hash()
473 	remain = gcm_remain(req->assoclen);  in gcm_hash()
475 	err = gcm_hash_update(req, pctx, compl, req->assoc, req->assoclen);  in gcm_hash()
479 		err = gcm_hash_remain(req, pctx, remain,  in gcm_hash()
486 	err = gcm_hash_update(req, pctx, compl, gctx->src, gctx->cryptlen);  in gcm_hash()
490 		err = gcm_hash_remain(req, pctx, remain,  in gcm_hash()
495 	err = gcm_hash_len(req, pctx);  in gcm_hash()
498 	err = gcm_hash_final(req, pctx);  in gcm_hash()
505 static void gcm_enc_copy_hash(struct aead_request *req,  in gcm_enc_copy_hash()  argument
508 	struct crypto_aead *aead = crypto_aead_reqtfm(req);  in gcm_enc_copy_hash()
511 	scatterwalk_map_and_copy(auth_tag, req->dst, req->cryptlen,  in gcm_enc_copy_hash()
515 static void gcm_enc_hash_done(struct aead_request *req, int err)  in gcm_enc_hash_done()  argument
517 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in gcm_enc_hash_done()
520 		gcm_enc_copy_hash(req, pctx);  in gcm_enc_hash_done()
522 	aead_request_complete(req, err);  in gcm_enc_hash_done()
527 	struct aead_request *req = areq->data;  in gcm_encrypt_done()  local
528 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in gcm_encrypt_done()
531 		err = gcm_hash(req, pctx);  in gcm_encrypt_done()
536 			gcm_enc_copy_hash(req, pctx);  in gcm_encrypt_done()
540 	aead_request_complete(req, err);  in gcm_encrypt_done()
543 static int crypto_gcm_encrypt(struct aead_request *req)  in crypto_gcm_encrypt()  argument
545 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in crypto_gcm_encrypt()
550 	crypto_gcm_init_crypt(abreq, req, req->cryptlen);  in crypto_gcm_encrypt()
551 	ablkcipher_request_set_callback(abreq, aead_request_flags(req),  in crypto_gcm_encrypt()
552 					gcm_encrypt_done, req);  in crypto_gcm_encrypt()
554 	gctx->src = req->dst;  in crypto_gcm_encrypt()
555 	gctx->cryptlen = req->cryptlen;  in crypto_gcm_encrypt()
562 	err = gcm_hash(req, pctx);  in crypto_gcm_encrypt()
567 	gcm_enc_copy_hash(req, pctx);  in crypto_gcm_encrypt()
572 static int crypto_gcm_verify(struct aead_request *req,  in crypto_gcm_verify()  argument
575 	struct crypto_aead *aead = crypto_aead_reqtfm(req);  in crypto_gcm_verify()
579 	unsigned int cryptlen = req->cryptlen - authsize;  in crypto_gcm_verify()
582 	scatterwalk_map_and_copy(iauth_tag, req->src, cryptlen, authsize, 0);  in crypto_gcm_verify()
588 	struct aead_request *req = areq->data;  in gcm_decrypt_done()  local
589 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in gcm_decrypt_done()
592 		err = crypto_gcm_verify(req, pctx);  in gcm_decrypt_done()
594 	aead_request_complete(req, err);  in gcm_decrypt_done()
597 static void gcm_dec_hash_done(struct aead_request *req, int err)  in gcm_dec_hash_done()  argument
599 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in gcm_dec_hash_done()
604 		ablkcipher_request_set_callback(abreq, aead_request_flags(req),  in gcm_dec_hash_done()
605 						gcm_decrypt_done, req);  in gcm_dec_hash_done()
606 		crypto_gcm_init_crypt(abreq, req, gctx->cryptlen);  in gcm_dec_hash_done()
611 			err = crypto_gcm_verify(req, pctx);  in gcm_dec_hash_done()
614 	aead_request_complete(req, err);  in gcm_dec_hash_done()
617 static int crypto_gcm_decrypt(struct aead_request *req)  in crypto_gcm_decrypt()  argument
619 	struct crypto_aead *aead = crypto_aead_reqtfm(req);  in crypto_gcm_decrypt()
620 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in crypto_gcm_decrypt()
624 	unsigned int cryptlen = req->cryptlen;  in crypto_gcm_decrypt()
631 	gctx->src = req->src;  in crypto_gcm_decrypt()
635 	err = gcm_hash(req, pctx);  in crypto_gcm_decrypt()
639 	ablkcipher_request_set_callback(abreq, aead_request_flags(req),  in crypto_gcm_decrypt()
640 					gcm_decrypt_done, req);  in crypto_gcm_decrypt()
641 	crypto_gcm_init_crypt(abreq, req, cryptlen);  in crypto_gcm_decrypt()
646 	return crypto_gcm_verify(req, pctx);  in crypto_gcm_decrypt()
898 static struct aead_request *crypto_rfc4106_crypt(struct aead_request *req)  in crypto_rfc4106_crypt()  argument
900 	struct aead_request *subreq = aead_request_ctx(req);  in crypto_rfc4106_crypt()
901 	struct crypto_aead *aead = crypto_aead_reqtfm(req);  in crypto_rfc4106_crypt()
908 	memcpy(iv + 4, req->iv, 8);  in crypto_rfc4106_crypt()
911 	aead_request_set_callback(subreq, req->base.flags, req->base.complete,  in crypto_rfc4106_crypt()
912 				  req->base.data);  in crypto_rfc4106_crypt()
913 	aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen, iv);  in crypto_rfc4106_crypt()
914 	aead_request_set_assoc(subreq, req->assoc, req->assoclen);  in crypto_rfc4106_crypt()
919 static int crypto_rfc4106_encrypt(struct aead_request *req)  in crypto_rfc4106_encrypt()  argument
921 	req = crypto_rfc4106_crypt(req);  in crypto_rfc4106_encrypt()
923 	return crypto_aead_encrypt(req);  in crypto_rfc4106_encrypt()
926 static int crypto_rfc4106_decrypt(struct aead_request *req)  in crypto_rfc4106_decrypt()  argument
928 	req = crypto_rfc4106_crypt(req);  in crypto_rfc4106_decrypt()
930 	return crypto_aead_decrypt(req);  in crypto_rfc4106_decrypt()
1062 	struct aead_request *req)  in crypto_rfc4543_reqctx()  argument
1064 	unsigned long align = crypto_aead_alignmask(crypto_aead_reqtfm(req));  in crypto_rfc4543_reqctx()
1066 	return (void *)PTR_ALIGN((u8 *)aead_request_ctx(req), align + 1);  in crypto_rfc4543_reqctx()
1105 	struct aead_request *req = areq->data;  in crypto_rfc4543_done()  local
1106 	struct crypto_aead *aead = crypto_aead_reqtfm(req);  in crypto_rfc4543_done()
1107 	struct crypto_rfc4543_req_ctx *rctx = crypto_rfc4543_reqctx(req);  in crypto_rfc4543_done()
1110 		scatterwalk_map_and_copy(rctx->auth_tag, req->dst,  in crypto_rfc4543_done()
1111 					 req->cryptlen,  in crypto_rfc4543_done()
1115 	aead_request_complete(req, err);  in crypto_rfc4543_done()
1118 static struct aead_request *crypto_rfc4543_crypt(struct aead_request *req,  in crypto_rfc4543_crypt()  argument
1121 	struct crypto_aead *aead = crypto_aead_reqtfm(req);  in crypto_rfc4543_crypt()
1123 	struct crypto_rfc4543_req_ctx *rctx = crypto_rfc4543_reqctx(req);  in crypto_rfc4543_crypt()
1125 	struct scatterlist *src = req->src;  in crypto_rfc4543_crypt()
1130 	unsigned int assoclen = req->assoclen;  in crypto_rfc4543_crypt()
1137 	memcpy(iv + 4, req->iv, 8);  in crypto_rfc4543_crypt()
1144 					 req->cryptlen - authsize,  in crypto_rfc4543_crypt()
1154 	sg_set_buf(payload, req->iv, 8);  in crypto_rfc4543_crypt()
1155 	scatterwalk_crypto_chain(payload, src, vsrc == req->iv + 8, 2);  in crypto_rfc4543_crypt()
1156 	assoclen += 8 + req->cryptlen - (enc ? 0 : authsize);  in crypto_rfc4543_crypt()
1158 	if (req->assoc->length == req->assoclen) {  in crypto_rfc4543_crypt()
1160 		sg_set_page(assoc, sg_page(req->assoc), req->assoc->length,  in crypto_rfc4543_crypt()
1161 			    req->assoc->offset);  in crypto_rfc4543_crypt()
1163 		BUG_ON(req->assoclen > sizeof(rctx->assocbuf));  in crypto_rfc4543_crypt()
1165 		scatterwalk_map_and_copy(rctx->assocbuf, req->assoc, 0,  in crypto_rfc4543_crypt()
1166 					 req->assoclen, 0);  in crypto_rfc4543_crypt()
1169 		sg_set_buf(assoc, rctx->assocbuf, req->assoclen);  in crypto_rfc4543_crypt()
1174 	aead_request_set_callback(subreq, req->base.flags, crypto_rfc4543_done,  in crypto_rfc4543_crypt()
1175 				  req);  in crypto_rfc4543_crypt()
1182 static int crypto_rfc4543_copy_src_to_dst(struct aead_request *req, bool enc)  in crypto_rfc4543_copy_src_to_dst()  argument
1184 	struct crypto_aead *aead = crypto_aead_reqtfm(req);  in crypto_rfc4543_copy_src_to_dst()
1187 	unsigned int nbytes = req->cryptlen - (enc ? 0 : authsize);  in crypto_rfc4543_copy_src_to_dst()
1192 	return crypto_blkcipher_encrypt(&desc, req->dst, req->src, nbytes);  in crypto_rfc4543_copy_src_to_dst()
1195 static int crypto_rfc4543_encrypt(struct aead_request *req)  in crypto_rfc4543_encrypt()  argument
1197 	struct crypto_aead *aead = crypto_aead_reqtfm(req);  in crypto_rfc4543_encrypt()
1198 	struct crypto_rfc4543_req_ctx *rctx = crypto_rfc4543_reqctx(req);  in crypto_rfc4543_encrypt()
1202 	if (req->src != req->dst) {  in crypto_rfc4543_encrypt()
1203 		err = crypto_rfc4543_copy_src_to_dst(req, true);  in crypto_rfc4543_encrypt()
1208 	subreq = crypto_rfc4543_crypt(req, true);  in crypto_rfc4543_encrypt()
1213 	scatterwalk_map_and_copy(rctx->auth_tag, req->dst, req->cryptlen,  in crypto_rfc4543_encrypt()
1219 static int crypto_rfc4543_decrypt(struct aead_request *req)  in crypto_rfc4543_decrypt()  argument
1223 	if (req->src != req->dst) {  in crypto_rfc4543_decrypt()
1224 		err = crypto_rfc4543_copy_src_to_dst(req, false);  in crypto_rfc4543_decrypt()
1229 	req = crypto_rfc4543_crypt(req, false);  in crypto_rfc4543_decrypt()
1231 	return crypto_aead_decrypt(req);  in crypto_rfc4543_decrypt()