63 	void (*complete)(struct aead_request *req, int err);
86 	struct aead_request *req)  in crypto_gcm_reqctx()  argument
88 	unsigned long align = crypto_aead_alignmask(crypto_aead_reqtfm(req));  in crypto_gcm_reqctx()
90 	return (void *)PTR_ALIGN((u8 *)aead_request_ctx(req), align + 1);  in crypto_gcm_reqctx()
93 static void crypto_gcm_setkey_done(struct crypto_async_request *req, int err)  in crypto_gcm_setkey_done()  argument
95 	struct crypto_gcm_setkey_result *result = req->data;  in crypto_gcm_setkey_done()
117 		struct ablkcipher_request req;  in crypto_gcm_setkey()  member
139 	ablkcipher_request_set_tfm(&data->req, ctr);  in crypto_gcm_setkey()
140 	ablkcipher_request_set_callback(&data->req, CRYPTO_TFM_REQ_MAY_SLEEP |  in crypto_gcm_setkey()
144 	ablkcipher_request_set_crypt(&data->req, data->sg, data->sg,  in crypto_gcm_setkey()
147 	err = crypto_ablkcipher_encrypt(&data->req);  in crypto_gcm_setkey()
190 				  struct aead_request *req,  in crypto_gcm_init_crypt()  argument
193 	struct crypto_aead *aead = crypto_aead_reqtfm(req);  in crypto_gcm_init_crypt()
195 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in crypto_gcm_init_crypt()
200 	memcpy(req->iv + 12, &counter, 4);  in crypto_gcm_init_crypt()
204 	scatterwalk_sg_chain(pctx->src, 2, req->src);  in crypto_gcm_init_crypt()
207 	if (req->src != req->dst) {  in crypto_gcm_init_crypt()
210 		scatterwalk_sg_chain(pctx->dst, 2, req->dst);  in crypto_gcm_init_crypt()
217 				     req->iv);  in crypto_gcm_init_crypt()
229 static int gcm_hash_update(struct aead_request *req,  in gcm_hash_update()  argument
237 	ahash_request_set_callback(ahreq, aead_request_flags(req),  in gcm_hash_update()
238 				   complete, req);  in gcm_hash_update()
244 static int gcm_hash_remain(struct aead_request *req,  in gcm_hash_remain()  argument
251 	ahash_request_set_callback(ahreq, aead_request_flags(req),  in gcm_hash_remain()
252 				   complete, req);  in gcm_hash_remain()
259 static int gcm_hash_len(struct aead_request *req,  in gcm_hash_len()  argument
266 	lengths.a = cpu_to_be64(req->assoclen * 8);  in gcm_hash_len()
270 	ahash_request_set_callback(ahreq, aead_request_flags(req),  in gcm_hash_len()
271 				   gcm_hash_len_done, req);  in gcm_hash_len()
278 static int gcm_hash_final(struct aead_request *req,  in gcm_hash_final()  argument
283 	ahash_request_set_callback(ahreq, aead_request_flags(req),  in gcm_hash_final()
284 				   gcm_hash_final_done, req);  in gcm_hash_final()
290 static void __gcm_hash_final_done(struct aead_request *req, int err)  in __gcm_hash_final_done()  argument
292 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in __gcm_hash_final_done()
298 	gctx->complete(req, err);  in __gcm_hash_final_done()
303 	struct aead_request *req = areq->data;  in gcm_hash_final_done()  local
305 	__gcm_hash_final_done(req, err);  in gcm_hash_final_done()
308 static void __gcm_hash_len_done(struct aead_request *req, int err)  in __gcm_hash_len_done()  argument
310 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in __gcm_hash_len_done()
313 		err = gcm_hash_final(req, pctx);  in __gcm_hash_len_done()
318 	__gcm_hash_final_done(req, err);  in __gcm_hash_len_done()
323 	struct aead_request *req = areq->data;  in gcm_hash_len_done()  local
325 	__gcm_hash_len_done(req, err);  in gcm_hash_len_done()
328 static void __gcm_hash_crypt_remain_done(struct aead_request *req, int err)  in __gcm_hash_crypt_remain_done()  argument
330 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in __gcm_hash_crypt_remain_done()
333 		err = gcm_hash_len(req, pctx);  in __gcm_hash_crypt_remain_done()
338 	__gcm_hash_len_done(req, err);  in __gcm_hash_crypt_remain_done()
344 	struct aead_request *req = areq->data;  in gcm_hash_crypt_remain_done()  local
346 	__gcm_hash_crypt_remain_done(req, err);  in gcm_hash_crypt_remain_done()
349 static void __gcm_hash_crypt_done(struct aead_request *req, int err)  in __gcm_hash_crypt_done()  argument
351 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in __gcm_hash_crypt_done()
358 		err = gcm_hash_remain(req, pctx, remain,  in __gcm_hash_crypt_done()
364 	__gcm_hash_crypt_remain_done(req, err);  in __gcm_hash_crypt_done()
369 	struct aead_request *req = areq->data;  in gcm_hash_crypt_done()  local
371 	__gcm_hash_crypt_done(req, err);  in gcm_hash_crypt_done()
374 static void __gcm_hash_assoc_remain_done(struct aead_request *req, int err)  in __gcm_hash_assoc_remain_done()  argument
376 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in __gcm_hash_assoc_remain_done()
385 		err = gcm_hash_update(req, pctx, complete,  in __gcm_hash_assoc_remain_done()
392 		__gcm_hash_crypt_done(req, err);  in __gcm_hash_assoc_remain_done()
394 		__gcm_hash_crypt_remain_done(req, err);  in __gcm_hash_assoc_remain_done()
400 	struct aead_request *req = areq->data;  in gcm_hash_assoc_remain_done()  local
402 	__gcm_hash_assoc_remain_done(req, err);  in gcm_hash_assoc_remain_done()
405 static void __gcm_hash_assoc_done(struct aead_request *req, int err)  in __gcm_hash_assoc_done()  argument
407 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in __gcm_hash_assoc_done()
411 		remain = gcm_remain(req->assoclen);  in __gcm_hash_assoc_done()
413 		err = gcm_hash_remain(req, pctx, remain,  in __gcm_hash_assoc_done()
419 	__gcm_hash_assoc_remain_done(req, err);  in __gcm_hash_assoc_done()
424 	struct aead_request *req = areq->data;  in gcm_hash_assoc_done()  local
426 	__gcm_hash_assoc_done(req, err);  in gcm_hash_assoc_done()
429 static void __gcm_hash_init_done(struct aead_request *req, int err)  in __gcm_hash_init_done()  argument
431 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in __gcm_hash_init_done()
435 	if (!err && req->assoclen) {  in __gcm_hash_init_done()
436 		remain = gcm_remain(req->assoclen);  in __gcm_hash_init_done()
439 		err = gcm_hash_update(req, pctx, complete,  in __gcm_hash_init_done()
440 				      req->assoc, req->assoclen);  in __gcm_hash_init_done()
446 		__gcm_hash_assoc_done(req, err);  in __gcm_hash_init_done()
448 		__gcm_hash_assoc_remain_done(req, err);  in __gcm_hash_init_done()
453 	struct aead_request *req = areq->data;  in gcm_hash_init_done()  local
455 	__gcm_hash_init_done(req, err);  in gcm_hash_init_done()
458 static int gcm_hash(struct aead_request *req,  in gcm_hash()  argument
463 	struct crypto_gcm_ctx *ctx = crypto_tfm_ctx(req->base.tfm);  in gcm_hash()
470 	ahash_request_set_callback(ahreq, aead_request_flags(req),  in gcm_hash()
471 				   gcm_hash_init_done, req);  in gcm_hash()
475 	remain = gcm_remain(req->assoclen);  in gcm_hash()
477 	err = gcm_hash_update(req, pctx, complete, req->assoc, req->assoclen);  in gcm_hash()
481 		err = gcm_hash_remain(req, pctx, remain,  in gcm_hash()
488 	err = gcm_hash_update(req, pctx, complete, gctx->src, gctx->cryptlen);  in gcm_hash()
492 		err = gcm_hash_remain(req, pctx, remain,  in gcm_hash()
497 	err = gcm_hash_len(req, pctx);  in gcm_hash()
500 	err = gcm_hash_final(req, pctx);  in gcm_hash()
507 static void gcm_enc_copy_hash(struct aead_request *req,  in gcm_enc_copy_hash()  argument
510 	struct crypto_aead *aead = crypto_aead_reqtfm(req);  in gcm_enc_copy_hash()
513 	scatterwalk_map_and_copy(auth_tag, req->dst, req->cryptlen,  in gcm_enc_copy_hash()
517 static void gcm_enc_hash_done(struct aead_request *req, int err)  in gcm_enc_hash_done()  argument
519 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in gcm_enc_hash_done()
522 		gcm_enc_copy_hash(req, pctx);  in gcm_enc_hash_done()
524 	aead_request_complete(req, err);  in gcm_enc_hash_done()
529 	struct aead_request *req = areq->data;  in gcm_encrypt_done()  local
530 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in gcm_encrypt_done()
533 		err = gcm_hash(req, pctx);  in gcm_encrypt_done()
538 			gcm_enc_copy_hash(req, pctx);  in gcm_encrypt_done()
542 	aead_request_complete(req, err);  in gcm_encrypt_done()
545 static int crypto_gcm_encrypt(struct aead_request *req)  in crypto_gcm_encrypt()  argument
547 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in crypto_gcm_encrypt()
552 	crypto_gcm_init_crypt(abreq, req, req->cryptlen);  in crypto_gcm_encrypt()
553 	ablkcipher_request_set_callback(abreq, aead_request_flags(req),  in crypto_gcm_encrypt()
554 					gcm_encrypt_done, req);  in crypto_gcm_encrypt()
556 	gctx->src = req->dst;  in crypto_gcm_encrypt()
557 	gctx->cryptlen = req->cryptlen;  in crypto_gcm_encrypt()
564 	err = gcm_hash(req, pctx);  in crypto_gcm_encrypt()
569 	gcm_enc_copy_hash(req, pctx);  in crypto_gcm_encrypt()
574 static int crypto_gcm_verify(struct aead_request *req,  in crypto_gcm_verify()  argument
577 	struct crypto_aead *aead = crypto_aead_reqtfm(req);  in crypto_gcm_verify()
581 	unsigned int cryptlen = req->cryptlen - authsize;  in crypto_gcm_verify()
584 	scatterwalk_map_and_copy(iauth_tag, req->src, cryptlen, authsize, 0);  in crypto_gcm_verify()
590 	struct aead_request *req = areq->data;  in gcm_decrypt_done()  local
591 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in gcm_decrypt_done()
594 		err = crypto_gcm_verify(req, pctx);  in gcm_decrypt_done()
596 	aead_request_complete(req, err);  in gcm_decrypt_done()
599 static void gcm_dec_hash_done(struct aead_request *req, int err)  in gcm_dec_hash_done()  argument
601 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in gcm_dec_hash_done()
606 		ablkcipher_request_set_callback(abreq, aead_request_flags(req),  in gcm_dec_hash_done()
607 						gcm_decrypt_done, req);  in gcm_dec_hash_done()
608 		crypto_gcm_init_crypt(abreq, req, gctx->cryptlen);  in gcm_dec_hash_done()
613 			err = crypto_gcm_verify(req, pctx);  in gcm_dec_hash_done()
616 	aead_request_complete(req, err);  in gcm_dec_hash_done()
619 static int crypto_gcm_decrypt(struct aead_request *req)  in crypto_gcm_decrypt()  argument
621 	struct crypto_aead *aead = crypto_aead_reqtfm(req);  in crypto_gcm_decrypt()
622 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in crypto_gcm_decrypt()
626 	unsigned int cryptlen = req->cryptlen;  in crypto_gcm_decrypt()
633 	gctx->src = req->src;  in crypto_gcm_decrypt()
637 	err = gcm_hash(req, pctx);  in crypto_gcm_decrypt()
641 	ablkcipher_request_set_callback(abreq, aead_request_flags(req),  in crypto_gcm_decrypt()
642 					gcm_decrypt_done, req);  in crypto_gcm_decrypt()
643 	crypto_gcm_init_crypt(abreq, req, cryptlen);  in crypto_gcm_decrypt()
648 	return crypto_gcm_verify(req, pctx);  in crypto_gcm_decrypt()
898 static struct aead_request *crypto_rfc4106_crypt(struct aead_request *req)  in crypto_rfc4106_crypt()  argument
900 	struct aead_request *subreq = aead_request_ctx(req);  in crypto_rfc4106_crypt()
901 	struct crypto_aead *aead = crypto_aead_reqtfm(req);  in crypto_rfc4106_crypt()
908 	memcpy(iv + 4, req->iv, 8);  in crypto_rfc4106_crypt()
911 	aead_request_set_callback(subreq, req->base.flags, req->base.complete,  in crypto_rfc4106_crypt()
912 				  req->base.data);  in crypto_rfc4106_crypt()
913 	aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen, iv);  in crypto_rfc4106_crypt()
914 	aead_request_set_assoc(subreq, req->assoc, req->assoclen);  in crypto_rfc4106_crypt()
919 static int crypto_rfc4106_encrypt(struct aead_request *req)  in crypto_rfc4106_encrypt()  argument
921 	req = crypto_rfc4106_crypt(req);  in crypto_rfc4106_encrypt()
923 	return crypto_aead_encrypt(req);  in crypto_rfc4106_encrypt()
926 static int crypto_rfc4106_decrypt(struct aead_request *req)  in crypto_rfc4106_decrypt()  argument
928 	req = crypto_rfc4106_crypt(req);  in crypto_rfc4106_decrypt()
930 	return crypto_aead_decrypt(req);  in crypto_rfc4106_decrypt()
1062 	struct aead_request *req)  in crypto_rfc4543_reqctx()  argument
1064 	unsigned long align = crypto_aead_alignmask(crypto_aead_reqtfm(req));  in crypto_rfc4543_reqctx()
1066 	return (void *)PTR_ALIGN((u8 *)aead_request_ctx(req), align + 1);  in crypto_rfc4543_reqctx()
1105 	struct aead_request *req = areq->data;  in crypto_rfc4543_done()  local
1106 	struct crypto_aead *aead = crypto_aead_reqtfm(req);  in crypto_rfc4543_done()
1107 	struct crypto_rfc4543_req_ctx *rctx = crypto_rfc4543_reqctx(req);  in crypto_rfc4543_done()
1110 		scatterwalk_map_and_copy(rctx->auth_tag, req->dst,  in crypto_rfc4543_done()
1111 					 req->cryptlen,  in crypto_rfc4543_done()
1115 	aead_request_complete(req, err);  in crypto_rfc4543_done()
1118 static struct aead_request *crypto_rfc4543_crypt(struct aead_request *req,  in crypto_rfc4543_crypt()  argument
1121 	struct crypto_aead *aead = crypto_aead_reqtfm(req);  in crypto_rfc4543_crypt()
1123 	struct crypto_rfc4543_req_ctx *rctx = crypto_rfc4543_reqctx(req);  in crypto_rfc4543_crypt()
1125 	struct scatterlist *src = req->src;  in crypto_rfc4543_crypt()
1130 	unsigned int assoclen = req->assoclen;  in crypto_rfc4543_crypt()
1137 	memcpy(iv + 4, req->iv, 8);  in crypto_rfc4543_crypt()
1144 					 req->cryptlen - authsize,  in crypto_rfc4543_crypt()
1154 	sg_set_buf(payload, req->iv, 8);  in crypto_rfc4543_crypt()
1155 	scatterwalk_crypto_chain(payload, src, vsrc == req->iv + 8, 2);  in crypto_rfc4543_crypt()
1156 	assoclen += 8 + req->cryptlen - (enc ? 0 : authsize);  in crypto_rfc4543_crypt()
1158 	if (req->assoc->length == req->assoclen) {  in crypto_rfc4543_crypt()
1160 		sg_set_page(assoc, sg_page(req->assoc), req->assoc->length,  in crypto_rfc4543_crypt()
1161 			    req->assoc->offset);  in crypto_rfc4543_crypt()
1163 		BUG_ON(req->assoclen > sizeof(rctx->assocbuf));  in crypto_rfc4543_crypt()
1165 		scatterwalk_map_and_copy(rctx->assocbuf, req->assoc, 0,  in crypto_rfc4543_crypt()
1166 					 req->assoclen, 0);  in crypto_rfc4543_crypt()
1169 		sg_set_buf(assoc, rctx->assocbuf, req->assoclen);  in crypto_rfc4543_crypt()
1174 	aead_request_set_callback(subreq, req->base.flags, crypto_rfc4543_done,  in crypto_rfc4543_crypt()
1175 				  req);  in crypto_rfc4543_crypt()
1182 static int crypto_rfc4543_copy_src_to_dst(struct aead_request *req, bool enc)  in crypto_rfc4543_copy_src_to_dst()  argument
1184 	struct crypto_aead *aead = crypto_aead_reqtfm(req);  in crypto_rfc4543_copy_src_to_dst()
1187 	unsigned int nbytes = req->cryptlen - (enc ? 0 : authsize);  in crypto_rfc4543_copy_src_to_dst()
1192 	return crypto_blkcipher_encrypt(&desc, req->dst, req->src, nbytes);  in crypto_rfc4543_copy_src_to_dst()
1195 static int crypto_rfc4543_encrypt(struct aead_request *req)  in crypto_rfc4543_encrypt()  argument
1197 	struct crypto_aead *aead = crypto_aead_reqtfm(req);  in crypto_rfc4543_encrypt()
1198 	struct crypto_rfc4543_req_ctx *rctx = crypto_rfc4543_reqctx(req);  in crypto_rfc4543_encrypt()
1202 	if (req->src != req->dst) {  in crypto_rfc4543_encrypt()
1203 		err = crypto_rfc4543_copy_src_to_dst(req, true);  in crypto_rfc4543_encrypt()
1208 	subreq = crypto_rfc4543_crypt(req, true);  in crypto_rfc4543_encrypt()
1213 	scatterwalk_map_and_copy(rctx->auth_tag, req->dst, req->cryptlen,  in crypto_rfc4543_encrypt()
1219 static int crypto_rfc4543_decrypt(struct aead_request *req)  in crypto_rfc4543_decrypt()  argument
1223 	if (req->src != req->dst) {  in crypto_rfc4543_decrypt()
1224 		err = crypto_rfc4543_copy_src_to_dst(req, false);  in crypto_rfc4543_decrypt()
1229 	req = crypto_rfc4543_crypt(req, false);  in crypto_rfc4543_decrypt()
1231 	return crypto_aead_decrypt(req);  in crypto_rfc4543_decrypt()