64 	int (*complete)(struct aead_request *req, u32 flags);
91 static int crypto_rfc4543_copy_src_to_dst(struct aead_request *req, bool enc);
94 	struct aead_request *req)  in crypto_gcm_reqctx()  argument
96 	unsigned long align = crypto_aead_alignmask(crypto_aead_reqtfm(req));  in crypto_gcm_reqctx()
98 	return (void *)PTR_ALIGN((u8 *)aead_request_ctx(req), align + 1);  in crypto_gcm_reqctx()
101 static void crypto_gcm_setkey_done(struct crypto_async_request *req, int err)  in crypto_gcm_setkey_done()  argument
103 	struct crypto_gcm_setkey_result *result = req->data;  in crypto_gcm_setkey_done()
125 		struct skcipher_request req;  in crypto_gcm_setkey()  member
145 	skcipher_request_set_tfm(&data->req, ctr);  in crypto_gcm_setkey()
146 	skcipher_request_set_callback(&data->req, CRYPTO_TFM_REQ_MAY_SLEEP |  in crypto_gcm_setkey()
150 	skcipher_request_set_crypt(&data->req, data->sg, data->sg,  in crypto_gcm_setkey()
153 	err = crypto_skcipher_encrypt(&data->req);  in crypto_gcm_setkey()
193 static void crypto_gcm_init_common(struct aead_request *req)  in crypto_gcm_init_common()  argument
195 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in crypto_gcm_init_common()
200 	memcpy(pctx->iv, req->iv, 12);  in crypto_gcm_init_common()
205 	sg = scatterwalk_ffwd(pctx->src + 1, req->src, req->assoclen);  in crypto_gcm_init_common()
209 	if (req->src != req->dst) {  in crypto_gcm_init_common()
212 		sg = scatterwalk_ffwd(pctx->dst + 1, req->dst, req->assoclen);  in crypto_gcm_init_common()
218 static void crypto_gcm_init_crypt(struct aead_request *req,  in crypto_gcm_init_crypt()  argument
221 	struct crypto_aead *aead = crypto_aead_reqtfm(req);  in crypto_gcm_init_crypt()
223 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in crypto_gcm_init_crypt()
227 	dst = req->src == req->dst ? pctx->src : pctx->dst;  in crypto_gcm_init_crypt()
243 static int gcm_hash_update(struct aead_request *req,  in gcm_hash_update()  argument
248 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in gcm_hash_update()
251 	ahash_request_set_callback(ahreq, flags, compl, req);  in gcm_hash_update()
257 static int gcm_hash_remain(struct aead_request *req,  in gcm_hash_remain()  argument
261 	return gcm_hash_update(req, compl, &gcm_zeroes->sg, remain, flags);  in gcm_hash_remain()
264 static int gcm_hash_len(struct aead_request *req, u32 flags)  in gcm_hash_len()  argument
266 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in gcm_hash_len()
271 	lengths.a = cpu_to_be64(req->assoclen * 8);  in gcm_hash_len()
275 	ahash_request_set_callback(ahreq, flags, gcm_hash_len_done, req);  in gcm_hash_len()
282 static int gcm_hash_len_continue(struct aead_request *req, u32 flags)  in gcm_hash_len_continue()  argument
284 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in gcm_hash_len_continue()
287 	return gctx->complete(req, flags);  in gcm_hash_len_continue()
292 	struct aead_request *req = areq->data;  in gcm_hash_len_done()  local
297 	err = gcm_hash_len_continue(req, 0);  in gcm_hash_len_done()
302 	aead_request_complete(req, err);  in gcm_hash_len_done()
305 static int gcm_hash_crypt_remain_continue(struct aead_request *req, u32 flags)  in gcm_hash_crypt_remain_continue()  argument
307 	return gcm_hash_len(req, flags) ?:  in gcm_hash_crypt_remain_continue()
308 	       gcm_hash_len_continue(req, flags);  in gcm_hash_crypt_remain_continue()
314 	struct aead_request *req = areq->data;  in gcm_hash_crypt_remain_done()  local
319 	err = gcm_hash_crypt_remain_continue(req, 0);  in gcm_hash_crypt_remain_done()
324 	aead_request_complete(req, err);  in gcm_hash_crypt_remain_done()
327 static int gcm_hash_crypt_continue(struct aead_request *req, u32 flags)  in gcm_hash_crypt_continue()  argument
329 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in gcm_hash_crypt_continue()
335 		return gcm_hash_remain(req, remain,  in gcm_hash_crypt_continue()
337 		       gcm_hash_crypt_remain_continue(req, flags);  in gcm_hash_crypt_continue()
339 	return gcm_hash_crypt_remain_continue(req, flags);  in gcm_hash_crypt_continue()
344 	struct aead_request *req = areq->data;  in gcm_hash_crypt_done()  local
349 	err = gcm_hash_crypt_continue(req, 0);  in gcm_hash_crypt_done()
354 	aead_request_complete(req, err);  in gcm_hash_crypt_done()
357 static int gcm_hash_assoc_remain_continue(struct aead_request *req, u32 flags)  in gcm_hash_assoc_remain_continue()  argument
359 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in gcm_hash_assoc_remain_continue()
363 		return gcm_hash_update(req, gcm_hash_crypt_done,  in gcm_hash_assoc_remain_continue()
365 		       gcm_hash_crypt_continue(req, flags);  in gcm_hash_assoc_remain_continue()
367 	return gcm_hash_crypt_remain_continue(req, flags);  in gcm_hash_assoc_remain_continue()
373 	struct aead_request *req = areq->data;  in gcm_hash_assoc_remain_done()  local
378 	err = gcm_hash_assoc_remain_continue(req, 0);  in gcm_hash_assoc_remain_done()
383 	aead_request_complete(req, err);  in gcm_hash_assoc_remain_done()
386 static int gcm_hash_assoc_continue(struct aead_request *req, u32 flags)  in gcm_hash_assoc_continue()  argument
390 	remain = gcm_remain(req->assoclen);  in gcm_hash_assoc_continue()
392 		return gcm_hash_remain(req, remain,  in gcm_hash_assoc_continue()
394 		       gcm_hash_assoc_remain_continue(req, flags);  in gcm_hash_assoc_continue()
396 	return gcm_hash_assoc_remain_continue(req, flags);  in gcm_hash_assoc_continue()
401 	struct aead_request *req = areq->data;  in gcm_hash_assoc_done()  local
406 	err = gcm_hash_assoc_continue(req, 0);  in gcm_hash_assoc_done()
411 	aead_request_complete(req, err);  in gcm_hash_assoc_done()
414 static int gcm_hash_init_continue(struct aead_request *req, u32 flags)  in gcm_hash_init_continue()  argument
416 	if (req->assoclen)  in gcm_hash_init_continue()
417 		return gcm_hash_update(req, gcm_hash_assoc_done,  in gcm_hash_init_continue()
418 				       req->src, req->assoclen, flags) ?:  in gcm_hash_init_continue()
419 		       gcm_hash_assoc_continue(req, flags);  in gcm_hash_init_continue()
421 	return gcm_hash_assoc_remain_continue(req, flags);  in gcm_hash_init_continue()
426 	struct aead_request *req = areq->data;  in gcm_hash_init_done()  local
431 	err = gcm_hash_init_continue(req, 0);  in gcm_hash_init_done()
436 	aead_request_complete(req, err);  in gcm_hash_init_done()
439 static int gcm_hash(struct aead_request *req, u32 flags)  in gcm_hash()  argument
441 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in gcm_hash()
443 	struct crypto_gcm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));  in gcm_hash()
447 	ahash_request_set_callback(ahreq, flags, gcm_hash_init_done, req);  in gcm_hash()
449 	       gcm_hash_init_continue(req, flags);  in gcm_hash()
452 static int gcm_enc_copy_hash(struct aead_request *req, u32 flags)  in gcm_enc_copy_hash()  argument
454 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in gcm_enc_copy_hash()
455 	struct crypto_aead *aead = crypto_aead_reqtfm(req);  in gcm_enc_copy_hash()
459 	scatterwalk_map_and_copy(auth_tag, req->dst,  in gcm_enc_copy_hash()
460 				 req->assoclen + req->cryptlen,  in gcm_enc_copy_hash()
465 static int gcm_encrypt_continue(struct aead_request *req, u32 flags)  in gcm_encrypt_continue()  argument
467 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in gcm_encrypt_continue()
470 	gctx->src = sg_next(req->src == req->dst ? pctx->src : pctx->dst);  in gcm_encrypt_continue()
471 	gctx->cryptlen = req->cryptlen;  in gcm_encrypt_continue()
474 	return gcm_hash(req, flags);  in gcm_encrypt_continue()
479 	struct aead_request *req = areq->data;  in gcm_encrypt_done()  local
484 	err = gcm_encrypt_continue(req, 0);  in gcm_encrypt_done()
489 	aead_request_complete(req, err);  in gcm_encrypt_done()
492 static int crypto_gcm_encrypt(struct aead_request *req)  in crypto_gcm_encrypt()  argument
494 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in crypto_gcm_encrypt()
496 	u32 flags = aead_request_flags(req);  in crypto_gcm_encrypt()
498 	crypto_gcm_init_common(req);  in crypto_gcm_encrypt()
499 	crypto_gcm_init_crypt(req, req->cryptlen);  in crypto_gcm_encrypt()
500 	skcipher_request_set_callback(skreq, flags, gcm_encrypt_done, req);  in crypto_gcm_encrypt()
503 	       gcm_encrypt_continue(req, flags);  in crypto_gcm_encrypt()
506 static int crypto_gcm_verify(struct aead_request *req)  in crypto_gcm_verify()  argument
508 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in crypto_gcm_verify()
509 	struct crypto_aead *aead = crypto_aead_reqtfm(req);  in crypto_gcm_verify()
513 	unsigned int cryptlen = req->cryptlen - authsize;  in crypto_gcm_verify()
516 	scatterwalk_map_and_copy(iauth_tag, req->src,  in crypto_gcm_verify()
517 				 req->assoclen + cryptlen, authsize, 0);  in crypto_gcm_verify()
523 	struct aead_request *req = areq->data;  in gcm_decrypt_done()  local
526 		err = crypto_gcm_verify(req);  in gcm_decrypt_done()
528 	aead_request_complete(req, err);  in gcm_decrypt_done()
531 static int gcm_dec_hash_continue(struct aead_request *req, u32 flags)  in gcm_dec_hash_continue()  argument
533 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in gcm_dec_hash_continue()
537 	crypto_gcm_init_crypt(req, gctx->cryptlen);  in gcm_dec_hash_continue()
538 	skcipher_request_set_callback(skreq, flags, gcm_decrypt_done, req);  in gcm_dec_hash_continue()
539 	return crypto_skcipher_decrypt(skreq) ?: crypto_gcm_verify(req);  in gcm_dec_hash_continue()
542 static int crypto_gcm_decrypt(struct aead_request *req)  in crypto_gcm_decrypt()  argument
544 	struct crypto_aead *aead = crypto_aead_reqtfm(req);  in crypto_gcm_decrypt()
545 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);  in crypto_gcm_decrypt()
548 	unsigned int cryptlen = req->cryptlen;  in crypto_gcm_decrypt()
549 	u32 flags = aead_request_flags(req);  in crypto_gcm_decrypt()
553 	crypto_gcm_init_common(req);  in crypto_gcm_decrypt()
559 	return gcm_hash(req, flags);  in crypto_gcm_decrypt()
824 static struct aead_request *crypto_rfc4106_crypt(struct aead_request *req)  in crypto_rfc4106_crypt()  argument
826 	struct crypto_rfc4106_req_ctx *rctx = aead_request_ctx(req);  in crypto_rfc4106_crypt()
827 	struct crypto_aead *aead = crypto_aead_reqtfm(req);  in crypto_rfc4106_crypt()
835 	scatterwalk_map_and_copy(iv + 12, req->src, 0, req->assoclen - 8, 0);  in crypto_rfc4106_crypt()
838 	memcpy(iv + 4, req->iv, 8);  in crypto_rfc4106_crypt()
841 	sg_set_buf(rctx->src, iv + 12, req->assoclen - 8);  in crypto_rfc4106_crypt()
842 	sg = scatterwalk_ffwd(rctx->src + 1, req->src, req->assoclen);  in crypto_rfc4106_crypt()
846 	if (req->src != req->dst) {  in crypto_rfc4106_crypt()
848 		sg_set_buf(rctx->dst, iv + 12, req->assoclen - 8);  in crypto_rfc4106_crypt()
849 		sg = scatterwalk_ffwd(rctx->dst + 1, req->dst, req->assoclen);  in crypto_rfc4106_crypt()
855 	aead_request_set_callback(subreq, req->base.flags, req->base.complete,  in crypto_rfc4106_crypt()
856 				  req->base.data);  in crypto_rfc4106_crypt()
858 			       req->src == req->dst ? rctx->src : rctx->dst,  in crypto_rfc4106_crypt()
859 			       req->cryptlen, iv);  in crypto_rfc4106_crypt()
860 	aead_request_set_ad(subreq, req->assoclen - 8);  in crypto_rfc4106_crypt()
865 static int crypto_rfc4106_encrypt(struct aead_request *req)  in crypto_rfc4106_encrypt()  argument
867 	if (req->assoclen != 16 && req->assoclen != 20)  in crypto_rfc4106_encrypt()
870 	req = crypto_rfc4106_crypt(req);  in crypto_rfc4106_encrypt()
872 	return crypto_aead_encrypt(req);  in crypto_rfc4106_encrypt()
875 static int crypto_rfc4106_decrypt(struct aead_request *req)  in crypto_rfc4106_decrypt()  argument
877 	if (req->assoclen != 16 && req->assoclen != 20)  in crypto_rfc4106_decrypt()
880 	req = crypto_rfc4106_crypt(req);  in crypto_rfc4106_decrypt()
882 	return crypto_aead_decrypt(req);  in crypto_rfc4106_decrypt()
1051 static int crypto_rfc4543_crypt(struct aead_request *req, bool enc)  in crypto_rfc4543_crypt()  argument
1053 	struct crypto_aead *aead = crypto_aead_reqtfm(req);  in crypto_rfc4543_crypt()
1055 	struct crypto_rfc4543_req_ctx *rctx = aead_request_ctx(req);  in crypto_rfc4543_crypt()
1062 	if (req->src != req->dst) {  in crypto_rfc4543_crypt()
1063 		err = crypto_rfc4543_copy_src_to_dst(req, enc);  in crypto_rfc4543_crypt()
1069 	memcpy(iv + 4, req->iv, 8);  in crypto_rfc4543_crypt()
1072 	aead_request_set_callback(subreq, req->base.flags,  in crypto_rfc4543_crypt()
1073 				  req->base.complete, req->base.data);  in crypto_rfc4543_crypt()
1074 	aead_request_set_crypt(subreq, req->src, req->dst,  in crypto_rfc4543_crypt()
1076 	aead_request_set_ad(subreq, req->assoclen + req->cryptlen -  in crypto_rfc4543_crypt()
1082 static int crypto_rfc4543_copy_src_to_dst(struct aead_request *req, bool enc)  in crypto_rfc4543_copy_src_to_dst()  argument
1084 	struct crypto_aead *aead = crypto_aead_reqtfm(req);  in crypto_rfc4543_copy_src_to_dst()
1087 	unsigned int nbytes = req->assoclen + req->cryptlen -  in crypto_rfc4543_copy_src_to_dst()
1092 	skcipher_request_set_callback(nreq, req->base.flags, NULL, NULL);  in crypto_rfc4543_copy_src_to_dst()
1093 	skcipher_request_set_crypt(nreq, req->src, req->dst, nbytes, NULL);  in crypto_rfc4543_copy_src_to_dst()
1098 static int crypto_rfc4543_encrypt(struct aead_request *req)  in crypto_rfc4543_encrypt()  argument
1100 	return crypto_rfc4543_crypt(req, true);  in crypto_rfc4543_encrypt()
1103 static int crypto_rfc4543_decrypt(struct aead_request *req)  in crypto_rfc4543_decrypt()  argument
1105 	return crypto_rfc4543_crypt(req, false);  in crypto_rfc4543_decrypt()