27 			    struct ahash_request *req)  in mv_cesa_ahash_req_iter_init()  argument
29 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_req_iter_init()
30 	unsigned int len = req->nbytes + creq->cache_ptr;  in mv_cesa_ahash_req_iter_init()
36 	mv_cesa_sg_dma_iter_init(&iter->src, req->src, DMA_TO_DEVICE);  in mv_cesa_ahash_req_iter_init()
49 mv_cesa_ahash_dma_alloc_cache(struct mv_cesa_ahash_dma_req *req, gfp_t flags)  in mv_cesa_ahash_dma_alloc_cache()  argument
51 	req->cache = dma_pool_alloc(cesa_dev->dma->cache_pool, flags,  in mv_cesa_ahash_dma_alloc_cache()
52 				    &req->cache_dma);  in mv_cesa_ahash_dma_alloc_cache()
53 	if (!req->cache)  in mv_cesa_ahash_dma_alloc_cache()
60 mv_cesa_ahash_dma_free_cache(struct mv_cesa_ahash_dma_req *req)  in mv_cesa_ahash_dma_free_cache()  argument
62 	if (!req->cache)  in mv_cesa_ahash_dma_free_cache()
65 	dma_pool_free(cesa_dev->dma->cache_pool, req->cache,  in mv_cesa_ahash_dma_free_cache()
66 		      req->cache_dma);  in mv_cesa_ahash_dma_free_cache()
69 static int mv_cesa_ahash_dma_alloc_padding(struct mv_cesa_ahash_dma_req *req,  in mv_cesa_ahash_dma_alloc_padding()  argument
72 	if (req->padding)  in mv_cesa_ahash_dma_alloc_padding()
75 	req->padding = dma_pool_alloc(cesa_dev->dma->padding_pool, flags,  in mv_cesa_ahash_dma_alloc_padding()
76 				      &req->padding_dma);  in mv_cesa_ahash_dma_alloc_padding()
77 	if (!req->padding)  in mv_cesa_ahash_dma_alloc_padding()
83 static void mv_cesa_ahash_dma_free_padding(struct mv_cesa_ahash_dma_req *req)  in mv_cesa_ahash_dma_free_padding()  argument
85 	if (!req->padding)  in mv_cesa_ahash_dma_free_padding()
88 	dma_pool_free(cesa_dev->dma->padding_pool, req->padding,  in mv_cesa_ahash_dma_free_padding()
89 		      req->padding_dma);  in mv_cesa_ahash_dma_free_padding()
90 	req->padding = NULL;  in mv_cesa_ahash_dma_free_padding()
93 static inline void mv_cesa_ahash_dma_last_cleanup(struct ahash_request *req)  in mv_cesa_ahash_dma_last_cleanup()  argument
95 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_dma_last_cleanup()
97 	mv_cesa_ahash_dma_free_padding(&creq->req.dma);  in mv_cesa_ahash_dma_last_cleanup()
100 static inline void mv_cesa_ahash_dma_cleanup(struct ahash_request *req)  in mv_cesa_ahash_dma_cleanup()  argument
102 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_dma_cleanup()
104 	dma_unmap_sg(cesa_dev->dev, req->src, creq->src_nents, DMA_TO_DEVICE);  in mv_cesa_ahash_dma_cleanup()
105 	mv_cesa_ahash_dma_free_cache(&creq->req.dma);  in mv_cesa_ahash_dma_cleanup()
109 static inline void mv_cesa_ahash_cleanup(struct ahash_request *req)  in mv_cesa_ahash_cleanup()  argument
111 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_cleanup()
114 		mv_cesa_ahash_dma_cleanup(req);  in mv_cesa_ahash_cleanup()
117 static void mv_cesa_ahash_last_cleanup(struct ahash_request *req)  in mv_cesa_ahash_last_cleanup()  argument
119 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_last_cleanup()
122 		mv_cesa_ahash_dma_last_cleanup(req);  in mv_cesa_ahash_last_cleanup()
157 static void mv_cesa_ahash_std_step(struct ahash_request *req)  in mv_cesa_ahash_std_step()  argument
159 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_std_step()
160 	struct mv_cesa_ahash_std_req *sreq = &creq->req.std;  in mv_cesa_ahash_std_step()
173 		digsize = crypto_ahash_digestsize(crypto_ahash_reqtfm(req));  in mv_cesa_ahash_std_step()
183 	len = min_t(size_t, req->nbytes + creq->cache_ptr - sreq->offset,  in mv_cesa_ahash_std_step()
192 		sreq->offset += sg_pcopy_to_buffer(req->src, creq->src_nents,  in mv_cesa_ahash_std_step()
203 	if (creq->last_req && sreq->offset == req->nbytes &&  in mv_cesa_ahash_std_step()
260 static int mv_cesa_ahash_std_process(struct ahash_request *req, u32 status)  in mv_cesa_ahash_std_process()  argument
262 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_std_process()
263 	struct mv_cesa_ahash_std_req *sreq = &creq->req.std;  in mv_cesa_ahash_std_process()
265 	if (sreq->offset < (req->nbytes - creq->cache_ptr))  in mv_cesa_ahash_std_process()
271 static inline void mv_cesa_ahash_dma_prepare(struct ahash_request *req)  in mv_cesa_ahash_dma_prepare()  argument
273 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_dma_prepare()
279 static void mv_cesa_ahash_std_prepare(struct ahash_request *req)  in mv_cesa_ahash_std_prepare()  argument
281 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_std_prepare()
282 	struct mv_cesa_ahash_std_req *sreq = &creq->req.std;  in mv_cesa_ahash_std_prepare()
287 static void mv_cesa_ahash_dma_step(struct ahash_request *req)  in mv_cesa_ahash_dma_step()  argument
289 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_dma_step()
306 static void mv_cesa_ahash_step(struct crypto_async_request *req)  in mv_cesa_ahash_step()  argument
308 	struct ahash_request *ahashreq = ahash_request_cast(req);  in mv_cesa_ahash_step()
317 static int mv_cesa_ahash_process(struct crypto_async_request *req, u32 status)  in mv_cesa_ahash_process()  argument
319 	struct ahash_request *ahashreq = ahash_request_cast(req);  in mv_cesa_ahash_process()
328 static void mv_cesa_ahash_complete(struct crypto_async_request *req)  in mv_cesa_ahash_complete()  argument
330 	struct ahash_request *ahashreq = ahash_request_cast(req);  in mv_cesa_ahash_complete()
378 static void mv_cesa_ahash_prepare(struct crypto_async_request *req,  in mv_cesa_ahash_prepare()  argument
381 	struct ahash_request *ahashreq = ahash_request_cast(req);  in mv_cesa_ahash_prepare()
392 static void mv_cesa_ahash_req_cleanup(struct crypto_async_request *req)  in mv_cesa_ahash_req_cleanup()  argument
394 	struct ahash_request *ahashreq = ahash_request_cast(req);  in mv_cesa_ahash_req_cleanup()
416 static void mv_cesa_ahash_init(struct ahash_request *req,  in mv_cesa_ahash_init()  argument
419 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_init()
445 static bool mv_cesa_ahash_cache_req(struct ahash_request *req)  in mv_cesa_ahash_cache_req()  argument
447 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_cache_req()
450 	if (creq->cache_ptr + req->nbytes < CESA_MAX_HASH_BLOCK_SIZE &&  in mv_cesa_ahash_cache_req()
454 		if (!req->nbytes)  in mv_cesa_ahash_cache_req()
457 		sg_pcopy_to_buffer(req->src, creq->src_nents,  in mv_cesa_ahash_cache_req()
459 				   req->nbytes, 0);  in mv_cesa_ahash_cache_req()
461 		creq->cache_ptr += req->nbytes;  in mv_cesa_ahash_cache_req()
500 	struct mv_cesa_ahash_dma_req *ahashdreq = &creq->req.dma;  in mv_cesa_ahash_dma_add_cache()
526 	struct mv_cesa_ahash_dma_req *ahashdreq = &creq->req.dma;  in mv_cesa_ahash_dma_last_req()
603 static int mv_cesa_ahash_dma_req_init(struct ahash_request *req)  in mv_cesa_ahash_dma_req_init()  argument
605 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_dma_req_init()
606 	gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?  in mv_cesa_ahash_dma_req_init()
623 		ret = dma_map_sg(cesa_dev->dev, req->src, creq->src_nents,  in mv_cesa_ahash_dma_req_init()
632 	mv_cesa_ahash_req_iter_init(&iter, req);  in mv_cesa_ahash_dma_req_init()
706 		creq->cache_ptr = req->nbytes + creq->cache_ptr -  in mv_cesa_ahash_dma_req_init()
729 	dma_unmap_sg(cesa_dev->dev, req->src, creq->src_nents, DMA_TO_DEVICE);  in mv_cesa_ahash_dma_req_init()
732 	mv_cesa_ahash_last_cleanup(req);  in mv_cesa_ahash_dma_req_init()
737 static int mv_cesa_ahash_req_init(struct ahash_request *req, bool *cached)  in mv_cesa_ahash_req_init()  argument
739 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_req_init()
741 	creq->src_nents = sg_nents_for_len(req->src, req->nbytes);  in mv_cesa_ahash_req_init()
747 	*cached = mv_cesa_ahash_cache_req(req);  in mv_cesa_ahash_req_init()
753 		return mv_cesa_ahash_dma_req_init(req);  in mv_cesa_ahash_req_init()
758 static int mv_cesa_ahash_queue_req(struct ahash_request *req)  in mv_cesa_ahash_queue_req()  argument
760 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_queue_req()
765 	ret = mv_cesa_ahash_req_init(req, &cached);  in mv_cesa_ahash_queue_req()
772 	engine = mv_cesa_select_engine(req->nbytes);  in mv_cesa_ahash_queue_req()
773 	mv_cesa_ahash_prepare(&req->base, engine);  in mv_cesa_ahash_queue_req()
775 	ret = mv_cesa_queue_req(&req->base, &creq->base);  in mv_cesa_ahash_queue_req()
777 	if (mv_cesa_req_needs_cleanup(&req->base, ret))  in mv_cesa_ahash_queue_req()
778 		mv_cesa_ahash_cleanup(req);  in mv_cesa_ahash_queue_req()
783 static int mv_cesa_ahash_update(struct ahash_request *req)  in mv_cesa_ahash_update()  argument
785 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_update()
787 	creq->len += req->nbytes;  in mv_cesa_ahash_update()
789 	return mv_cesa_ahash_queue_req(req);  in mv_cesa_ahash_update()
792 static int mv_cesa_ahash_final(struct ahash_request *req)  in mv_cesa_ahash_final()  argument
794 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_final()
799 	req->nbytes = 0;  in mv_cesa_ahash_final()
801 	return mv_cesa_ahash_queue_req(req);  in mv_cesa_ahash_final()
804 static int mv_cesa_ahash_finup(struct ahash_request *req)  in mv_cesa_ahash_finup()  argument
806 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_finup()
809 	creq->len += req->nbytes;  in mv_cesa_ahash_finup()
813 	return mv_cesa_ahash_queue_req(req);  in mv_cesa_ahash_finup()
816 static int mv_cesa_ahash_export(struct ahash_request *req, void *hash,  in mv_cesa_ahash_export()  argument
819 	struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);  in mv_cesa_ahash_export()
820 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_export()
834 static int mv_cesa_ahash_import(struct ahash_request *req, const void *hash,  in mv_cesa_ahash_import()  argument
837 	struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);  in mv_cesa_ahash_import()
838 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_ahash_import()
844 	ret = crypto_ahash_init(req);  in mv_cesa_ahash_import()
868 static int mv_cesa_md5_init(struct ahash_request *req)  in mv_cesa_md5_init()  argument
870 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_md5_init()
875 	mv_cesa_ahash_init(req, &tmpl, true);  in mv_cesa_md5_init()
885 static int mv_cesa_md5_export(struct ahash_request *req, void *out)  in mv_cesa_md5_export()  argument
889 	return mv_cesa_ahash_export(req, out_state->hash,  in mv_cesa_md5_export()
893 static int mv_cesa_md5_import(struct ahash_request *req, const void *in)  in mv_cesa_md5_import()  argument
897 	return mv_cesa_ahash_import(req, in_state->hash, in_state->byte_count,  in mv_cesa_md5_import()
901 static int mv_cesa_md5_digest(struct ahash_request *req)  in mv_cesa_md5_digest()  argument
905 	ret = mv_cesa_md5_init(req);  in mv_cesa_md5_digest()
909 	return mv_cesa_ahash_finup(req);  in mv_cesa_md5_digest()
938 static int mv_cesa_sha1_init(struct ahash_request *req)  in mv_cesa_sha1_init()  argument
940 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_sha1_init()
945 	mv_cesa_ahash_init(req, &tmpl, false);  in mv_cesa_sha1_init()
956 static int mv_cesa_sha1_export(struct ahash_request *req, void *out)  in mv_cesa_sha1_export()  argument
960 	return mv_cesa_ahash_export(req, out_state->state, &out_state->count,  in mv_cesa_sha1_export()
964 static int mv_cesa_sha1_import(struct ahash_request *req, const void *in)  in mv_cesa_sha1_import()  argument
968 	return mv_cesa_ahash_import(req, in_state->state, in_state->count,  in mv_cesa_sha1_import()
972 static int mv_cesa_sha1_digest(struct ahash_request *req)  in mv_cesa_sha1_digest()  argument
976 	ret = mv_cesa_sha1_init(req);  in mv_cesa_sha1_digest()
980 	return mv_cesa_ahash_finup(req);  in mv_cesa_sha1_digest()
1009 static int mv_cesa_sha256_init(struct ahash_request *req)  in mv_cesa_sha256_init()  argument
1011 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);  in mv_cesa_sha256_init()
1016 	mv_cesa_ahash_init(req, &tmpl, false);  in mv_cesa_sha256_init()
1030 static int mv_cesa_sha256_digest(struct ahash_request *req)  in mv_cesa_sha256_digest()  argument
1034 	ret = mv_cesa_sha256_init(req);  in mv_cesa_sha256_digest()
1038 	return mv_cesa_ahash_finup(req);  in mv_cesa_sha256_digest()
1041 static int mv_cesa_sha256_export(struct ahash_request *req, void *out)  in mv_cesa_sha256_export()  argument
1045 	return mv_cesa_ahash_export(req, out_state->state, &out_state->count,  in mv_cesa_sha256_export()
1049 static int mv_cesa_sha256_import(struct ahash_request *req, const void *in)  in mv_cesa_sha256_import()  argument
1053 	return mv_cesa_ahash_import(req, in_state->state, in_state->count,  in mv_cesa_sha256_import()
1088 static void mv_cesa_hmac_ahash_complete(struct crypto_async_request *req,  in mv_cesa_hmac_ahash_complete()  argument
1091 	struct mv_cesa_ahash_result *result = req->data;  in mv_cesa_hmac_ahash_complete()
1100 static int mv_cesa_ahmac_iv_state_init(struct ahash_request *req, u8 *pad,  in mv_cesa_ahmac_iv_state_init()  argument
1107 	ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,  in mv_cesa_ahmac_iv_state_init()
1110 	ahash_request_set_crypt(req, &sg, pad, blocksize);  in mv_cesa_ahmac_iv_state_init()
1113 	ret = crypto_ahash_init(req);  in mv_cesa_ahmac_iv_state_init()
1117 	ret = crypto_ahash_update(req);  in mv_cesa_ahmac_iv_state_init()
1125 	ret = crypto_ahash_export(req, state);  in mv_cesa_ahmac_iv_state_init()
1132 static int mv_cesa_ahmac_pad_init(struct ahash_request *req,  in mv_cesa_ahmac_pad_init()  argument
1150 		ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,  in mv_cesa_ahmac_pad_init()
1154 		ahash_request_set_crypt(req, &sg, ipad, keylen);  in mv_cesa_ahmac_pad_init()
1157 		ret = crypto_ahash_digest(req);  in mv_cesa_ahmac_pad_init()
1169 		keylen = crypto_ahash_digestsize(crypto_ahash_reqtfm(req));  in mv_cesa_ahmac_pad_init()
1187 	struct ahash_request *req;  in mv_cesa_ahmac_setkey()  local
1198 	req = ahash_request_alloc(tfm, GFP_KERNEL);  in mv_cesa_ahmac_setkey()
1199 	if (!req) {  in mv_cesa_ahmac_setkey()
1216 	ret = mv_cesa_ahmac_pad_init(req, key, keylen, ipad, opad, blocksize);  in mv_cesa_ahmac_setkey()
1220 	ret = mv_cesa_ahmac_iv_state_init(req, ipad, istate, blocksize);  in mv_cesa_ahmac_setkey()
1224 	ret = mv_cesa_ahmac_iv_state_init(req, opad, ostate, blocksize);  in mv_cesa_ahmac_setkey()
1229 	ahash_request_free(req);  in mv_cesa_ahmac_setkey()
1247 static int mv_cesa_ahmac_md5_init(struct ahash_request *req)  in mv_cesa_ahmac_md5_init()  argument
1249 	struct mv_cesa_hmac_ctx *ctx = crypto_tfm_ctx(req->base.tfm);  in mv_cesa_ahmac_md5_init()
1255 	mv_cesa_ahash_init(req, &tmpl, true);  in mv_cesa_ahmac_md5_init()
1280 static int mv_cesa_ahmac_md5_digest(struct ahash_request *req)  in mv_cesa_ahmac_md5_digest()  argument
1284 	ret = mv_cesa_ahmac_md5_init(req);  in mv_cesa_ahmac_md5_digest()
1288 	return mv_cesa_ahash_finup(req);  in mv_cesa_ahmac_md5_digest()
1318 static int mv_cesa_ahmac_sha1_init(struct ahash_request *req)  in mv_cesa_ahmac_sha1_init()  argument
1320 	struct mv_cesa_hmac_ctx *ctx = crypto_tfm_ctx(req->base.tfm);  in mv_cesa_ahmac_sha1_init()
1326 	mv_cesa_ahash_init(req, &tmpl, false);  in mv_cesa_ahmac_sha1_init()
1351 static int mv_cesa_ahmac_sha1_digest(struct ahash_request *req)  in mv_cesa_ahmac_sha1_digest()  argument
1355 	ret = mv_cesa_ahmac_sha1_init(req);  in mv_cesa_ahmac_sha1_digest()
1359 	return mv_cesa_ahash_finup(req);  in mv_cesa_ahmac_sha1_digest()
1409 static int mv_cesa_ahmac_sha256_init(struct ahash_request *req)  in mv_cesa_ahmac_sha256_init()  argument
1411 	struct mv_cesa_hmac_ctx *ctx = crypto_tfm_ctx(req->base.tfm);  in mv_cesa_ahmac_sha256_init()
1417 	mv_cesa_ahash_init(req, &tmpl, false);  in mv_cesa_ahmac_sha256_init()
1422 static int mv_cesa_ahmac_sha256_digest(struct ahash_request *req)  in mv_cesa_ahmac_sha256_digest()  argument
1426 	ret = mv_cesa_ahmac_sha256_init(req);  in mv_cesa_ahmac_sha256_digest()
1430 	return mv_cesa_ahash_finup(req);  in mv_cesa_ahmac_sha256_digest()