1 // SPDX-License-Identifier: GPL-2.0-or-later
3  * sun4i-ss-cipher.c - hardware cryptographic accelerator for Allwinner A20 SoC
5  * Copyright (C) 2013-2015 Corentin LABBE <clabbe.montjoie@gmail.com>
8  * keysize in CBC and ECB mode.
9  * Add support also for DES and 3DES in CBC and ECB mode.
13 #include "sun4i-ss.h"
18 	struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);  in sun4i_ss_opti_poll()  local
19 	struct sun4i_ss_ctx *ss = op->ss;  in sun4i_ss_opti_poll()
22 	u32 mode = ctx->mode;  in sun4i_ss_opti_poll()  local
31 	unsigned int ileft = areq->cryptlen;  in sun4i_ss_opti_poll()
32 	unsigned int oleft = areq->cryptlen;  in sun4i_ss_opti_poll()
40 	if (!areq->cryptlen)  in sun4i_ss_opti_poll()
43 	if (!areq->src || !areq->dst) {  in sun4i_ss_opti_poll()
44 		dev_err_ratelimited(ss->dev, "ERROR: Some SGs are NULL\n");  in sun4i_ss_opti_poll()
45 		return -EINVAL;  in sun4i_ss_opti_poll()
48 	if (areq->iv && ivsize > 0 && mode & SS_DECRYPTION) {  in sun4i_ss_opti_poll()
51 			return -ENOMEM;  in sun4i_ss_opti_poll()
52 		scatterwalk_map_and_copy(backup_iv, areq->src, areq->cryptlen - ivsize, ivsize, 0);  in sun4i_ss_opti_poll()
55 	spin_lock_irqsave(&ss->slock, flags);  in sun4i_ss_opti_poll()
57 	for (i = 0; i < op->keylen / 4; i++)  in sun4i_ss_opti_poll()
58 		writesl(ss->base + SS_KEY0 + i * 4, &op->key[i], 1);  in sun4i_ss_opti_poll()
60 	if (areq->iv) {  in sun4i_ss_opti_poll()
62 			v = *(u32 *)(areq->iv + i * 4);  in sun4i_ss_opti_poll()
63 			writesl(ss->base + SS_IV0 + i * 4, &v, 1);  in sun4i_ss_opti_poll()
66 	writel(mode, ss->base + SS_CTL);  in sun4i_ss_opti_poll()
69 	ileft = areq->cryptlen / 4;  in sun4i_ss_opti_poll()
70 	oleft = areq->cryptlen / 4;  in sun4i_ss_opti_poll()
75 			sg_miter_start(&mi, areq->src, sg_nents(areq->src),  in sun4i_ss_opti_poll()
81 				dev_err_ratelimited(ss->dev, "ERROR: sg_miter return null\n");  in sun4i_ss_opti_poll()
82 				err = -EINVAL;  in sun4i_ss_opti_poll()
86 			todo = min_t(size_t, todo, (mi.length - oi) / 4);  in sun4i_ss_opti_poll()
88 				ileft -= todo;  in sun4i_ss_opti_poll()
89 				writesl(ss->base + SS_RXFIFO, mi.addr + oi, todo);  in sun4i_ss_opti_poll()
99 		spaces = readl(ss->base + SS_FCSR);  in sun4i_ss_opti_poll()
103 		sg_miter_start(&mo, areq->dst, sg_nents(areq->dst),  in sun4i_ss_opti_poll()
109 			dev_err_ratelimited(ss->dev, "ERROR: sg_miter return null\n");  in sun4i_ss_opti_poll()
110 			err = -EINVAL;  in sun4i_ss_opti_poll()
114 		todo = min_t(size_t, todo, (mo.length - oo) / 4);  in sun4i_ss_opti_poll()
116 			oleft -= todo;  in sun4i_ss_opti_poll()
117 			readsl(ss->base + SS_TXFIFO, mo.addr + oo, todo);  in sun4i_ss_opti_poll()
127 	if (areq->iv) {  in sun4i_ss_opti_poll()
128 		if (mode & SS_DECRYPTION) {  in sun4i_ss_opti_poll()
129 			memcpy(areq->iv, backup_iv, ivsize);  in sun4i_ss_opti_poll()
132 			scatterwalk_map_and_copy(areq->iv, areq->dst, areq->cryptlen - ivsize,  in sun4i_ss_opti_poll()
138 	writel(0, ss->base + SS_CTL);  in sun4i_ss_opti_poll()
139 	spin_unlock_irqrestore(&ss->slock, flags);  in sun4i_ss_opti_poll()
147 	struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);  in sun4i_ss_cipher_poll_fallback()  local
151 	skcipher_request_set_tfm(&ctx->fallback_req, op->fallback_tfm);  in sun4i_ss_cipher_poll_fallback()
152 	skcipher_request_set_callback(&ctx->fallback_req, areq->base.flags,  in sun4i_ss_cipher_poll_fallback()
153 				      areq->base.complete, areq->base.data);  in sun4i_ss_cipher_poll_fallback()
154 	skcipher_request_set_crypt(&ctx->fallback_req, areq->src, areq->dst,  in sun4i_ss_cipher_poll_fallback()
155 				   areq->cryptlen, areq->iv);  in sun4i_ss_cipher_poll_fallback()
156 	if (ctx->mode & SS_DECRYPTION)  in sun4i_ss_cipher_poll_fallback()
157 		err = crypto_skcipher_decrypt(&ctx->fallback_req);  in sun4i_ss_cipher_poll_fallback()
159 		err = crypto_skcipher_encrypt(&ctx->fallback_req);  in sun4i_ss_cipher_poll_fallback()
168 	struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);  in sun4i_ss_cipher_poll()  local
169 	struct sun4i_ss_ctx *ss = op->ss;  in sun4i_ss_cipher_poll()
171 	struct scatterlist *in_sg = areq->src;  in sun4i_ss_cipher_poll()
172 	struct scatterlist *out_sg = areq->dst;  in sun4i_ss_cipher_poll()
177 	u32 mode = ctx->mode;  in sun4i_ss_cipher_poll()  local
185 	unsigned int ileft = areq->cryptlen;  in sun4i_ss_cipher_poll()
186 	unsigned int oleft = areq->cryptlen;  in sun4i_ss_cipher_poll()
199 	if (!areq->cryptlen)  in sun4i_ss_cipher_poll()
202 	if (!areq->src || !areq->dst) {  in sun4i_ss_cipher_poll()
203 		dev_err_ratelimited(ss->dev, "ERROR: Some SGs are NULL\n");  in sun4i_ss_cipher_poll()
204 		return -EINVAL;  in sun4i_ss_cipher_poll()
208 	if (areq->cryptlen % algt->alg.crypto.base.cra_blocksize)  in sun4i_ss_cipher_poll()
216 		if ((in_sg->length | in_sg->offset) & 3u)  in sun4i_ss_cipher_poll()
221 		if ((out_sg->length | out_sg->offset) & 3u)  in sun4i_ss_cipher_poll()
232 	if (areq->iv && ivsize > 0 && mode & SS_DECRYPTION) {  in sun4i_ss_cipher_poll()
235 			return -ENOMEM;  in sun4i_ss_cipher_poll()
236 		scatterwalk_map_and_copy(backup_iv, areq->src, areq->cryptlen - ivsize, ivsize, 0);  in sun4i_ss_cipher_poll()
239 	spin_lock_irqsave(&ss->slock, flags);  in sun4i_ss_cipher_poll()
241 	for (i = 0; i < op->keylen / 4; i++)  in sun4i_ss_cipher_poll()
242 		writesl(ss->base + SS_KEY0 + i * 4, &op->key[i], 1);  in sun4i_ss_cipher_poll()
244 	if (areq->iv) {  in sun4i_ss_cipher_poll()
246 			v = *(u32 *)(areq->iv + i * 4);  in sun4i_ss_cipher_poll()
247 			writesl(ss->base + SS_IV0 + i * 4, &v, 1);  in sun4i_ss_cipher_poll()
250 	writel(mode, ss->base + SS_CTL);  in sun4i_ss_cipher_poll()
252 	ileft = areq->cryptlen;  in sun4i_ss_cipher_poll()
253 	oleft = areq->cryptlen;  in sun4i_ss_cipher_poll()
259 			sg_miter_start(&mi, areq->src, sg_nents(areq->src),  in sun4i_ss_cipher_poll()
265 				dev_err_ratelimited(ss->dev, "ERROR: sg_miter return null\n");  in sun4i_ss_cipher_poll()
266 				err = -EINVAL;  in sun4i_ss_cipher_poll()
274 			todo = min_t(size_t, todo, (mi.length - oi) / 4);  in sun4i_ss_cipher_poll()
276 				writesl(ss->base + SS_RXFIFO, mi.addr + oi,  in sun4i_ss_cipher_poll()
278 				ileft -= todo * 4;  in sun4i_ss_cipher_poll()
288 				todo = min(rx_cnt * 4 - ob, ileft);  in sun4i_ss_cipher_poll()
289 				todo = min_t(size_t, todo, mi.length - oi);  in sun4i_ss_cipher_poll()
290 				memcpy(ss->buf + ob, mi.addr + oi, todo);  in sun4i_ss_cipher_poll()
291 				ileft -= todo;  in sun4i_ss_cipher_poll()
295 					writesl(ss->base + SS_RXFIFO, ss->buf,  in sun4i_ss_cipher_poll()
307 		spaces = readl(ss->base + SS_FCSR);  in sun4i_ss_cipher_poll()
313 		sg_miter_start(&mo, areq->dst, sg_nents(areq->dst),  in sun4i_ss_cipher_poll()
319 			dev_err_ratelimited(ss->dev, "ERROR: sg_miter return null\n");  in sun4i_ss_cipher_poll()
320 			err = -EINVAL;  in sun4i_ss_cipher_poll()
325 		todo = min_t(size_t, todo, (mo.length - oo) / 4);  in sun4i_ss_cipher_poll()
328 			readsl(ss->base + SS_TXFIFO, mo.addr + oo, todo);  in sun4i_ss_cipher_poll()
329 			oleft -= todo * 4;  in sun4i_ss_cipher_poll()
340 			readsl(ss->base + SS_TXFIFO, ss->bufo, tx_cnt);  in sun4i_ss_cipher_poll()
351 					     mo.length - oo, obl - obo);  in sun4i_ss_cipher_poll()
352 				memcpy(mo.addr + oo, ss->bufo + obo, todo);  in sun4i_ss_cipher_poll()
353 				oleft -= todo;  in sun4i_ss_cipher_poll()
366 	if (areq->iv) {  in sun4i_ss_cipher_poll()
367 		if (mode & SS_DECRYPTION) {  in sun4i_ss_cipher_poll()
368 			memcpy(areq->iv, backup_iv, ivsize);  in sun4i_ss_cipher_poll()
371 			scatterwalk_map_and_copy(areq->iv, areq->dst, areq->cryptlen - ivsize,  in sun4i_ss_cipher_poll()
377 	writel(0, ss->base + SS_CTL);  in sun4i_ss_cipher_poll()
378 	spin_unlock_irqrestore(&ss->slock, flags);  in sun4i_ss_cipher_poll()
387 	struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);  in sun4i_ss_cbc_aes_encrypt()  local
390 	rctx->mode = SS_OP_AES | SS_CBC | SS_ENABLED | SS_ENCRYPTION |  in sun4i_ss_cbc_aes_encrypt()
391 		op->keymode;  in sun4i_ss_cbc_aes_encrypt()
398 	struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);  in sun4i_ss_cbc_aes_decrypt()  local
401 	rctx->mode = SS_OP_AES | SS_CBC | SS_ENABLED | SS_DECRYPTION |  in sun4i_ss_cbc_aes_decrypt()
402 		op->keymode;  in sun4i_ss_cbc_aes_decrypt()
410 	struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);  in sun4i_ss_ecb_aes_encrypt()  local
413 	rctx->mode = SS_OP_AES | SS_ECB | SS_ENABLED | SS_ENCRYPTION |  in sun4i_ss_ecb_aes_encrypt()
414 		op->keymode;  in sun4i_ss_ecb_aes_encrypt()
421 	struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);  in sun4i_ss_ecb_aes_decrypt()  local
424 	rctx->mode = SS_OP_AES | SS_ECB | SS_ENABLED | SS_DECRYPTION |  in sun4i_ss_ecb_aes_decrypt()
425 		op->keymode;  in sun4i_ss_ecb_aes_decrypt()
433 	struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);  in sun4i_ss_cbc_des_encrypt()  local
436 	rctx->mode = SS_OP_DES | SS_CBC | SS_ENABLED | SS_ENCRYPTION |  in sun4i_ss_cbc_des_encrypt()
437 		op->keymode;  in sun4i_ss_cbc_des_encrypt()
444 	struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);  in sun4i_ss_cbc_des_decrypt()  local
447 	rctx->mode = SS_OP_DES | SS_CBC | SS_ENABLED | SS_DECRYPTION |  in sun4i_ss_cbc_des_decrypt()
448 		op->keymode;  in sun4i_ss_cbc_des_decrypt()
456 	struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);  in sun4i_ss_ecb_des_encrypt()  local
459 	rctx->mode = SS_OP_DES | SS_ECB | SS_ENABLED | SS_ENCRYPTION |  in sun4i_ss_ecb_des_encrypt()
460 		op->keymode;  in sun4i_ss_ecb_des_encrypt()
467 	struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);  in sun4i_ss_ecb_des_decrypt()  local
470 	rctx->mode = SS_OP_DES | SS_ECB | SS_ENABLED | SS_DECRYPTION |  in sun4i_ss_ecb_des_decrypt()
471 		op->keymode;  in sun4i_ss_ecb_des_decrypt()
479 	struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);  in sun4i_ss_cbc_des3_encrypt()  local
482 	rctx->mode = SS_OP_3DES | SS_CBC | SS_ENABLED | SS_ENCRYPTION |  in sun4i_ss_cbc_des3_encrypt()
483 		op->keymode;  in sun4i_ss_cbc_des3_encrypt()
490 	struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);  in sun4i_ss_cbc_des3_decrypt()  local
493 	rctx->mode = SS_OP_3DES | SS_CBC | SS_ENABLED | SS_DECRYPTION |  in sun4i_ss_cbc_des3_decrypt()
494 		op->keymode;  in sun4i_ss_cbc_des3_decrypt()
502 	struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);  in sun4i_ss_ecb_des3_encrypt()  local
505 	rctx->mode = SS_OP_3DES | SS_ECB | SS_ENABLED | SS_ENCRYPTION |  in sun4i_ss_ecb_des3_encrypt()
506 		op->keymode;  in sun4i_ss_ecb_des3_encrypt()
513 	struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);  in sun4i_ss_ecb_des3_decrypt()  local
516 	rctx->mode = SS_OP_3DES | SS_ECB | SS_ENABLED | SS_DECRYPTION |  in sun4i_ss_ecb_des3_decrypt()
517 		op->keymode;  in sun4i_ss_ecb_des3_decrypt()
523 	struct sun4i_tfm_ctx *op = crypto_tfm_ctx(tfm);  in sun4i_ss_cipher_init()  local
528 	memset(op, 0, sizeof(struct sun4i_tfm_ctx));  in sun4i_ss_cipher_init()
530 	algt = container_of(tfm->__crt_alg, struct sun4i_ss_alg_template,  in sun4i_ss_cipher_init()
532 	op->ss = algt->ss;  in sun4i_ss_cipher_init()
534 	op->fallback_tfm = crypto_alloc_skcipher(name, 0, CRYPTO_ALG_NEED_FALLBACK);  in sun4i_ss_cipher_init()
535 	if (IS_ERR(op->fallback_tfm)) {  in sun4i_ss_cipher_init()
536 		dev_err(op->ss->dev, "ERROR: Cannot allocate fallback for %s %ld\n",  in sun4i_ss_cipher_init()
537 			name, PTR_ERR(op->fallback_tfm));  in sun4i_ss_cipher_init()
538 		return PTR_ERR(op->fallback_tfm);  in sun4i_ss_cipher_init()
543 				    crypto_skcipher_reqsize(op->fallback_tfm));  in sun4i_ss_cipher_init()
546 	err = pm_runtime_get_sync(op->ss->dev);  in sun4i_ss_cipher_init()
552 	crypto_free_skcipher(op->fallback_tfm);  in sun4i_ss_cipher_init()
558 	struct sun4i_tfm_ctx *op = crypto_tfm_ctx(tfm);  in sun4i_ss_cipher_exit()  local
560 	crypto_free_skcipher(op->fallback_tfm);  in sun4i_ss_cipher_exit()
561 	pm_runtime_put(op->ss->dev);  in sun4i_ss_cipher_exit()
564 /* check and set the AES key, prepare the mode to be used */
568 	struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);  in sun4i_ss_aes_setkey()  local
569 	struct sun4i_ss_ctx *ss = op->ss;  in sun4i_ss_aes_setkey()
573 		op->keymode = SS_AES_128BITS;  in sun4i_ss_aes_setkey()
576 		op->keymode = SS_AES_192BITS;  in sun4i_ss_aes_setkey()
579 		op->keymode = SS_AES_256BITS;  in sun4i_ss_aes_setkey()
582 		dev_dbg(ss->dev, "ERROR: Invalid keylen %u\n", keylen);  in sun4i_ss_aes_setkey()
583 		return -EINVAL;  in sun4i_ss_aes_setkey()
585 	op->keylen = keylen;  in sun4i_ss_aes_setkey()
586 	memcpy(op->key, key, keylen);  in sun4i_ss_aes_setkey()
588 	crypto_skcipher_clear_flags(op->fallback_tfm, CRYPTO_TFM_REQ_MASK);  in sun4i_ss_aes_setkey()
589 	crypto_skcipher_set_flags(op->fallback_tfm, tfm->base.crt_flags & CRYPTO_TFM_REQ_MASK);  in sun4i_ss_aes_setkey()
591 	return crypto_skcipher_setkey(op->fallback_tfm, key, keylen);  in sun4i_ss_aes_setkey()
594 /* check and set the DES key, prepare the mode to be used */
598 	struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);  in sun4i_ss_des_setkey()  local
605 	op->keylen = keylen;  in sun4i_ss_des_setkey()
606 	memcpy(op->key, key, keylen);  in sun4i_ss_des_setkey()
608 	crypto_skcipher_clear_flags(op->fallback_tfm, CRYPTO_TFM_REQ_MASK);  in sun4i_ss_des_setkey()
609 	crypto_skcipher_set_flags(op->fallback_tfm, tfm->base.crt_flags & CRYPTO_TFM_REQ_MASK);  in sun4i_ss_des_setkey()
611 	return crypto_skcipher_setkey(op->fallback_tfm, key, keylen);  in sun4i_ss_des_setkey()
614 /* check and set the 3DES key, prepare the mode to be used */
618 	struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);  in sun4i_ss_des3_setkey()  local
625 	op->keylen = keylen;  in sun4i_ss_des3_setkey()
626 	memcpy(op->key, key, keylen);  in sun4i_ss_des3_setkey()
628 	crypto_skcipher_clear_flags(op->fallback_tfm, CRYPTO_TFM_REQ_MASK);  in sun4i_ss_des3_setkey()
629 	crypto_skcipher_set_flags(op->fallback_tfm, tfm->base.crt_flags & CRYPTO_TFM_REQ_MASK);  in sun4i_ss_des3_setkey()
631 	return crypto_skcipher_setkey(op->fallback_tfm, key, keylen);  in sun4i_ss_des3_setkey()