Lines Matching +full:inline +full:- +full:crypto +full:- +full:engine
1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * Copyright (c) 2010-2011 Picochip Ltd., Jamie Iles
5 #include <crypto/internal/aead.h>
6 #include <crypto/aes.h>
7 #include <crypto/algapi.h>
8 #include <crypto/authenc.h>
9 #include <crypto/internal/des.h>
10 #include <crypto/md5.h>
11 #include <crypto/sha.h>
12 #include <crypto/internal/skcipher.h>
14 #include <linux/crypto.h>
16 #include <linux/dma-mapping.h>
74 * Asynchronous crypto request structure.
81 struct spacc_engine *engine; member
96 struct spacc_engine *engine; member
134 /* SPACC definition of a crypto algorithm. */
139 struct spacc_engine *engine; member
147 struct spacc_engine *engine; member
177 static inline struct spacc_alg *to_spacc_skcipher(struct skcipher_alg *alg) in to_spacc_skcipher()
182 static inline struct spacc_aead *to_spacc_aead(struct aead_alg *alg) in to_spacc_aead()
187 static inline int spacc_fifo_cmd_full(struct spacc_engine *engine) in spacc_fifo_cmd_full() argument
189 u32 fifo_stat = readl(engine->regs + SPA_FIFO_STAT_REG_OFFSET); in spacc_fifo_cmd_full()
201 static inline void __iomem *spacc_ctx_page_addr(struct spacc_generic_ctx *ctx, in spacc_ctx_page_addr()
205 return is_cipher_ctx ? ctx->engine->cipher_ctx_base + in spacc_ctx_page_addr()
206 (indx * ctx->engine->cipher_pg_sz) : in spacc_ctx_page_addr()
207 ctx->engine->hash_key_base + (indx * ctx->engine->hash_pg_sz); in spacc_ctx_page_addr()
210 /* The context pages can only be written with 32-bit accesses. */
211 static inline void memcpy_toio32(u32 __iomem *dst, const void *src, in memcpy_toio32()
216 while (count--) in memcpy_toio32()
224 void __iomem *key_ptr = page_addr + ctx->key_offs; in spacc_cipher_write_ctx()
225 void __iomem *iv_ptr = page_addr + ctx->iv_offs; in spacc_cipher_write_ctx()
241 unsigned indx = ctx->engine->next_ctx++; in spacc_load_ctx()
247 ctx->engine->next_ctx &= ctx->engine->fifo_sz - 1; in spacc_load_ctx()
252 ctx->engine->regs + SPA_KEY_SZ_REG_OFFSET); in spacc_load_ctx()
257 ctx->engine->regs + SPA_KEY_SZ_REG_OFFSET); in spacc_load_ctx()
263 static inline void ddt_set(struct spacc_ddt *ddt, dma_addr_t phys, size_t len) in ddt_set()
265 ddt->p = phys; in ddt_set()
266 ddt->len = len; in ddt_set()
270 * Take a crypto request and scatterlists for the data and turn them into DDTs
271 * for passing to the crypto engines. This also DMA maps the data so that the
272 * crypto engines can DMA to/from them.
274 static struct spacc_ddt *spacc_sg_to_ddt(struct spacc_engine *engine, in spacc_sg_to_ddt() argument
288 dev_err(engine->dev, "Invalid numbers of SG.\n"); in spacc_sg_to_ddt()
291 mapped_ents = dma_map_sg(engine->dev, payload, nents, dir); in spacc_sg_to_ddt()
296 ddt = dma_pool_alloc(engine->req_pool, GFP_ATOMIC, ddt_phys); in spacc_sg_to_ddt()
307 dma_unmap_sg(engine->dev, payload, nents, dir); in spacc_sg_to_ddt()
315 struct spacc_engine *engine = req->engine; in spacc_aead_make_ddts() local
322 total = areq->assoclen + areq->cryptlen; in spacc_aead_make_ddts()
323 if (req->is_encrypt) in spacc_aead_make_ddts()
326 src_nents = sg_nents_for_len(areq->src, total); in spacc_aead_make_ddts()
328 dev_err(engine->dev, "Invalid numbers of src SG.\n"); in spacc_aead_make_ddts()
332 return -E2BIG; in spacc_aead_make_ddts()
335 if (areq->src != areq->dst) { in spacc_aead_make_ddts()
336 dst_nents = sg_nents_for_len(areq->dst, total); in spacc_aead_make_ddts()
338 dev_err(engine->dev, "Invalid numbers of dst SG.\n"); in spacc_aead_make_ddts()
342 return -E2BIG; in spacc_aead_make_ddts()
345 src_ddt = dma_pool_alloc(engine->req_pool, GFP_ATOMIC, &req->src_addr); in spacc_aead_make_ddts()
349 dst_ddt = dma_pool_alloc(engine->req_pool, GFP_ATOMIC, &req->dst_addr); in spacc_aead_make_ddts()
353 req->src_ddt = src_ddt; in spacc_aead_make_ddts()
354 req->dst_ddt = dst_ddt; in spacc_aead_make_ddts()
357 src_ents = dma_map_sg(engine->dev, areq->src, src_nents, in spacc_aead_make_ddts()
362 dst_ents = dma_map_sg(engine->dev, areq->dst, dst_nents, in spacc_aead_make_ddts()
366 dma_unmap_sg(engine->dev, areq->src, src_nents, in spacc_aead_make_ddts()
371 src_ents = dma_map_sg(engine->dev, areq->src, src_nents, in spacc_aead_make_ddts()
382 for_each_sg(areq->src, cur, src_ents, i) in spacc_aead_make_ddts()
386 total = req->is_encrypt ? 0 : areq->assoclen; in spacc_aead_make_ddts()
387 for_each_sg(areq->dst, cur, dst_ents, i) { in spacc_aead_make_ddts()
391 total -= len; in spacc_aead_make_ddts()
395 ddt_set(dst_ddt++, sg_dma_address(cur) + total, len - total); in spacc_aead_make_ddts()
404 dma_pool_free(engine->req_pool, dst_ddt, req->dst_addr); in spacc_aead_make_ddts()
406 dma_pool_free(engine->req_pool, src_ddt, req->src_addr); in spacc_aead_make_ddts()
408 return -ENOMEM; in spacc_aead_make_ddts()
413 struct aead_request *areq = container_of(req->req, struct aead_request, in spacc_aead_free_ddts()
416 unsigned total = areq->assoclen + areq->cryptlen + in spacc_aead_free_ddts()
417 (req->is_encrypt ? crypto_aead_authsize(aead) : 0); in spacc_aead_free_ddts()
419 struct spacc_engine *engine = aead_ctx->generic.engine; in spacc_aead_free_ddts() local
420 int nents = sg_nents_for_len(areq->src, total); in spacc_aead_free_ddts()
424 dev_err(engine->dev, "Invalid numbers of src SG.\n"); in spacc_aead_free_ddts()
428 if (areq->src != areq->dst) { in spacc_aead_free_ddts()
429 dma_unmap_sg(engine->dev, areq->src, nents, DMA_TO_DEVICE); in spacc_aead_free_ddts()
430 nents = sg_nents_for_len(areq->dst, total); in spacc_aead_free_ddts()
432 dev_err(engine->dev, "Invalid numbers of dst SG.\n"); in spacc_aead_free_ddts()
435 dma_unmap_sg(engine->dev, areq->dst, nents, DMA_FROM_DEVICE); in spacc_aead_free_ddts()
437 dma_unmap_sg(engine->dev, areq->src, nents, DMA_BIDIRECTIONAL); in spacc_aead_free_ddts()
439 dma_pool_free(engine->req_pool, req->src_ddt, req->src_addr); in spacc_aead_free_ddts()
440 dma_pool_free(engine->req_pool, req->dst_ddt, req->dst_addr); in spacc_aead_free_ddts()
450 dev_err(req->engine->dev, "Invalid numbers of SG.\n"); in spacc_free_ddt()
454 dma_unmap_sg(req->engine->dev, payload, nents, dir); in spacc_free_ddt()
455 dma_pool_free(req->engine->req_pool, ddt, ddt_addr); in spacc_free_ddt()
465 crypto_aead_clear_flags(ctx->sw_cipher, CRYPTO_TFM_REQ_MASK); in spacc_aead_setkey()
466 crypto_aead_set_flags(ctx->sw_cipher, crypto_aead_get_flags(tfm) & in spacc_aead_setkey()
468 err = crypto_aead_setkey(ctx->sw_cipher, key, keylen); in spacc_aead_setkey()
478 if (keys.authkeylen > sizeof(ctx->hash_ctx)) in spacc_aead_setkey()
481 memcpy(ctx->cipher_key, keys.enckey, keys.enckeylen); in spacc_aead_setkey()
482 ctx->cipher_key_len = keys.enckeylen; in spacc_aead_setkey()
484 memcpy(ctx->hash_ctx, keys.authkey, keys.authkeylen); in spacc_aead_setkey()
485 ctx->hash_key_len = keys.authkeylen; in spacc_aead_setkey()
492 return -EINVAL; in spacc_aead_setkey()
500 return crypto_aead_setauthsize(ctx->sw_cipher, authsize); in spacc_aead_setauthsize()
516 * If we have a non-supported key-length, then we need to do a in spacc_aead_need_fallback()
519 if ((spacc_alg->ctrl_default & SPACC_CRYPTO_ALG_MASK) == in spacc_aead_need_fallback()
521 ctx->cipher_key_len != AES_KEYSIZE_128 && in spacc_aead_need_fallback()
522 ctx->cipher_key_len != AES_KEYSIZE_256) in spacc_aead_need_fallback()
535 aead_request_set_tfm(subreq, ctx->sw_cipher); in spacc_aead_do_fallback()
536 aead_request_set_callback(subreq, req->base.flags, in spacc_aead_do_fallback()
537 req->base.complete, req->base.data); in spacc_aead_do_fallback()
538 aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen, in spacc_aead_do_fallback()
539 req->iv); in spacc_aead_do_fallback()
540 aead_request_set_ad(subreq, req->assoclen); in spacc_aead_do_fallback()
549 req->req->complete(req->req, req->result); in spacc_aead_complete()
555 container_of(req->req, struct aead_request, base); in spacc_aead_submit()
561 struct spacc_engine *engine = ctx->generic.engine; in spacc_aead_submit() local
564 req->result = -EINPROGRESS; in spacc_aead_submit()
565 req->ctx_id = spacc_load_ctx(&ctx->generic, ctx->cipher_key, in spacc_aead_submit()
566 ctx->cipher_key_len, aead_req->iv, crypto_aead_ivsize(aead), in spacc_aead_submit()
567 ctx->hash_ctx, ctx->hash_key_len); in spacc_aead_submit()
570 writel(req->src_addr, engine->regs + SPA_SRC_PTR_REG_OFFSET); in spacc_aead_submit()
571 writel(req->dst_addr, engine->regs + SPA_DST_PTR_REG_OFFSET); in spacc_aead_submit()
572 writel(0, engine->regs + SPA_OFFSET_REG_OFFSET); in spacc_aead_submit()
574 assoc_len = aead_req->assoclen; in spacc_aead_submit()
575 proc_len = aead_req->cryptlen + assoc_len; in spacc_aead_submit()
581 if (!req->is_encrypt) in spacc_aead_submit()
582 proc_len -= authsize; in spacc_aead_submit()
584 writel(proc_len, engine->regs + SPA_PROC_LEN_REG_OFFSET); in spacc_aead_submit()
585 writel(assoc_len, engine->regs + SPA_AAD_LEN_REG_OFFSET); in spacc_aead_submit()
586 writel(authsize, engine->regs + SPA_ICV_LEN_REG_OFFSET); in spacc_aead_submit()
587 writel(0, engine->regs + SPA_ICV_OFFSET_REG_OFFSET); in spacc_aead_submit()
588 writel(0, engine->regs + SPA_AUX_INFO_REG_OFFSET); in spacc_aead_submit()
590 ctrl = spacc_alg->ctrl_default | (req->ctx_id << SPA_CTRL_CTX_IDX) | in spacc_aead_submit()
592 if (req->is_encrypt) in spacc_aead_submit()
597 mod_timer(&engine->packet_timeout, jiffies + PACKET_TIMEOUT); in spacc_aead_submit()
599 writel(ctrl, engine->regs + SPA_CTRL_REG_OFFSET); in spacc_aead_submit()
601 return -EINPROGRESS; in spacc_aead_submit()
606 static void spacc_push(struct spacc_engine *engine) in spacc_push() argument
610 while (!list_empty(&engine->pending) && in spacc_push()
611 engine->in_flight + 1 <= engine->fifo_sz) { in spacc_push()
613 ++engine->in_flight; in spacc_push()
614 req = list_first_entry(&engine->pending, struct spacc_req, in spacc_push()
616 list_move_tail(&req->list, &engine->in_progress); in spacc_push()
618 req->result = spacc_req_submit(req); in spacc_push()
623 * Setup an AEAD request for processing. This will configure the engine, load
631 struct spacc_engine *engine = to_spacc_aead(alg)->engine; in spacc_aead_setup() local
636 dev_req->req = &req->base; in spacc_aead_setup()
637 dev_req->is_encrypt = is_encrypt; in spacc_aead_setup()
638 dev_req->result = -EBUSY; in spacc_aead_setup()
639 dev_req->engine = engine; in spacc_aead_setup()
640 dev_req->complete = spacc_aead_complete; in spacc_aead_setup()
643 ((err = spacc_aead_make_ddts(req)) == -E2BIG))) in spacc_aead_setup()
649 err = -EINPROGRESS; in spacc_aead_setup()
650 spin_lock_irqsave(&engine->hw_lock, flags); in spacc_aead_setup()
651 if (unlikely(spacc_fifo_cmd_full(engine)) || in spacc_aead_setup()
652 engine->in_flight + 1 > engine->fifo_sz) { in spacc_aead_setup()
653 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { in spacc_aead_setup()
654 err = -EBUSY; in spacc_aead_setup()
655 spin_unlock_irqrestore(&engine->hw_lock, flags); in spacc_aead_setup()
658 list_add_tail(&dev_req->list, &engine->pending); in spacc_aead_setup()
660 list_add_tail(&dev_req->list, &engine->pending); in spacc_aead_setup()
661 spacc_push(engine); in spacc_aead_setup()
663 spin_unlock_irqrestore(&engine->hw_lock, flags); in spacc_aead_setup()
678 return spacc_aead_setup(req, alg->type, 1); in spacc_aead_encrypt()
686 return spacc_aead_setup(req, alg->type, 0); in spacc_aead_decrypt()
698 struct spacc_engine *engine = spacc_alg->engine; in spacc_aead_cra_init() local
700 ctx->generic.flags = spacc_alg->type; in spacc_aead_cra_init()
701 ctx->generic.engine = engine; in spacc_aead_cra_init()
702 ctx->sw_cipher = crypto_alloc_aead(alg->base.cra_name, 0, in spacc_aead_cra_init()
704 if (IS_ERR(ctx->sw_cipher)) in spacc_aead_cra_init()
705 return PTR_ERR(ctx->sw_cipher); in spacc_aead_cra_init()
706 ctx->generic.key_offs = spacc_alg->key_offs; in spacc_aead_cra_init()
707 ctx->generic.iv_offs = spacc_alg->iv_offs; in spacc_aead_cra_init()
713 crypto_aead_reqsize(ctx->sw_cipher))); in spacc_aead_cra_init()
726 crypto_free_aead(ctx->sw_cipher); in spacc_aead_cra_exit()
743 memcpy(ctx->key, key, len); in spacc_des_setkey()
744 ctx->key_len = len; in spacc_des_setkey()
763 memcpy(ctx->key, key, len); in spacc_des3_setkey()
764 ctx->key_len = len; in spacc_des3_setkey()
781 return -EINVAL; in spacc_aes_setkey()
784 * IPSec engine only supports 128 and 256 bit AES keys. If we get a in spacc_aes_setkey()
789 if (!ctx->sw_cipher) in spacc_aes_setkey()
790 return -EINVAL; in spacc_aes_setkey()
796 crypto_skcipher_clear_flags(ctx->sw_cipher, in spacc_aes_setkey()
798 crypto_skcipher_set_flags(ctx->sw_cipher, in spacc_aes_setkey()
799 cipher->base.crt_flags & in spacc_aes_setkey()
802 err = crypto_skcipher_setkey(ctx->sw_cipher, key, len); in spacc_aes_setkey()
807 memcpy(ctx->key, key, len); in spacc_aes_setkey()
808 ctx->key_len = len; in spacc_aes_setkey()
822 err = -EINVAL; in spacc_kasumi_f8_setkey()
826 memcpy(ctx->key, key, len); in spacc_kasumi_f8_setkey()
827 ctx->key_len = len; in spacc_kasumi_f8_setkey()
835 struct skcipher_request *ablk_req = skcipher_request_cast(req->req); in spacc_ablk_need_fallback()
842 return (spacc_alg->ctrl_default & SPACC_CRYPTO_ALG_MASK) == in spacc_ablk_need_fallback()
844 ctx->key_len != AES_KEYSIZE_128 && in spacc_ablk_need_fallback()
845 ctx->key_len != AES_KEYSIZE_256; in spacc_ablk_need_fallback()
850 struct skcipher_request *ablk_req = skcipher_request_cast(req->req); in spacc_ablk_complete()
852 if (ablk_req->src != ablk_req->dst) { in spacc_ablk_complete()
853 spacc_free_ddt(req, req->src_ddt, req->src_addr, ablk_req->src, in spacc_ablk_complete()
854 ablk_req->cryptlen, DMA_TO_DEVICE); in spacc_ablk_complete()
855 spacc_free_ddt(req, req->dst_ddt, req->dst_addr, ablk_req->dst, in spacc_ablk_complete()
856 ablk_req->cryptlen, DMA_FROM_DEVICE); in spacc_ablk_complete()
858 spacc_free_ddt(req, req->dst_ddt, req->dst_addr, ablk_req->dst, in spacc_ablk_complete()
859 ablk_req->cryptlen, DMA_BIDIRECTIONAL); in spacc_ablk_complete()
861 req->req->complete(req->req, req->result); in spacc_ablk_complete()
866 struct skcipher_request *ablk_req = skcipher_request_cast(req->req); in spacc_ablk_submit()
871 struct spacc_engine *engine = ctx->generic.engine; in spacc_ablk_submit() local
874 req->ctx_id = spacc_load_ctx(&ctx->generic, ctx->key, in spacc_ablk_submit()
875 ctx->key_len, ablk_req->iv, alg->ivsize, in spacc_ablk_submit()
878 writel(req->src_addr, engine->regs + SPA_SRC_PTR_REG_OFFSET); in spacc_ablk_submit()
879 writel(req->dst_addr, engine->regs + SPA_DST_PTR_REG_OFFSET); in spacc_ablk_submit()
880 writel(0, engine->regs + SPA_OFFSET_REG_OFFSET); in spacc_ablk_submit()
882 writel(ablk_req->cryptlen, engine->regs + SPA_PROC_LEN_REG_OFFSET); in spacc_ablk_submit()
883 writel(0, engine->regs + SPA_ICV_OFFSET_REG_OFFSET); in spacc_ablk_submit()
884 writel(0, engine->regs + SPA_AUX_INFO_REG_OFFSET); in spacc_ablk_submit()
885 writel(0, engine->regs + SPA_AAD_LEN_REG_OFFSET); in spacc_ablk_submit()
887 ctrl = spacc_alg->ctrl_default | (req->ctx_id << SPA_CTRL_CTX_IDX) | in spacc_ablk_submit()
888 (req->is_encrypt ? (1 << SPA_CTRL_ENCRYPT_IDX) : in spacc_ablk_submit()
891 mod_timer(&engine->packet_timeout, jiffies + PACKET_TIMEOUT); in spacc_ablk_submit()
893 writel(ctrl, engine->regs + SPA_CTRL_REG_OFFSET); in spacc_ablk_submit()
895 return -EINPROGRESS; in spacc_ablk_submit()
912 skcipher_request_set_tfm(&dev_req->fallback_req, ctx->sw_cipher); in spacc_ablk_do_fallback()
913 skcipher_request_set_callback(&dev_req->fallback_req, req->base.flags, in spacc_ablk_do_fallback()
914 req->base.complete, req->base.data); in spacc_ablk_do_fallback()
915 skcipher_request_set_crypt(&dev_req->fallback_req, req->src, req->dst, in spacc_ablk_do_fallback()
916 req->cryptlen, req->iv); in spacc_ablk_do_fallback()
917 err = is_encrypt ? crypto_skcipher_encrypt(&dev_req->fallback_req) : in spacc_ablk_do_fallback()
918 crypto_skcipher_decrypt(&dev_req->fallback_req); in spacc_ablk_do_fallback()
928 struct spacc_engine *engine = to_spacc_skcipher(alg)->engine; in spacc_ablk_setup() local
931 int err = -ENOMEM; in spacc_ablk_setup()
933 dev_req->req = &req->base; in spacc_ablk_setup()
934 dev_req->is_encrypt = is_encrypt; in spacc_ablk_setup()
935 dev_req->engine = engine; in spacc_ablk_setup()
936 dev_req->complete = spacc_ablk_complete; in spacc_ablk_setup()
937 dev_req->result = -EINPROGRESS; in spacc_ablk_setup()
943 * Create the DDT's for the engine. If we share the same source and in spacc_ablk_setup()
946 if (req->src != req->dst) { in spacc_ablk_setup()
947 dev_req->src_ddt = spacc_sg_to_ddt(engine, req->src, in spacc_ablk_setup()
948 req->cryptlen, DMA_TO_DEVICE, &dev_req->src_addr); in spacc_ablk_setup()
949 if (!dev_req->src_ddt) in spacc_ablk_setup()
952 dev_req->dst_ddt = spacc_sg_to_ddt(engine, req->dst, in spacc_ablk_setup()
953 req->cryptlen, DMA_FROM_DEVICE, &dev_req->dst_addr); in spacc_ablk_setup()
954 if (!dev_req->dst_ddt) in spacc_ablk_setup()
957 dev_req->dst_ddt = spacc_sg_to_ddt(engine, req->dst, in spacc_ablk_setup()
958 req->cryptlen, DMA_BIDIRECTIONAL, &dev_req->dst_addr); in spacc_ablk_setup()
959 if (!dev_req->dst_ddt) in spacc_ablk_setup()
962 dev_req->src_ddt = NULL; in spacc_ablk_setup()
963 dev_req->src_addr = dev_req->dst_addr; in spacc_ablk_setup()
966 err = -EINPROGRESS; in spacc_ablk_setup()
967 spin_lock_irqsave(&engine->hw_lock, flags); in spacc_ablk_setup()
969 * Check if the engine will accept the operation now. If it won't then in spacc_ablk_setup()
973 if (unlikely(spacc_fifo_cmd_full(engine)) || in spacc_ablk_setup()
974 engine->in_flight + 1 > engine->fifo_sz) { in spacc_ablk_setup()
975 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { in spacc_ablk_setup()
976 err = -EBUSY; in spacc_ablk_setup()
977 spin_unlock_irqrestore(&engine->hw_lock, flags); in spacc_ablk_setup()
980 list_add_tail(&dev_req->list, &engine->pending); in spacc_ablk_setup()
982 list_add_tail(&dev_req->list, &engine->pending); in spacc_ablk_setup()
983 spacc_push(engine); in spacc_ablk_setup()
985 spin_unlock_irqrestore(&engine->hw_lock, flags); in spacc_ablk_setup()
990 spacc_free_ddt(dev_req, dev_req->dst_ddt, dev_req->dst_addr, req->dst, in spacc_ablk_setup()
991 req->cryptlen, req->src == req->dst ? in spacc_ablk_setup()
994 if (req->src != req->dst) in spacc_ablk_setup()
995 spacc_free_ddt(dev_req, dev_req->src_ddt, dev_req->src_addr, in spacc_ablk_setup()
996 req->src, req->cryptlen, DMA_TO_DEVICE); in spacc_ablk_setup()
1006 struct spacc_engine *engine = spacc_alg->engine; in spacc_ablk_init_tfm() local
1008 ctx->generic.flags = spacc_alg->type; in spacc_ablk_init_tfm()
1009 ctx->generic.engine = engine; in spacc_ablk_init_tfm()
1010 if (alg->base.cra_flags & CRYPTO_ALG_NEED_FALLBACK) { in spacc_ablk_init_tfm()
1011 ctx->sw_cipher = crypto_alloc_skcipher(alg->base.cra_name, 0, in spacc_ablk_init_tfm()
1013 if (IS_ERR(ctx->sw_cipher)) { in spacc_ablk_init_tfm()
1014 dev_warn(engine->dev, "failed to allocate fallback for %s\n", in spacc_ablk_init_tfm()
1015 alg->base.cra_name); in spacc_ablk_init_tfm()
1016 return PTR_ERR(ctx->sw_cipher); in spacc_ablk_init_tfm()
1019 crypto_skcipher_reqsize(ctx->sw_cipher)); in spacc_ablk_init_tfm()
1026 ctx->generic.key_offs = spacc_alg->key_offs; in spacc_ablk_init_tfm()
1027 ctx->generic.iv_offs = spacc_alg->iv_offs; in spacc_ablk_init_tfm()
1036 crypto_free_skcipher(ctx->sw_cipher); in spacc_ablk_exit_tfm()
1045 return spacc_ablk_setup(req, spacc_alg->type, 1); in spacc_ablk_encrypt()
1054 return spacc_ablk_setup(req, spacc_alg->type, 0); in spacc_ablk_decrypt()
1057 static inline int spacc_fifo_stat_empty(struct spacc_engine *engine) in spacc_fifo_stat_empty() argument
1059 return readl(engine->regs + SPA_FIFO_STAT_REG_OFFSET) & in spacc_fifo_stat_empty()
1063 static void spacc_process_done(struct spacc_engine *engine) in spacc_process_done() argument
1068 spin_lock_irqsave(&engine->hw_lock, flags); in spacc_process_done()
1070 while (!spacc_fifo_stat_empty(engine)) { in spacc_process_done()
1071 req = list_first_entry(&engine->in_progress, struct spacc_req, in spacc_process_done()
1073 list_move_tail(&req->list, &engine->completed); in spacc_process_done()
1074 --engine->in_flight; in spacc_process_done()
1077 writel(~0, engine->regs + SPA_STAT_POP_REG_OFFSET); in spacc_process_done()
1078 req->result = (readl(engine->regs + SPA_STATUS_REG_OFFSET) & in spacc_process_done()
1085 if (unlikely(req->result)) { in spacc_process_done()
1086 switch (req->result) { in spacc_process_done()
1088 req->result = -EBADMSG; in spacc_process_done()
1092 dev_warn(engine->dev, in spacc_process_done()
1094 req->result = -EFAULT; in spacc_process_done()
1098 dev_warn(engine->dev, in spacc_process_done()
1100 req->result = -EIO; in spacc_process_done()
1106 tasklet_schedule(&engine->complete); in spacc_process_done()
1108 spin_unlock_irqrestore(&engine->hw_lock, flags); in spacc_process_done()
1113 struct spacc_engine *engine = (struct spacc_engine *)dev; in spacc_spacc_irq() local
1114 u32 spacc_irq_stat = readl(engine->regs + SPA_IRQ_STAT_REG_OFFSET); in spacc_spacc_irq()
1116 writel(spacc_irq_stat, engine->regs + SPA_IRQ_STAT_REG_OFFSET); in spacc_spacc_irq()
1117 spacc_process_done(engine); in spacc_spacc_irq()
1124 struct spacc_engine *engine = from_timer(engine, t, packet_timeout); in spacc_packet_timeout() local
1126 spacc_process_done(engine); in spacc_packet_timeout()
1131 struct crypto_alg *alg = req->req->tfm->__crt_alg; in spacc_req_submit()
1133 if (CRYPTO_ALG_TYPE_AEAD == (CRYPTO_ALG_TYPE_MASK & alg->cra_flags)) in spacc_req_submit()
1141 struct spacc_engine *engine = (struct spacc_engine *)data; in spacc_spacc_complete() local
1146 spin_lock_irqsave(&engine->hw_lock, flags); in spacc_spacc_complete()
1148 list_splice_init(&engine->completed, &completed); in spacc_spacc_complete()
1149 spacc_push(engine); in spacc_spacc_complete()
1150 if (engine->in_flight) in spacc_spacc_complete()
1151 mod_timer(&engine->packet_timeout, jiffies + PACKET_TIMEOUT); in spacc_spacc_complete()
1153 spin_unlock_irqrestore(&engine->hw_lock, flags); in spacc_spacc_complete()
1156 list_del(&req->list); in spacc_spacc_complete()
1157 req->complete(req); in spacc_spacc_complete()
1164 struct spacc_engine *engine = dev_get_drvdata(dev); in spacc_suspend() local
1171 clk_disable(engine->clk); in spacc_suspend()
1178 struct spacc_engine *engine = dev_get_drvdata(dev); in spacc_resume() local
1180 return clk_enable(engine->clk); in spacc_resume()
1189 static inline struct spacc_engine *spacc_dev_to_engine(struct device *dev) in spacc_dev_to_engine()
1198 struct spacc_engine *engine = spacc_dev_to_engine(dev); in spacc_stat_irq_thresh_show() local
1200 return snprintf(buf, PAGE_SIZE, "%u\n", engine->stat_irq_thresh); in spacc_stat_irq_thresh_show()
1207 struct spacc_engine *engine = spacc_dev_to_engine(dev); in spacc_stat_irq_thresh_store() local
1211 return -EINVAL; in spacc_stat_irq_thresh_store()
1213 thresh = clamp(thresh, 1UL, engine->fifo_sz - 1); in spacc_stat_irq_thresh_store()
1215 engine->stat_irq_thresh = thresh; in spacc_stat_irq_thresh_store()
1216 writel(engine->stat_irq_thresh << SPA_IRQ_CTRL_STAT_CNT_OFFSET, in spacc_stat_irq_thresh_store()
1217 engine->regs + SPA_IRQ_CTRL_REG_OFFSET); in spacc_stat_irq_thresh_store()
1231 .base.cra_driver_name = "cbc-aes-picoxcell",
1257 .base.cra_driver_name = "ecb-aes-picoxcell",
1282 .base.cra_driver_name = "cbc-des-picoxcell",
1307 .base.cra_driver_name = "ecb-des-picoxcell",
1331 .base.cra_driver_name = "cbc-des3-ede-picoxcell",
1356 .base.cra_driver_name = "ecb-des3-ede-picoxcell",
1387 .cra_driver_name = "authenc-hmac-sha1-"
1388 "cbc-aes-picoxcell",
1418 .cra_driver_name = "authenc-hmac-sha256-"
1419 "cbc-aes-picoxcell",
1449 .cra_driver_name = "authenc-hmac-md5-"
1450 "cbc-aes-picoxcell",
1480 .cra_driver_name = "authenc-hmac-sha1-"
1481 "cbc-3des-picoxcell",
1512 .cra_driver_name = "authenc-hmac-sha256-"
1513 "cbc-3des-picoxcell",
1543 .cra_driver_name = "authenc-hmac-md5-"
1544 "cbc-3des-picoxcell",
1574 .base.cra_driver_name = "f8-kasumi-picoxcell",
1597 { .compatible = "picochip,spacc-ipsec" },
1598 { .compatible = "picochip,spacc-l2" },
1613 struct device_node *np = pdev->dev.of_node; in spacc_probe()
1614 struct spacc_engine *engine = devm_kzalloc(&pdev->dev, sizeof(*engine), in spacc_probe() local
1616 if (!engine) in spacc_probe()
1617 return -ENOMEM; in spacc_probe()
1619 if (of_device_is_compatible(np, "picochip,spacc-ipsec")) { in spacc_probe()
1620 engine->max_ctxs = SPACC_CRYPTO_IPSEC_MAX_CTXS; in spacc_probe()
1621 engine->cipher_pg_sz = SPACC_CRYPTO_IPSEC_CIPHER_PG_SZ; in spacc_probe()
1622 engine->hash_pg_sz = SPACC_CRYPTO_IPSEC_HASH_PG_SZ; in spacc_probe()
1623 engine->fifo_sz = SPACC_CRYPTO_IPSEC_FIFO_SZ; in spacc_probe()
1624 engine->algs = ipsec_engine_algs; in spacc_probe()
1625 engine->num_algs = ARRAY_SIZE(ipsec_engine_algs); in spacc_probe()
1626 engine->aeads = ipsec_engine_aeads; in spacc_probe()
1627 engine->num_aeads = ARRAY_SIZE(ipsec_engine_aeads); in spacc_probe()
1628 } else if (of_device_is_compatible(np, "picochip,spacc-l2")) { in spacc_probe()
1629 engine->max_ctxs = SPACC_CRYPTO_L2_MAX_CTXS; in spacc_probe()
1630 engine->cipher_pg_sz = SPACC_CRYPTO_L2_CIPHER_PG_SZ; in spacc_probe()
1631 engine->hash_pg_sz = SPACC_CRYPTO_L2_HASH_PG_SZ; in spacc_probe()
1632 engine->fifo_sz = SPACC_CRYPTO_L2_FIFO_SZ; in spacc_probe()
1633 engine->algs = l2_engine_algs; in spacc_probe()
1634 engine->num_algs = ARRAY_SIZE(l2_engine_algs); in spacc_probe()
1636 return -EINVAL; in spacc_probe()
1639 engine->name = dev_name(&pdev->dev); in spacc_probe()
1641 engine->regs = devm_platform_ioremap_resource(pdev, 0); in spacc_probe()
1642 if (IS_ERR(engine->regs)) in spacc_probe()
1643 return PTR_ERR(engine->regs); in spacc_probe()
1647 dev_err(&pdev->dev, "no memory/irq resource for engine\n"); in spacc_probe()
1648 return -ENXIO; in spacc_probe()
1651 tasklet_init(&engine->complete, spacc_spacc_complete, in spacc_probe()
1652 (unsigned long)engine); in spacc_probe()
1654 ret = devm_add_action(&pdev->dev, spacc_tasklet_kill, in spacc_probe()
1655 &engine->complete); in spacc_probe()
1659 if (devm_request_irq(&pdev->dev, irq->start, spacc_spacc_irq, 0, in spacc_probe()
1660 engine->name, engine)) { in spacc_probe()
1661 dev_err(engine->dev, "failed to request IRQ\n"); in spacc_probe()
1662 return -EBUSY; in spacc_probe()
1665 engine->dev = &pdev->dev; in spacc_probe()
1666 engine->cipher_ctx_base = engine->regs + SPA_CIPH_KEY_BASE_REG_OFFSET; in spacc_probe()
1667 engine->hash_key_base = engine->regs + SPA_HASH_KEY_BASE_REG_OFFSET; in spacc_probe()
1669 engine->req_pool = dmam_pool_create(engine->name, engine->dev, in spacc_probe()
1671 if (!engine->req_pool) in spacc_probe()
1672 return -ENOMEM; in spacc_probe()
1674 spin_lock_init(&engine->hw_lock); in spacc_probe()
1676 engine->clk = clk_get(&pdev->dev, "ref"); in spacc_probe()
1677 if (IS_ERR(engine->clk)) { in spacc_probe()
1678 dev_info(&pdev->dev, "clk unavailable\n"); in spacc_probe()
1679 return PTR_ERR(engine->clk); in spacc_probe()
1682 if (clk_prepare_enable(engine->clk)) { in spacc_probe()
1683 dev_info(&pdev->dev, "unable to prepare/enable clk\n"); in spacc_probe()
1684 ret = -EIO; in spacc_probe()
1693 engine->stat_irq_thresh = (engine->fifo_sz / 2); in spacc_probe()
1695 ret = device_create_file(&pdev->dev, &dev_attr_stat_irq_thresh); in spacc_probe()
1704 writel(engine->stat_irq_thresh << SPA_IRQ_CTRL_STAT_CNT_OFFSET, in spacc_probe()
1705 engine->regs + SPA_IRQ_CTRL_REG_OFFSET); in spacc_probe()
1707 engine->regs + SPA_IRQ_EN_REG_OFFSET); in spacc_probe()
1709 timer_setup(&engine->packet_timeout, spacc_packet_timeout, 0); in spacc_probe()
1711 INIT_LIST_HEAD(&engine->pending); in spacc_probe()
1712 INIT_LIST_HEAD(&engine->completed); in spacc_probe()
1713 INIT_LIST_HEAD(&engine->in_progress); in spacc_probe()
1714 engine->in_flight = 0; in spacc_probe()
1716 platform_set_drvdata(pdev, engine); in spacc_probe()
1718 ret = -EINVAL; in spacc_probe()
1719 INIT_LIST_HEAD(&engine->registered_algs); in spacc_probe()
1720 for (i = 0; i < engine->num_algs; ++i) { in spacc_probe()
1721 engine->algs[i].engine = engine; in spacc_probe()
1722 err = crypto_register_skcipher(&engine->algs[i].alg); in spacc_probe()
1724 list_add_tail(&engine->algs[i].entry, in spacc_probe()
1725 &engine->registered_algs); in spacc_probe()
1729 dev_err(engine->dev, "failed to register alg \"%s\"\n", in spacc_probe()
1730 engine->algs[i].alg.base.cra_name); in spacc_probe()
1732 dev_dbg(engine->dev, "registered alg \"%s\"\n", in spacc_probe()
1733 engine->algs[i].alg.base.cra_name); in spacc_probe()
1736 INIT_LIST_HEAD(&engine->registered_aeads); in spacc_probe()
1737 for (i = 0; i < engine->num_aeads; ++i) { in spacc_probe()
1738 engine->aeads[i].engine = engine; in spacc_probe()
1739 err = crypto_register_aead(&engine->aeads[i].alg); in spacc_probe()
1741 list_add_tail(&engine->aeads[i].entry, in spacc_probe()
1742 &engine->registered_aeads); in spacc_probe()
1746 dev_err(engine->dev, "failed to register alg \"%s\"\n", in spacc_probe()
1747 engine->aeads[i].alg.base.cra_name); in spacc_probe()
1749 dev_dbg(engine->dev, "registered alg \"%s\"\n", in spacc_probe()
1750 engine->aeads[i].alg.base.cra_name); in spacc_probe()
1756 del_timer_sync(&engine->packet_timeout); in spacc_probe()
1757 device_remove_file(&pdev->dev, &dev_attr_stat_irq_thresh); in spacc_probe()
1759 clk_disable_unprepare(engine->clk); in spacc_probe()
1761 clk_put(engine->clk); in spacc_probe()
1770 struct spacc_engine *engine = platform_get_drvdata(pdev); in spacc_remove() local
1772 del_timer_sync(&engine->packet_timeout); in spacc_remove()
1773 device_remove_file(&pdev->dev, &dev_attr_stat_irq_thresh); in spacc_remove()
1775 list_for_each_entry_safe(aead, an, &engine->registered_aeads, entry) { in spacc_remove()
1776 list_del(&aead->entry); in spacc_remove()
1777 crypto_unregister_aead(&aead->alg); in spacc_remove()
1780 list_for_each_entry_safe(alg, next, &engine->registered_algs, entry) { in spacc_remove()
1781 list_del(&alg->entry); in spacc_remove()
1782 crypto_unregister_skcipher(&alg->alg); in spacc_remove()
1785 clk_disable_unprepare(engine->clk); in spacc_remove()
1786 clk_put(engine->clk); in spacc_remove()