Lines Matching full:engine
94 struct spacc_engine *engine; member
108 struct spacc_engine *engine; member
151 struct spacc_engine *engine; member
159 struct spacc_engine *engine; member
199 static inline int spacc_fifo_cmd_full(struct spacc_engine *engine) in spacc_fifo_cmd_full() argument
201 u32 fifo_stat = readl(engine->regs + SPA_FIFO_STAT_REG_OFFSET); in spacc_fifo_cmd_full()
217 return is_cipher_ctx ? ctx->engine->cipher_ctx_base + in spacc_ctx_page_addr()
218 (indx * ctx->engine->cipher_pg_sz) : in spacc_ctx_page_addr()
219 ctx->engine->hash_key_base + (indx * ctx->engine->hash_pg_sz); in spacc_ctx_page_addr()
253 unsigned indx = ctx->engine->next_ctx++; in spacc_load_ctx()
259 ctx->engine->next_ctx &= ctx->engine->fifo_sz - 1; in spacc_load_ctx()
264 ctx->engine->regs + SPA_KEY_SZ_REG_OFFSET); in spacc_load_ctx()
269 ctx->engine->regs + SPA_KEY_SZ_REG_OFFSET); in spacc_load_ctx()
286 static struct spacc_ddt *spacc_sg_to_ddt(struct spacc_engine *engine, in spacc_sg_to_ddt() argument
300 dev_err(engine->dev, "Invalid numbers of SG.\n"); in spacc_sg_to_ddt()
303 mapped_ents = dma_map_sg(engine->dev, payload, nents, dir); in spacc_sg_to_ddt()
308 ddt = dma_pool_alloc(engine->req_pool, GFP_ATOMIC, ddt_phys); in spacc_sg_to_ddt()
319 dma_unmap_sg(engine->dev, payload, nents, dir); in spacc_sg_to_ddt()
327 struct spacc_engine *engine = req->engine; in spacc_aead_make_ddts() local
340 dev_err(engine->dev, "Invalid numbers of src SG.\n"); in spacc_aead_make_ddts()
350 dev_err(engine->dev, "Invalid numbers of dst SG.\n"); in spacc_aead_make_ddts()
357 src_ddt = dma_pool_alloc(engine->req_pool, GFP_ATOMIC, &req->src_addr); in spacc_aead_make_ddts()
361 dst_ddt = dma_pool_alloc(engine->req_pool, GFP_ATOMIC, &req->dst_addr); in spacc_aead_make_ddts()
369 src_ents = dma_map_sg(engine->dev, areq->src, src_nents, in spacc_aead_make_ddts()
374 dst_ents = dma_map_sg(engine->dev, areq->dst, dst_nents, in spacc_aead_make_ddts()
378 dma_unmap_sg(engine->dev, areq->src, src_nents, in spacc_aead_make_ddts()
383 src_ents = dma_map_sg(engine->dev, areq->src, src_nents, in spacc_aead_make_ddts()
416 dma_pool_free(engine->req_pool, dst_ddt, req->dst_addr); in spacc_aead_make_ddts()
418 dma_pool_free(engine->req_pool, src_ddt, req->src_addr); in spacc_aead_make_ddts()
431 struct spacc_engine *engine = aead_ctx->generic.engine; in spacc_aead_free_ddts() local
436 dev_err(engine->dev, "Invalid numbers of src SG.\n"); in spacc_aead_free_ddts()
441 dma_unmap_sg(engine->dev, areq->src, nents, DMA_TO_DEVICE); in spacc_aead_free_ddts()
444 dev_err(engine->dev, "Invalid numbers of dst SG.\n"); in spacc_aead_free_ddts()
447 dma_unmap_sg(engine->dev, areq->dst, nents, DMA_FROM_DEVICE); in spacc_aead_free_ddts()
449 dma_unmap_sg(engine->dev, areq->src, nents, DMA_BIDIRECTIONAL); in spacc_aead_free_ddts()
451 dma_pool_free(engine->req_pool, req->src_ddt, req->src_addr); in spacc_aead_free_ddts()
452 dma_pool_free(engine->req_pool, req->dst_ddt, req->dst_addr); in spacc_aead_free_ddts()
462 dev_err(req->engine->dev, "Invalid numbers of SG.\n"); in spacc_free_ddt()
466 dma_unmap_sg(req->engine->dev, payload, nents, dir); in spacc_free_ddt()
467 dma_pool_free(req->engine->req_pool, ddt, ddt_addr); in spacc_free_ddt()
577 struct spacc_engine *engine = ctx->generic.engine; in spacc_aead_submit() local
586 writel(req->src_addr, engine->regs + SPA_SRC_PTR_REG_OFFSET); in spacc_aead_submit()
587 writel(req->dst_addr, engine->regs + SPA_DST_PTR_REG_OFFSET); in spacc_aead_submit()
588 writel(0, engine->regs + SPA_OFFSET_REG_OFFSET); in spacc_aead_submit()
600 writel(proc_len, engine->regs + SPA_PROC_LEN_REG_OFFSET); in spacc_aead_submit()
601 writel(assoc_len, engine->regs + SPA_AAD_LEN_REG_OFFSET); in spacc_aead_submit()
602 writel(authsize, engine->regs + SPA_ICV_LEN_REG_OFFSET); in spacc_aead_submit()
603 writel(0, engine->regs + SPA_ICV_OFFSET_REG_OFFSET); in spacc_aead_submit()
604 writel(0, engine->regs + SPA_AUX_INFO_REG_OFFSET); in spacc_aead_submit()
613 mod_timer(&engine->packet_timeout, jiffies + PACKET_TIMEOUT); in spacc_aead_submit()
615 writel(ctrl, engine->regs + SPA_CTRL_REG_OFFSET); in spacc_aead_submit()
622 static void spacc_push(struct spacc_engine *engine) in spacc_push() argument
626 while (!list_empty(&engine->pending) && in spacc_push()
627 engine->in_flight + 1 <= engine->fifo_sz) { in spacc_push()
629 ++engine->in_flight; in spacc_push()
630 req = list_first_entry(&engine->pending, struct spacc_req, in spacc_push()
632 list_move_tail(&req->list, &engine->in_progress); in spacc_push()
639 * Setup an AEAD request for processing. This will configure the engine, load
647 struct spacc_engine *engine = to_spacc_aead(alg)->engine; in spacc_aead_setup() local
655 dev_req->engine = engine; in spacc_aead_setup()
666 spin_lock_irqsave(&engine->hw_lock, flags); in spacc_aead_setup()
667 if (unlikely(spacc_fifo_cmd_full(engine)) || in spacc_aead_setup()
668 engine->in_flight + 1 > engine->fifo_sz) { in spacc_aead_setup()
671 spin_unlock_irqrestore(&engine->hw_lock, flags); in spacc_aead_setup()
674 list_add_tail(&dev_req->list, &engine->pending); in spacc_aead_setup()
676 list_add_tail(&dev_req->list, &engine->pending); in spacc_aead_setup()
677 spacc_push(engine); in spacc_aead_setup()
679 spin_unlock_irqrestore(&engine->hw_lock, flags); in spacc_aead_setup()
714 struct spacc_engine *engine = spacc_alg->engine; in spacc_aead_cra_init() local
717 ctx->generic.engine = engine; in spacc_aead_cra_init()
790 * IPSec engine only supports 128 and 256 bit AES keys. If we get a in spacc_aes_setkey()
884 struct spacc_engine *engine = ctx->generic.engine; in spacc_ablk_submit() local
891 writel(req->src_addr, engine->regs + SPA_SRC_PTR_REG_OFFSET); in spacc_ablk_submit()
892 writel(req->dst_addr, engine->regs + SPA_DST_PTR_REG_OFFSET); in spacc_ablk_submit()
893 writel(0, engine->regs + SPA_OFFSET_REG_OFFSET); in spacc_ablk_submit()
895 writel(ablk_req->nbytes, engine->regs + SPA_PROC_LEN_REG_OFFSET); in spacc_ablk_submit()
896 writel(0, engine->regs + SPA_ICV_OFFSET_REG_OFFSET); in spacc_ablk_submit()
897 writel(0, engine->regs + SPA_AUX_INFO_REG_OFFSET); in spacc_ablk_submit()
898 writel(0, engine->regs + SPA_AAD_LEN_REG_OFFSET); in spacc_ablk_submit()
904 mod_timer(&engine->packet_timeout, jiffies + PACKET_TIMEOUT); in spacc_ablk_submit()
906 writel(ctrl, engine->regs + SPA_CTRL_REG_OFFSET); in spacc_ablk_submit()
940 struct spacc_engine *engine = to_spacc_alg(alg)->engine; in spacc_ablk_setup() local
947 dev_req->engine = engine; in spacc_ablk_setup()
955 * Create the DDT's for the engine. If we share the same source and in spacc_ablk_setup()
959 dev_req->src_ddt = spacc_sg_to_ddt(engine, req->src, in spacc_ablk_setup()
964 dev_req->dst_ddt = spacc_sg_to_ddt(engine, req->dst, in spacc_ablk_setup()
969 dev_req->dst_ddt = spacc_sg_to_ddt(engine, req->dst, in spacc_ablk_setup()
979 spin_lock_irqsave(&engine->hw_lock, flags); in spacc_ablk_setup()
981 * Check if the engine will accept the operation now. If it won't then in spacc_ablk_setup()
985 if (unlikely(spacc_fifo_cmd_full(engine)) || in spacc_ablk_setup()
986 engine->in_flight + 1 > engine->fifo_sz) { in spacc_ablk_setup()
989 spin_unlock_irqrestore(&engine->hw_lock, flags); in spacc_ablk_setup()
992 list_add_tail(&dev_req->list, &engine->pending); in spacc_ablk_setup()
994 list_add_tail(&dev_req->list, &engine->pending); in spacc_ablk_setup()
995 spacc_push(engine); in spacc_ablk_setup()
997 spin_unlock_irqrestore(&engine->hw_lock, flags); in spacc_ablk_setup()
1018 struct spacc_engine *engine = spacc_alg->engine; in spacc_ablk_cra_init() local
1021 ctx->generic.engine = engine; in spacc_ablk_cra_init()
1027 dev_warn(engine->dev, "failed to allocate fallback for %s\n", in spacc_ablk_cra_init()
1065 static inline int spacc_fifo_stat_empty(struct spacc_engine *engine) in spacc_fifo_stat_empty() argument
1067 return readl(engine->regs + SPA_FIFO_STAT_REG_OFFSET) & in spacc_fifo_stat_empty()
1071 static void spacc_process_done(struct spacc_engine *engine) in spacc_process_done() argument
1076 spin_lock_irqsave(&engine->hw_lock, flags); in spacc_process_done()
1078 while (!spacc_fifo_stat_empty(engine)) { in spacc_process_done()
1079 req = list_first_entry(&engine->in_progress, struct spacc_req, in spacc_process_done()
1081 list_move_tail(&req->list, &engine->completed); in spacc_process_done()
1082 --engine->in_flight; in spacc_process_done()
1085 writel(~0, engine->regs + SPA_STAT_POP_REG_OFFSET); in spacc_process_done()
1086 req->result = (readl(engine->regs + SPA_STATUS_REG_OFFSET) & in spacc_process_done()
1100 dev_warn(engine->dev, in spacc_process_done()
1106 dev_warn(engine->dev, in spacc_process_done()
1114 tasklet_schedule(&engine->complete); in spacc_process_done()
1116 spin_unlock_irqrestore(&engine->hw_lock, flags); in spacc_process_done()
1121 struct spacc_engine *engine = (struct spacc_engine *)dev; in spacc_spacc_irq() local
1122 u32 spacc_irq_stat = readl(engine->regs + SPA_IRQ_STAT_REG_OFFSET); in spacc_spacc_irq()
1124 writel(spacc_irq_stat, engine->regs + SPA_IRQ_STAT_REG_OFFSET); in spacc_spacc_irq()
1125 spacc_process_done(engine); in spacc_spacc_irq()
1132 struct spacc_engine *engine = from_timer(engine, t, packet_timeout); in spacc_packet_timeout() local
1134 spacc_process_done(engine); in spacc_packet_timeout()
1149 struct spacc_engine *engine = (struct spacc_engine *)data; in spacc_spacc_complete() local
1154 spin_lock_irqsave(&engine->hw_lock, flags); in spacc_spacc_complete()
1156 list_splice_init(&engine->completed, &completed); in spacc_spacc_complete()
1157 spacc_push(engine); in spacc_spacc_complete()
1158 if (engine->in_flight) in spacc_spacc_complete()
1159 mod_timer(&engine->packet_timeout, jiffies + PACKET_TIMEOUT); in spacc_spacc_complete()
1161 spin_unlock_irqrestore(&engine->hw_lock, flags); in spacc_spacc_complete()
1172 struct spacc_engine *engine = dev_get_drvdata(dev); in spacc_suspend() local
1179 clk_disable(engine->clk); in spacc_suspend()
1186 struct spacc_engine *engine = dev_get_drvdata(dev); in spacc_resume() local
1188 return clk_enable(engine->clk); in spacc_resume()
1206 struct spacc_engine *engine = spacc_dev_to_engine(dev); in spacc_stat_irq_thresh_show() local
1208 return snprintf(buf, PAGE_SIZE, "%u\n", engine->stat_irq_thresh); in spacc_stat_irq_thresh_show()
1215 struct spacc_engine *engine = spacc_dev_to_engine(dev); in spacc_stat_irq_thresh_store() local
1221 thresh = clamp(thresh, 1UL, engine->fifo_sz - 1); in spacc_stat_irq_thresh_store()
1223 engine->stat_irq_thresh = thresh; in spacc_stat_irq_thresh_store()
1224 writel(engine->stat_irq_thresh << SPA_IRQ_CTRL_STAT_CNT_OFFSET, in spacc_stat_irq_thresh_store()
1225 engine->regs + SPA_IRQ_CTRL_REG_OFFSET); in spacc_stat_irq_thresh_store()
1629 struct spacc_engine *engine = devm_kzalloc(&pdev->dev, sizeof(*engine), in spacc_probe() local
1631 if (!engine) in spacc_probe()
1635 engine->max_ctxs = SPACC_CRYPTO_IPSEC_MAX_CTXS; in spacc_probe()
1636 engine->cipher_pg_sz = SPACC_CRYPTO_IPSEC_CIPHER_PG_SZ; in spacc_probe()
1637 engine->hash_pg_sz = SPACC_CRYPTO_IPSEC_HASH_PG_SZ; in spacc_probe()
1638 engine->fifo_sz = SPACC_CRYPTO_IPSEC_FIFO_SZ; in spacc_probe()
1639 engine->algs = ipsec_engine_algs; in spacc_probe()
1640 engine->num_algs = ARRAY_SIZE(ipsec_engine_algs); in spacc_probe()
1641 engine->aeads = ipsec_engine_aeads; in spacc_probe()
1642 engine->num_aeads = ARRAY_SIZE(ipsec_engine_aeads); in spacc_probe()
1644 engine->max_ctxs = SPACC_CRYPTO_L2_MAX_CTXS; in spacc_probe()
1645 engine->cipher_pg_sz = SPACC_CRYPTO_L2_CIPHER_PG_SZ; in spacc_probe()
1646 engine->hash_pg_sz = SPACC_CRYPTO_L2_HASH_PG_SZ; in spacc_probe()
1647 engine->fifo_sz = SPACC_CRYPTO_L2_FIFO_SZ; in spacc_probe()
1648 engine->algs = l2_engine_algs; in spacc_probe()
1649 engine->num_algs = ARRAY_SIZE(l2_engine_algs); in spacc_probe()
1654 engine->name = dev_name(&pdev->dev); in spacc_probe()
1657 engine->regs = devm_ioremap_resource(&pdev->dev, mem); in spacc_probe()
1658 if (IS_ERR(engine->regs)) in spacc_probe()
1659 return PTR_ERR(engine->regs); in spacc_probe()
1663 dev_err(&pdev->dev, "no memory/irq resource for engine\n"); in spacc_probe()
1667 tasklet_init(&engine->complete, spacc_spacc_complete, in spacc_probe()
1668 (unsigned long)engine); in spacc_probe()
1671 &engine->complete); in spacc_probe()
1676 engine->name, engine)) { in spacc_probe()
1677 dev_err(engine->dev, "failed to request IRQ\n"); in spacc_probe()
1681 engine->dev = &pdev->dev; in spacc_probe()
1682 engine->cipher_ctx_base = engine->regs + SPA_CIPH_KEY_BASE_REG_OFFSET; in spacc_probe()
1683 engine->hash_key_base = engine->regs + SPA_HASH_KEY_BASE_REG_OFFSET; in spacc_probe()
1685 engine->req_pool = dmam_pool_create(engine->name, engine->dev, in spacc_probe()
1687 if (!engine->req_pool) in spacc_probe()
1690 spin_lock_init(&engine->hw_lock); in spacc_probe()
1692 engine->clk = clk_get(&pdev->dev, "ref"); in spacc_probe()
1693 if (IS_ERR(engine->clk)) { in spacc_probe()
1695 return PTR_ERR(engine->clk); in spacc_probe()
1698 if (clk_prepare_enable(engine->clk)) { in spacc_probe()
1709 engine->stat_irq_thresh = (engine->fifo_sz / 2); in spacc_probe()
1720 writel(engine->stat_irq_thresh << SPA_IRQ_CTRL_STAT_CNT_OFFSET, in spacc_probe()
1721 engine->regs + SPA_IRQ_CTRL_REG_OFFSET); in spacc_probe()
1723 engine->regs + SPA_IRQ_EN_REG_OFFSET); in spacc_probe()
1725 timer_setup(&engine->packet_timeout, spacc_packet_timeout, 0); in spacc_probe()
1727 INIT_LIST_HEAD(&engine->pending); in spacc_probe()
1728 INIT_LIST_HEAD(&engine->completed); in spacc_probe()
1729 INIT_LIST_HEAD(&engine->in_progress); in spacc_probe()
1730 engine->in_flight = 0; in spacc_probe()
1732 platform_set_drvdata(pdev, engine); in spacc_probe()
1735 INIT_LIST_HEAD(&engine->registered_algs); in spacc_probe()
1736 for (i = 0; i < engine->num_algs; ++i) { in spacc_probe()
1737 engine->algs[i].engine = engine; in spacc_probe()
1738 err = crypto_register_alg(&engine->algs[i].alg); in spacc_probe()
1740 list_add_tail(&engine->algs[i].entry, in spacc_probe()
1741 &engine->registered_algs); in spacc_probe()
1745 dev_err(engine->dev, "failed to register alg \"%s\"\n", in spacc_probe()
1746 engine->algs[i].alg.cra_name); in spacc_probe()
1748 dev_dbg(engine->dev, "registered alg \"%s\"\n", in spacc_probe()
1749 engine->algs[i].alg.cra_name); in spacc_probe()
1752 INIT_LIST_HEAD(&engine->registered_aeads); in spacc_probe()
1753 for (i = 0; i < engine->num_aeads; ++i) { in spacc_probe()
1754 engine->aeads[i].engine = engine; in spacc_probe()
1755 err = crypto_register_aead(&engine->aeads[i].alg); in spacc_probe()
1757 list_add_tail(&engine->aeads[i].entry, in spacc_probe()
1758 &engine->registered_aeads); in spacc_probe()
1762 dev_err(engine->dev, "failed to register alg \"%s\"\n", in spacc_probe()
1763 engine->aeads[i].alg.base.cra_name); in spacc_probe()
1765 dev_dbg(engine->dev, "registered alg \"%s\"\n", in spacc_probe()
1766 engine->aeads[i].alg.base.cra_name); in spacc_probe()
1772 del_timer_sync(&engine->packet_timeout); in spacc_probe()
1775 clk_disable_unprepare(engine->clk); in spacc_probe()
1777 clk_put(engine->clk); in spacc_probe()
1786 struct spacc_engine *engine = platform_get_drvdata(pdev); in spacc_remove() local
1788 del_timer_sync(&engine->packet_timeout); in spacc_remove()
1791 list_for_each_entry_safe(aead, an, &engine->registered_aeads, entry) { in spacc_remove()
1796 list_for_each_entry_safe(alg, next, &engine->registered_algs, entry) { in spacc_remove()
1801 clk_disable_unprepare(engine->clk); in spacc_remove()
1802 clk_put(engine->clk); in spacc_remove()