Lines Matching refs:dd
46 #define AES_REG_KEY(dd, x) ((dd)->pdata->key_ofs - \ argument
48 #define AES_REG_IV(dd, x) ((dd)->pdata->iv_ofs + ((x) * 0x04)) argument
50 #define AES_REG_CTRL(dd) ((dd)->pdata->ctrl_ofs) argument
63 #define AES_REG_DATA_N(dd, x) ((dd)->pdata->data_ofs + ((x) * 0x04)) argument
65 #define AES_REG_REV(dd) ((dd)->pdata->rev_ofs) argument
67 #define AES_REG_MASK(dd) ((dd)->pdata->mask_ofs) argument
90 struct omap_aes_dev *dd; member
114 void (*trigger)(struct omap_aes_dev *dd, int length);
175 static inline u32 omap_aes_read(struct omap_aes_dev *dd, u32 offset) in omap_aes_read() argument
177 return __raw_readl(dd->io_base + offset); in omap_aes_read()
180 static inline void omap_aes_write(struct omap_aes_dev *dd, u32 offset, in omap_aes_write() argument
183 __raw_writel(value, dd->io_base + offset); in omap_aes_write()
186 static inline void omap_aes_write_mask(struct omap_aes_dev *dd, u32 offset, in omap_aes_write_mask() argument
191 val = omap_aes_read(dd, offset); in omap_aes_write_mask()
194 omap_aes_write(dd, offset, val); in omap_aes_write_mask()
197 static void omap_aes_write_n(struct omap_aes_dev *dd, u32 offset, in omap_aes_write_n() argument
201 omap_aes_write(dd, offset, *value); in omap_aes_write_n()
204 static int omap_aes_hw_init(struct omap_aes_dev *dd) in omap_aes_hw_init() argument
211 pm_runtime_get_sync(dd->dev); in omap_aes_hw_init()
213 if (!(dd->flags & FLAGS_INIT)) { in omap_aes_hw_init()
214 dd->flags |= FLAGS_INIT; in omap_aes_hw_init()
215 dd->err = 0; in omap_aes_hw_init()
221 static int omap_aes_write_ctrl(struct omap_aes_dev *dd) in omap_aes_write_ctrl() argument
227 err = omap_aes_hw_init(dd); in omap_aes_write_ctrl()
231 key32 = dd->ctx->keylen / sizeof(u32); in omap_aes_write_ctrl()
235 omap_aes_write(dd, AES_REG_KEY(dd, i), in omap_aes_write_ctrl()
236 __le32_to_cpu(dd->ctx->key[i])); in omap_aes_write_ctrl()
239 if ((dd->flags & (FLAGS_CBC | FLAGS_CTR)) && dd->req->info) in omap_aes_write_ctrl()
240 omap_aes_write_n(dd, AES_REG_IV(dd, 0), dd->req->info, 4); in omap_aes_write_ctrl()
242 val = FLD_VAL(((dd->ctx->keylen >> 3) - 1), 4, 3); in omap_aes_write_ctrl()
243 if (dd->flags & FLAGS_CBC) in omap_aes_write_ctrl()
245 if (dd->flags & FLAGS_CTR) { in omap_aes_write_ctrl()
249 if (dd->flags & FLAGS_ENCRYPT) in omap_aes_write_ctrl()
255 omap_aes_write_mask(dd, AES_REG_CTRL(dd), val, mask); in omap_aes_write_ctrl()
260 static void omap_aes_dma_trigger_omap2(struct omap_aes_dev *dd, int length) in omap_aes_dma_trigger_omap2() argument
264 val = dd->pdata->dma_start; in omap_aes_dma_trigger_omap2()
266 if (dd->dma_lch_out != NULL) in omap_aes_dma_trigger_omap2()
267 val |= dd->pdata->dma_enable_out; in omap_aes_dma_trigger_omap2()
268 if (dd->dma_lch_in != NULL) in omap_aes_dma_trigger_omap2()
269 val |= dd->pdata->dma_enable_in; in omap_aes_dma_trigger_omap2()
271 mask = dd->pdata->dma_enable_out | dd->pdata->dma_enable_in | in omap_aes_dma_trigger_omap2()
272 dd->pdata->dma_start; in omap_aes_dma_trigger_omap2()
274 omap_aes_write_mask(dd, AES_REG_MASK(dd), val, mask); in omap_aes_dma_trigger_omap2()
278 static void omap_aes_dma_trigger_omap4(struct omap_aes_dev *dd, int length) in omap_aes_dma_trigger_omap4() argument
280 omap_aes_write(dd, AES_REG_LENGTH_N(0), length); in omap_aes_dma_trigger_omap4()
281 omap_aes_write(dd, AES_REG_LENGTH_N(1), 0); in omap_aes_dma_trigger_omap4()
283 omap_aes_dma_trigger_omap2(dd, length); in omap_aes_dma_trigger_omap4()
286 static void omap_aes_dma_stop(struct omap_aes_dev *dd) in omap_aes_dma_stop() argument
290 mask = dd->pdata->dma_enable_out | dd->pdata->dma_enable_in | in omap_aes_dma_stop()
291 dd->pdata->dma_start; in omap_aes_dma_stop()
293 omap_aes_write_mask(dd, AES_REG_MASK(dd), 0, mask); in omap_aes_dma_stop()
298 struct omap_aes_dev *dd = NULL, *tmp; in omap_aes_find_dev() local
301 if (!ctx->dd) { in omap_aes_find_dev()
304 dd = tmp; in omap_aes_find_dev()
307 ctx->dd = dd; in omap_aes_find_dev()
310 dd = ctx->dd; in omap_aes_find_dev()
314 return dd; in omap_aes_find_dev()
319 struct omap_aes_dev *dd = data; in omap_aes_dma_out_callback() local
322 tasklet_schedule(&dd->done_task); in omap_aes_dma_out_callback()
325 static int omap_aes_dma_init(struct omap_aes_dev *dd) in omap_aes_dma_init() argument
330 dd->dma_lch_out = NULL; in omap_aes_dma_init()
331 dd->dma_lch_in = NULL; in omap_aes_dma_init()
333 dd->buf_in = (void *)__get_free_pages(GFP_KERNEL, OMAP_AES_CACHE_SIZE); in omap_aes_dma_init()
334 dd->buf_out = (void *)__get_free_pages(GFP_KERNEL, OMAP_AES_CACHE_SIZE); in omap_aes_dma_init()
335 dd->buflen = PAGE_SIZE << OMAP_AES_CACHE_SIZE; in omap_aes_dma_init()
336 dd->buflen &= ~(AES_BLOCK_SIZE - 1); in omap_aes_dma_init()
338 if (!dd->buf_in || !dd->buf_out) { in omap_aes_dma_init()
339 dev_err(dd->dev, "unable to alloc pages.\n"); in omap_aes_dma_init()
344 dd->dma_addr_in = dma_map_single(dd->dev, dd->buf_in, dd->buflen, in omap_aes_dma_init()
346 if (dma_mapping_error(dd->dev, dd->dma_addr_in)) { in omap_aes_dma_init()
347 dev_err(dd->dev, "dma %d bytes error\n", dd->buflen); in omap_aes_dma_init()
352 dd->dma_addr_out = dma_map_single(dd->dev, dd->buf_out, dd->buflen, in omap_aes_dma_init()
354 if (dma_mapping_error(dd->dev, dd->dma_addr_out)) { in omap_aes_dma_init()
355 dev_err(dd->dev, "dma %d bytes error\n", dd->buflen); in omap_aes_dma_init()
363 dd->dma_lch_in = dma_request_slave_channel_compat(mask, in omap_aes_dma_init()
365 &dd->dma_in, in omap_aes_dma_init()
366 dd->dev, "rx"); in omap_aes_dma_init()
367 if (!dd->dma_lch_in) { in omap_aes_dma_init()
368 dev_err(dd->dev, "Unable to request in DMA channel\n"); in omap_aes_dma_init()
372 dd->dma_lch_out = dma_request_slave_channel_compat(mask, in omap_aes_dma_init()
374 &dd->dma_out, in omap_aes_dma_init()
375 dd->dev, "tx"); in omap_aes_dma_init()
376 if (!dd->dma_lch_out) { in omap_aes_dma_init()
377 dev_err(dd->dev, "Unable to request out DMA channel\n"); in omap_aes_dma_init()
384 dma_release_channel(dd->dma_lch_in); in omap_aes_dma_init()
386 dma_unmap_single(dd->dev, dd->dma_addr_out, dd->buflen, in omap_aes_dma_init()
389 dma_unmap_single(dd->dev, dd->dma_addr_in, dd->buflen, DMA_TO_DEVICE); in omap_aes_dma_init()
391 free_pages((unsigned long)dd->buf_out, OMAP_AES_CACHE_SIZE); in omap_aes_dma_init()
392 free_pages((unsigned long)dd->buf_in, OMAP_AES_CACHE_SIZE); in omap_aes_dma_init()
399 static void omap_aes_dma_cleanup(struct omap_aes_dev *dd) in omap_aes_dma_cleanup() argument
401 dma_release_channel(dd->dma_lch_out); in omap_aes_dma_cleanup()
402 dma_release_channel(dd->dma_lch_in); in omap_aes_dma_cleanup()
403 dma_unmap_single(dd->dev, dd->dma_addr_out, dd->buflen, in omap_aes_dma_cleanup()
405 dma_unmap_single(dd->dev, dd->dma_addr_in, dd->buflen, DMA_TO_DEVICE); in omap_aes_dma_cleanup()
406 free_pages((unsigned long)dd->buf_out, OMAP_AES_CACHE_SIZE); in omap_aes_dma_cleanup()
407 free_pages((unsigned long)dd->buf_in, OMAP_AES_CACHE_SIZE); in omap_aes_dma_cleanup()
464 struct omap_aes_dev *dd = ctx->dd; in omap_aes_crypt_dma() local
472 dd->dma_size = length; in omap_aes_crypt_dma()
474 if (!(dd->flags & FLAGS_FAST)) in omap_aes_crypt_dma()
475 dma_sync_single_for_device(dd->dev, dma_addr_in, length, in omap_aes_crypt_dma()
480 cfg.src_addr = dd->phys_base + AES_REG_DATA_N(dd, 0); in omap_aes_crypt_dma()
481 cfg.dst_addr = dd->phys_base + AES_REG_DATA_N(dd, 0); in omap_aes_crypt_dma()
488 ret = dmaengine_slave_config(dd->dma_lch_in, &cfg); in omap_aes_crypt_dma()
490 dev_err(dd->dev, "can't configure IN dmaengine slave: %d\n", in omap_aes_crypt_dma()
495 tx_in = dmaengine_prep_slave_sg(dd->dma_lch_in, in_sg, 1, in omap_aes_crypt_dma()
499 dev_err(dd->dev, "IN prep_slave_sg() failed\n"); in omap_aes_crypt_dma()
504 tx_in->callback_param = dd; in omap_aes_crypt_dma()
507 ret = dmaengine_slave_config(dd->dma_lch_out, &cfg); in omap_aes_crypt_dma()
509 dev_err(dd->dev, "can't configure OUT dmaengine slave: %d\n", in omap_aes_crypt_dma()
514 tx_out = dmaengine_prep_slave_sg(dd->dma_lch_out, out_sg, 1, in omap_aes_crypt_dma()
518 dev_err(dd->dev, "OUT prep_slave_sg() failed\n"); in omap_aes_crypt_dma()
523 tx_out->callback_param = dd; in omap_aes_crypt_dma()
528 dma_async_issue_pending(dd->dma_lch_in); in omap_aes_crypt_dma()
529 dma_async_issue_pending(dd->dma_lch_out); in omap_aes_crypt_dma()
532 dd->pdata->trigger(dd, length); in omap_aes_crypt_dma()
537 static int omap_aes_crypt_dma_start(struct omap_aes_dev *dd) in omap_aes_crypt_dma_start() argument
540 crypto_ablkcipher_reqtfm(dd->req)); in omap_aes_crypt_dma_start()
547 pr_debug("total: %d\n", dd->total); in omap_aes_crypt_dma_start()
549 if (sg_is_last(dd->in_sg) && sg_is_last(dd->out_sg)) { in omap_aes_crypt_dma_start()
551 in = IS_ALIGNED((u32)dd->in_sg->offset, sizeof(u32)); in omap_aes_crypt_dma_start()
552 out = IS_ALIGNED((u32)dd->out_sg->offset, sizeof(u32)); in omap_aes_crypt_dma_start()
558 count = min(dd->total, sg_dma_len(dd->in_sg)); in omap_aes_crypt_dma_start()
559 count = min(count, sg_dma_len(dd->out_sg)); in omap_aes_crypt_dma_start()
561 if (count != dd->total) { in omap_aes_crypt_dma_start()
568 err = dma_map_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE); in omap_aes_crypt_dma_start()
570 dev_err(dd->dev, "dma_map_sg() error\n"); in omap_aes_crypt_dma_start()
574 err = dma_map_sg(dd->dev, dd->out_sg, 1, DMA_FROM_DEVICE); in omap_aes_crypt_dma_start()
576 dev_err(dd->dev, "dma_map_sg() error\n"); in omap_aes_crypt_dma_start()
577 dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE); in omap_aes_crypt_dma_start()
581 addr_in = sg_dma_address(dd->in_sg); in omap_aes_crypt_dma_start()
582 addr_out = sg_dma_address(dd->out_sg); in omap_aes_crypt_dma_start()
584 in_sg = dd->in_sg; in omap_aes_crypt_dma_start()
585 out_sg = dd->out_sg; in omap_aes_crypt_dma_start()
587 dd->flags |= FLAGS_FAST; in omap_aes_crypt_dma_start()
591 count = sg_copy(&dd->in_sg, &dd->in_offset, dd->buf_in, in omap_aes_crypt_dma_start()
592 dd->buflen, dd->total, 0); in omap_aes_crypt_dma_start()
602 sg_init_table(&dd->in_sgl, 1); in omap_aes_crypt_dma_start()
603 dd->in_sgl.offset = dd->in_offset; in omap_aes_crypt_dma_start()
604 sg_dma_len(&dd->in_sgl) = len32; in omap_aes_crypt_dma_start()
605 sg_dma_address(&dd->in_sgl) = dd->dma_addr_in; in omap_aes_crypt_dma_start()
607 sg_init_table(&dd->out_sgl, 1); in omap_aes_crypt_dma_start()
608 dd->out_sgl.offset = dd->out_offset; in omap_aes_crypt_dma_start()
609 sg_dma_len(&dd->out_sgl) = len32; in omap_aes_crypt_dma_start()
610 sg_dma_address(&dd->out_sgl) = dd->dma_addr_out; in omap_aes_crypt_dma_start()
612 in_sg = &dd->in_sgl; in omap_aes_crypt_dma_start()
613 out_sg = &dd->out_sgl; in omap_aes_crypt_dma_start()
615 addr_in = dd->dma_addr_in; in omap_aes_crypt_dma_start()
616 addr_out = dd->dma_addr_out; in omap_aes_crypt_dma_start()
618 dd->flags &= ~FLAGS_FAST; in omap_aes_crypt_dma_start()
622 dd->total -= count; in omap_aes_crypt_dma_start()
626 dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE); in omap_aes_crypt_dma_start()
627 dma_unmap_sg(dd->dev, dd->out_sg, 1, DMA_TO_DEVICE); in omap_aes_crypt_dma_start()
633 static void omap_aes_finish_req(struct omap_aes_dev *dd, int err) in omap_aes_finish_req() argument
635 struct ablkcipher_request *req = dd->req; in omap_aes_finish_req()
639 pm_runtime_put(dd->dev); in omap_aes_finish_req()
640 dd->flags &= ~FLAGS_BUSY; in omap_aes_finish_req()
645 static int omap_aes_crypt_dma_stop(struct omap_aes_dev *dd) in omap_aes_crypt_dma_stop() argument
650 pr_debug("total: %d\n", dd->total); in omap_aes_crypt_dma_stop()
652 omap_aes_dma_stop(dd); in omap_aes_crypt_dma_stop()
654 dmaengine_terminate_all(dd->dma_lch_in); in omap_aes_crypt_dma_stop()
655 dmaengine_terminate_all(dd->dma_lch_out); in omap_aes_crypt_dma_stop()
657 if (dd->flags & FLAGS_FAST) { in omap_aes_crypt_dma_stop()
658 dma_unmap_sg(dd->dev, dd->out_sg, 1, DMA_FROM_DEVICE); in omap_aes_crypt_dma_stop()
659 dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE); in omap_aes_crypt_dma_stop()
661 dma_sync_single_for_device(dd->dev, dd->dma_addr_out, in omap_aes_crypt_dma_stop()
662 dd->dma_size, DMA_FROM_DEVICE); in omap_aes_crypt_dma_stop()
665 count = sg_copy(&dd->out_sg, &dd->out_offset, dd->buf_out, in omap_aes_crypt_dma_stop()
666 dd->buflen, dd->dma_size, 1); in omap_aes_crypt_dma_stop()
667 if (count != dd->dma_size) { in omap_aes_crypt_dma_stop()
676 static int omap_aes_handle_queue(struct omap_aes_dev *dd, in omap_aes_handle_queue() argument
685 spin_lock_irqsave(&dd->lock, flags); in omap_aes_handle_queue()
687 ret = ablkcipher_enqueue_request(&dd->queue, req); in omap_aes_handle_queue()
688 if (dd->flags & FLAGS_BUSY) { in omap_aes_handle_queue()
689 spin_unlock_irqrestore(&dd->lock, flags); in omap_aes_handle_queue()
692 backlog = crypto_get_backlog(&dd->queue); in omap_aes_handle_queue()
693 async_req = crypto_dequeue_request(&dd->queue); in omap_aes_handle_queue()
695 dd->flags |= FLAGS_BUSY; in omap_aes_handle_queue()
696 spin_unlock_irqrestore(&dd->lock, flags); in omap_aes_handle_queue()
707 dd->req = req; in omap_aes_handle_queue()
708 dd->total = req->nbytes; in omap_aes_handle_queue()
709 dd->in_offset = 0; in omap_aes_handle_queue()
710 dd->in_sg = req->src; in omap_aes_handle_queue()
711 dd->out_offset = 0; in omap_aes_handle_queue()
712 dd->out_sg = req->dst; in omap_aes_handle_queue()
717 dd->flags = (dd->flags & ~FLAGS_MODE_MASK) | rctx->mode; in omap_aes_handle_queue()
719 dd->ctx = ctx; in omap_aes_handle_queue()
720 ctx->dd = dd; in omap_aes_handle_queue()
722 err = omap_aes_write_ctrl(dd); in omap_aes_handle_queue()
724 err = omap_aes_crypt_dma_start(dd); in omap_aes_handle_queue()
727 omap_aes_finish_req(dd, err); in omap_aes_handle_queue()
728 tasklet_schedule(&dd->queue_task); in omap_aes_handle_queue()
736 struct omap_aes_dev *dd = (struct omap_aes_dev *)data; in omap_aes_done_task() local
741 err = omap_aes_crypt_dma_stop(dd); in omap_aes_done_task()
743 err = dd->err ? : err; in omap_aes_done_task()
745 if (dd->total && !err) { in omap_aes_done_task()
746 err = omap_aes_crypt_dma_start(dd); in omap_aes_done_task()
751 omap_aes_finish_req(dd, err); in omap_aes_done_task()
752 omap_aes_handle_queue(dd, NULL); in omap_aes_done_task()
759 struct omap_aes_dev *dd = (struct omap_aes_dev *)data; in omap_aes_queue_task() local
761 omap_aes_handle_queue(dd, NULL); in omap_aes_queue_task()
769 struct omap_aes_dev *dd; in omap_aes_crypt() local
780 dd = omap_aes_find_dev(ctx); in omap_aes_crypt()
781 if (!dd) in omap_aes_crypt()
786 return omap_aes_handle_queue(dd, req); in omap_aes_crypt()
1021 static int omap_aes_get_res_of(struct omap_aes_dev *dd, in omap_aes_get_res_of() argument
1042 dd->dma_out = -1; /* Dummy value that's unused */ in omap_aes_get_res_of()
1043 dd->dma_in = -1; /* Dummy value that's unused */ in omap_aes_get_res_of()
1045 dd->pdata = match->data; in omap_aes_get_res_of()
1055 static int omap_aes_get_res_of(struct omap_aes_dev *dd, in omap_aes_get_res_of() argument
1062 static int omap_aes_get_res_pdev(struct omap_aes_dev *dd, in omap_aes_get_res_pdev() argument
1085 dd->dma_out = r->start; in omap_aes_get_res_pdev()
1094 dd->dma_in = r->start; in omap_aes_get_res_pdev()
1097 dd->pdata = &omap_aes_pdata_omap2; in omap_aes_get_res_pdev()
1106 struct omap_aes_dev *dd; in omap_aes_probe() local
1112 dd = kzalloc(sizeof(struct omap_aes_dev), GFP_KERNEL); in omap_aes_probe()
1113 if (dd == NULL) { in omap_aes_probe()
1117 dd->dev = dev; in omap_aes_probe()
1118 platform_set_drvdata(pdev, dd); in omap_aes_probe()
1120 spin_lock_init(&dd->lock); in omap_aes_probe()
1121 crypto_init_queue(&dd->queue, OMAP_AES_QUEUE_LENGTH); in omap_aes_probe()
1123 err = (dev->of_node) ? omap_aes_get_res_of(dd, dev, &res) : in omap_aes_probe()
1124 omap_aes_get_res_pdev(dd, pdev, &res); in omap_aes_probe()
1128 dd->io_base = devm_request_and_ioremap(dev, &res); in omap_aes_probe()
1129 if (!dd->io_base) { in omap_aes_probe()
1134 dd->phys_base = res.start; in omap_aes_probe()
1139 omap_aes_dma_stop(dd); in omap_aes_probe()
1141 reg = omap_aes_read(dd, AES_REG_REV(dd)); in omap_aes_probe()
1146 (reg & dd->pdata->major_mask) >> dd->pdata->major_shift, in omap_aes_probe()
1147 (reg & dd->pdata->minor_mask) >> dd->pdata->minor_shift); in omap_aes_probe()
1149 tasklet_init(&dd->done_task, omap_aes_done_task, (unsigned long)dd); in omap_aes_probe()
1150 tasklet_init(&dd->queue_task, omap_aes_queue_task, (unsigned long)dd); in omap_aes_probe()
1152 err = omap_aes_dma_init(dd); in omap_aes_probe()
1156 INIT_LIST_HEAD(&dd->list); in omap_aes_probe()
1158 list_add_tail(&dd->list, &dev_list); in omap_aes_probe()
1161 for (i = 0; i < dd->pdata->algs_info_size; i++) { in omap_aes_probe()
1162 for (j = 0; j < dd->pdata->algs_info[i].size; j++) { in omap_aes_probe()
1163 algp = &dd->pdata->algs_info[i].algs_list[j]; in omap_aes_probe()
1172 dd->pdata->algs_info[i].registered++; in omap_aes_probe()
1178 for (i = dd->pdata->algs_info_size - 1; i >= 0; i--) in omap_aes_probe()
1179 for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--) in omap_aes_probe()
1181 &dd->pdata->algs_info[i].algs_list[j]); in omap_aes_probe()
1182 omap_aes_dma_cleanup(dd); in omap_aes_probe()
1184 tasklet_kill(&dd->done_task); in omap_aes_probe()
1185 tasklet_kill(&dd->queue_task); in omap_aes_probe()
1188 kfree(dd); in omap_aes_probe()
1189 dd = NULL; in omap_aes_probe()
1197 struct omap_aes_dev *dd = platform_get_drvdata(pdev); in omap_aes_remove() local
1200 if (!dd) in omap_aes_remove()
1204 list_del(&dd->list); in omap_aes_remove()
1207 for (i = dd->pdata->algs_info_size - 1; i >= 0; i--) in omap_aes_remove()
1208 for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--) in omap_aes_remove()
1210 &dd->pdata->algs_info[i].algs_list[j]); in omap_aes_remove()
1212 tasklet_kill(&dd->done_task); in omap_aes_remove()
1213 tasklet_kill(&dd->queue_task); in omap_aes_remove()
1214 omap_aes_dma_cleanup(dd); in omap_aes_remove()
1215 pm_runtime_disable(dd->dev); in omap_aes_remove()
1216 kfree(dd); in omap_aes_remove()
1217 dd = NULL; in omap_aes_remove()