1 /*
2 * Cryptographic API.
3 *
4 * Support for Samsung S5PV210 HW acceleration.
5 *
6 * Copyright (C) 2011 NetUP Inc. All rights reserved.
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License version 2 as published
10 * by the Free Software Foundation.
11 *
12 */
13
14 #include <linux/clk.h>
15 #include <linux/crypto.h>
16 #include <linux/dma-mapping.h>
17 #include <linux/err.h>
18 #include <linux/errno.h>
19 #include <linux/init.h>
20 #include <linux/interrupt.h>
21 #include <linux/io.h>
22 #include <linux/kernel.h>
23 #include <linux/module.h>
24 #include <linux/of.h>
25 #include <linux/platform_device.h>
26 #include <linux/scatterlist.h>
27
28 #include <crypto/ctr.h>
29 #include <crypto/aes.h>
30 #include <crypto/algapi.h>
31 #include <crypto/scatterwalk.h>
32
33 #define _SBF(s, v) ((v) << (s))
34
35 /* Feed control registers */
36 #define SSS_REG_FCINTSTAT 0x0000
37 #define SSS_FCINTSTAT_BRDMAINT BIT(3)
38 #define SSS_FCINTSTAT_BTDMAINT BIT(2)
39 #define SSS_FCINTSTAT_HRDMAINT BIT(1)
40 #define SSS_FCINTSTAT_PKDMAINT BIT(0)
41
42 #define SSS_REG_FCINTENSET 0x0004
43 #define SSS_FCINTENSET_BRDMAINTENSET BIT(3)
44 #define SSS_FCINTENSET_BTDMAINTENSET BIT(2)
45 #define SSS_FCINTENSET_HRDMAINTENSET BIT(1)
46 #define SSS_FCINTENSET_PKDMAINTENSET BIT(0)
47
48 #define SSS_REG_FCINTENCLR 0x0008
49 #define SSS_FCINTENCLR_BRDMAINTENCLR BIT(3)
50 #define SSS_FCINTENCLR_BTDMAINTENCLR BIT(2)
51 #define SSS_FCINTENCLR_HRDMAINTENCLR BIT(1)
52 #define SSS_FCINTENCLR_PKDMAINTENCLR BIT(0)
53
54 #define SSS_REG_FCINTPEND 0x000C
55 #define SSS_FCINTPEND_BRDMAINTP BIT(3)
56 #define SSS_FCINTPEND_BTDMAINTP BIT(2)
57 #define SSS_FCINTPEND_HRDMAINTP BIT(1)
58 #define SSS_FCINTPEND_PKDMAINTP BIT(0)
59
60 #define SSS_REG_FCFIFOSTAT 0x0010
61 #define SSS_FCFIFOSTAT_BRFIFOFUL BIT(7)
62 #define SSS_FCFIFOSTAT_BRFIFOEMP BIT(6)
63 #define SSS_FCFIFOSTAT_BTFIFOFUL BIT(5)
64 #define SSS_FCFIFOSTAT_BTFIFOEMP BIT(4)
65 #define SSS_FCFIFOSTAT_HRFIFOFUL BIT(3)
66 #define SSS_FCFIFOSTAT_HRFIFOEMP BIT(2)
67 #define SSS_FCFIFOSTAT_PKFIFOFUL BIT(1)
68 #define SSS_FCFIFOSTAT_PKFIFOEMP BIT(0)
69
70 #define SSS_REG_FCFIFOCTRL 0x0014
71 #define SSS_FCFIFOCTRL_DESSEL BIT(2)
72 #define SSS_HASHIN_INDEPENDENT _SBF(0, 0x00)
73 #define SSS_HASHIN_CIPHER_INPUT _SBF(0, 0x01)
74 #define SSS_HASHIN_CIPHER_OUTPUT _SBF(0, 0x02)
75
76 #define SSS_REG_FCBRDMAS 0x0020
77 #define SSS_REG_FCBRDMAL 0x0024
78 #define SSS_REG_FCBRDMAC 0x0028
79 #define SSS_FCBRDMAC_BYTESWAP BIT(1)
80 #define SSS_FCBRDMAC_FLUSH BIT(0)
81
82 #define SSS_REG_FCBTDMAS 0x0030
83 #define SSS_REG_FCBTDMAL 0x0034
84 #define SSS_REG_FCBTDMAC 0x0038
85 #define SSS_FCBTDMAC_BYTESWAP BIT(1)
86 #define SSS_FCBTDMAC_FLUSH BIT(0)
87
88 #define SSS_REG_FCHRDMAS 0x0040
89 #define SSS_REG_FCHRDMAL 0x0044
90 #define SSS_REG_FCHRDMAC 0x0048
91 #define SSS_FCHRDMAC_BYTESWAP BIT(1)
92 #define SSS_FCHRDMAC_FLUSH BIT(0)
93
94 #define SSS_REG_FCPKDMAS 0x0050
95 #define SSS_REG_FCPKDMAL 0x0054
96 #define SSS_REG_FCPKDMAC 0x0058
97 #define SSS_FCPKDMAC_BYTESWAP BIT(3)
98 #define SSS_FCPKDMAC_DESCEND BIT(2)
99 #define SSS_FCPKDMAC_TRANSMIT BIT(1)
100 #define SSS_FCPKDMAC_FLUSH BIT(0)
101
102 #define SSS_REG_FCPKDMAO 0x005C
103
104 /* AES registers */
105 #define SSS_REG_AES_CONTROL 0x00
106 #define SSS_AES_BYTESWAP_DI BIT(11)
107 #define SSS_AES_BYTESWAP_DO BIT(10)
108 #define SSS_AES_BYTESWAP_IV BIT(9)
109 #define SSS_AES_BYTESWAP_CNT BIT(8)
110 #define SSS_AES_BYTESWAP_KEY BIT(7)
111 #define SSS_AES_KEY_CHANGE_MODE BIT(6)
112 #define SSS_AES_KEY_SIZE_128 _SBF(4, 0x00)
113 #define SSS_AES_KEY_SIZE_192 _SBF(4, 0x01)
114 #define SSS_AES_KEY_SIZE_256 _SBF(4, 0x02)
115 #define SSS_AES_FIFO_MODE BIT(3)
116 #define SSS_AES_CHAIN_MODE_ECB _SBF(1, 0x00)
117 #define SSS_AES_CHAIN_MODE_CBC _SBF(1, 0x01)
118 #define SSS_AES_CHAIN_MODE_CTR _SBF(1, 0x02)
119 #define SSS_AES_MODE_DECRYPT BIT(0)
120
121 #define SSS_REG_AES_STATUS 0x04
122 #define SSS_AES_BUSY BIT(2)
123 #define SSS_AES_INPUT_READY BIT(1)
124 #define SSS_AES_OUTPUT_READY BIT(0)
125
126 #define SSS_REG_AES_IN_DATA(s) (0x10 + (s << 2))
127 #define SSS_REG_AES_OUT_DATA(s) (0x20 + (s << 2))
128 #define SSS_REG_AES_IV_DATA(s) (0x30 + (s << 2))
129 #define SSS_REG_AES_CNT_DATA(s) (0x40 + (s << 2))
130 #define SSS_REG_AES_KEY_DATA(s) (0x80 + (s << 2))
131
132 #define SSS_REG(dev, reg) ((dev)->ioaddr + (SSS_REG_##reg))
133 #define SSS_READ(dev, reg) __raw_readl(SSS_REG(dev, reg))
134 #define SSS_WRITE(dev, reg, val) __raw_writel((val), SSS_REG(dev, reg))
135
136 #define SSS_AES_REG(dev, reg) ((dev)->aes_ioaddr + SSS_REG_##reg)
137 #define SSS_AES_WRITE(dev, reg, val) __raw_writel((val), \
138 SSS_AES_REG(dev, reg))
139
140 /* HW engine modes */
141 #define FLAGS_AES_DECRYPT BIT(0)
142 #define FLAGS_AES_MODE_MASK _SBF(1, 0x03)
143 #define FLAGS_AES_CBC _SBF(1, 0x01)
144 #define FLAGS_AES_CTR _SBF(1, 0x02)
145
146 #define AES_KEY_LEN 16
147 #define CRYPTO_QUEUE_LEN 1
148
149 /**
150 * struct samsung_aes_variant - platform specific SSS driver data
151 * @aes_offset: AES register offset from SSS module's base.
152 *
153 * Specifies platform specific configuration of SSS module.
154 * Note: A structure for driver specific platform data is used for future
155 * expansion of its usage.
156 */
157 struct samsung_aes_variant {
158 unsigned int aes_offset;
159 };
160
161 struct s5p_aes_reqctx {
162 unsigned long mode;
163 };
164
165 struct s5p_aes_ctx {
166 struct s5p_aes_dev *dev;
167
168 uint8_t aes_key[AES_MAX_KEY_SIZE];
169 uint8_t nonce[CTR_RFC3686_NONCE_SIZE];
170 int keylen;
171 };
172
173 /**
174 * struct s5p_aes_dev - Crypto device state container
175 * @dev: Associated device
176 * @clk: Clock for accessing hardware
177 * @ioaddr: Mapped IO memory region
178 * @aes_ioaddr: Per-varian offset for AES block IO memory
179 * @irq_fc: Feed control interrupt line
180 * @req: Crypto request currently handled by the device
181 * @ctx: Configuration for currently handled crypto request
182 * @sg_src: Scatter list with source data for currently handled block
183 * in device. This is DMA-mapped into device.
184 * @sg_dst: Scatter list with destination data for currently handled block
185 * in device. This is DMA-mapped into device.
186 * @sg_src_cpy: In case of unaligned access, copied scatter list
187 * with source data.
188 * @sg_dst_cpy: In case of unaligned access, copied scatter list
189 * with destination data.
190 * @tasklet: New request scheduling jib
191 * @queue: Crypto queue
192 * @busy: Indicates whether the device is currently handling some request
193 * thus it uses some of the fields from this state, like:
194 * req, ctx, sg_src/dst (and copies). This essentially
195 * protects against concurrent access to these fields.
196 * @lock: Lock for protecting both access to device hardware registers
197 * and fields related to current request (including the busy field).
198 */
199 struct s5p_aes_dev {
200 struct device *dev;
201 struct clk *clk;
202 void __iomem *ioaddr;
203 void __iomem *aes_ioaddr;
204 int irq_fc;
205
206 struct ablkcipher_request *req;
207 struct s5p_aes_ctx *ctx;
208 struct scatterlist *sg_src;
209 struct scatterlist *sg_dst;
210
211 struct scatterlist *sg_src_cpy;
212 struct scatterlist *sg_dst_cpy;
213
214 struct tasklet_struct tasklet;
215 struct crypto_queue queue;
216 bool busy;
217 spinlock_t lock;
218 };
219
220 static struct s5p_aes_dev *s5p_dev;
221
222 static const struct samsung_aes_variant s5p_aes_data = {
223 .aes_offset = 0x4000,
224 };
225
226 static const struct samsung_aes_variant exynos_aes_data = {
227 .aes_offset = 0x200,
228 };
229
230 static const struct of_device_id s5p_sss_dt_match[] = {
231 {
232 .compatible = "samsung,s5pv210-secss",
233 .data = &s5p_aes_data,
234 },
235 {
236 .compatible = "samsung,exynos4210-secss",
237 .data = &exynos_aes_data,
238 },
239 { },
240 };
241 MODULE_DEVICE_TABLE(of, s5p_sss_dt_match);
242
find_s5p_sss_version(struct platform_device * pdev)243 static inline struct samsung_aes_variant *find_s5p_sss_version
244 (struct platform_device *pdev)
245 {
246 if (IS_ENABLED(CONFIG_OF) && (pdev->dev.of_node)) {
247 const struct of_device_id *match;
248
249 match = of_match_node(s5p_sss_dt_match,
250 pdev->dev.of_node);
251 return (struct samsung_aes_variant *)match->data;
252 }
253 return (struct samsung_aes_variant *)
254 platform_get_device_id(pdev)->driver_data;
255 }
256
s5p_set_dma_indata(struct s5p_aes_dev * dev,struct scatterlist * sg)257 static void s5p_set_dma_indata(struct s5p_aes_dev *dev, struct scatterlist *sg)
258 {
259 SSS_WRITE(dev, FCBRDMAS, sg_dma_address(sg));
260 SSS_WRITE(dev, FCBRDMAL, sg_dma_len(sg));
261 }
262
s5p_set_dma_outdata(struct s5p_aes_dev * dev,struct scatterlist * sg)263 static void s5p_set_dma_outdata(struct s5p_aes_dev *dev, struct scatterlist *sg)
264 {
265 SSS_WRITE(dev, FCBTDMAS, sg_dma_address(sg));
266 SSS_WRITE(dev, FCBTDMAL, sg_dma_len(sg));
267 }
268
s5p_free_sg_cpy(struct s5p_aes_dev * dev,struct scatterlist ** sg)269 static void s5p_free_sg_cpy(struct s5p_aes_dev *dev, struct scatterlist **sg)
270 {
271 int len;
272
273 if (!*sg)
274 return;
275
276 len = ALIGN(dev->req->nbytes, AES_BLOCK_SIZE);
277 free_pages((unsigned long)sg_virt(*sg), get_order(len));
278
279 kfree(*sg);
280 *sg = NULL;
281 }
282
s5p_sg_copy_buf(void * buf,struct scatterlist * sg,unsigned int nbytes,int out)283 static void s5p_sg_copy_buf(void *buf, struct scatterlist *sg,
284 unsigned int nbytes, int out)
285 {
286 struct scatter_walk walk;
287
288 if (!nbytes)
289 return;
290
291 scatterwalk_start(&walk, sg);
292 scatterwalk_copychunks(buf, &walk, nbytes, out);
293 scatterwalk_done(&walk, out, 0);
294 }
295
s5p_sg_done(struct s5p_aes_dev * dev)296 static void s5p_sg_done(struct s5p_aes_dev *dev)
297 {
298 if (dev->sg_dst_cpy) {
299 dev_dbg(dev->dev,
300 "Copying %d bytes of output data back to original place\n",
301 dev->req->nbytes);
302 s5p_sg_copy_buf(sg_virt(dev->sg_dst_cpy), dev->req->dst,
303 dev->req->nbytes, 1);
304 }
305 s5p_free_sg_cpy(dev, &dev->sg_src_cpy);
306 s5p_free_sg_cpy(dev, &dev->sg_dst_cpy);
307 }
308
309 /* Calls the completion. Cannot be called with dev->lock hold. */
s5p_aes_complete(struct s5p_aes_dev * dev,int err)310 static void s5p_aes_complete(struct s5p_aes_dev *dev, int err)
311 {
312 dev->req->base.complete(&dev->req->base, err);
313 }
314
s5p_unset_outdata(struct s5p_aes_dev * dev)315 static void s5p_unset_outdata(struct s5p_aes_dev *dev)
316 {
317 dma_unmap_sg(dev->dev, dev->sg_dst, 1, DMA_FROM_DEVICE);
318 }
319
s5p_unset_indata(struct s5p_aes_dev * dev)320 static void s5p_unset_indata(struct s5p_aes_dev *dev)
321 {
322 dma_unmap_sg(dev->dev, dev->sg_src, 1, DMA_TO_DEVICE);
323 }
324
s5p_make_sg_cpy(struct s5p_aes_dev * dev,struct scatterlist * src,struct scatterlist ** dst)325 static int s5p_make_sg_cpy(struct s5p_aes_dev *dev, struct scatterlist *src,
326 struct scatterlist **dst)
327 {
328 void *pages;
329 int len;
330
331 *dst = kmalloc(sizeof(**dst), GFP_ATOMIC);
332 if (!*dst)
333 return -ENOMEM;
334
335 len = ALIGN(dev->req->nbytes, AES_BLOCK_SIZE);
336 pages = (void *)__get_free_pages(GFP_ATOMIC, get_order(len));
337 if (!pages) {
338 kfree(*dst);
339 *dst = NULL;
340 return -ENOMEM;
341 }
342
343 s5p_sg_copy_buf(pages, src, dev->req->nbytes, 0);
344
345 sg_init_table(*dst, 1);
346 sg_set_buf(*dst, pages, len);
347
348 return 0;
349 }
350
s5p_set_outdata(struct s5p_aes_dev * dev,struct scatterlist * sg)351 static int s5p_set_outdata(struct s5p_aes_dev *dev, struct scatterlist *sg)
352 {
353 int err;
354
355 if (!sg->length) {
356 err = -EINVAL;
357 goto exit;
358 }
359
360 err = dma_map_sg(dev->dev, sg, 1, DMA_FROM_DEVICE);
361 if (!err) {
362 err = -ENOMEM;
363 goto exit;
364 }
365
366 dev->sg_dst = sg;
367 err = 0;
368
369 exit:
370 return err;
371 }
372
s5p_set_indata(struct s5p_aes_dev * dev,struct scatterlist * sg)373 static int s5p_set_indata(struct s5p_aes_dev *dev, struct scatterlist *sg)
374 {
375 int err;
376
377 if (!sg->length) {
378 err = -EINVAL;
379 goto exit;
380 }
381
382 err = dma_map_sg(dev->dev, sg, 1, DMA_TO_DEVICE);
383 if (!err) {
384 err = -ENOMEM;
385 goto exit;
386 }
387
388 dev->sg_src = sg;
389 err = 0;
390
391 exit:
392 return err;
393 }
394
395 /*
396 * Returns -ERRNO on error (mapping of new data failed).
397 * On success returns:
398 * - 0 if there is no more data,
399 * - 1 if new transmitting (output) data is ready and its address+length
400 * have to be written to device (by calling s5p_set_dma_outdata()).
401 */
s5p_aes_tx(struct s5p_aes_dev * dev)402 static int s5p_aes_tx(struct s5p_aes_dev *dev)
403 {
404 int ret = 0;
405
406 s5p_unset_outdata(dev);
407
408 if (!sg_is_last(dev->sg_dst)) {
409 ret = s5p_set_outdata(dev, sg_next(dev->sg_dst));
410 if (!ret)
411 ret = 1;
412 }
413
414 return ret;
415 }
416
417 /*
418 * Returns -ERRNO on error (mapping of new data failed).
419 * On success returns:
420 * - 0 if there is no more data,
421 * - 1 if new receiving (input) data is ready and its address+length
422 * have to be written to device (by calling s5p_set_dma_indata()).
423 */
s5p_aes_rx(struct s5p_aes_dev * dev)424 static int s5p_aes_rx(struct s5p_aes_dev *dev/*, bool *set_dma*/)
425 {
426 int ret = 0;
427
428 s5p_unset_indata(dev);
429
430 if (!sg_is_last(dev->sg_src)) {
431 ret = s5p_set_indata(dev, sg_next(dev->sg_src));
432 if (!ret)
433 ret = 1;
434 }
435
436 return ret;
437 }
438
s5p_aes_interrupt(int irq,void * dev_id)439 static irqreturn_t s5p_aes_interrupt(int irq, void *dev_id)
440 {
441 struct platform_device *pdev = dev_id;
442 struct s5p_aes_dev *dev = platform_get_drvdata(pdev);
443 int err_dma_tx = 0;
444 int err_dma_rx = 0;
445 bool tx_end = false;
446 unsigned long flags;
447 uint32_t status;
448 int err;
449
450 spin_lock_irqsave(&dev->lock, flags);
451
452 /*
453 * Handle rx or tx interrupt. If there is still data (scatterlist did not
454 * reach end), then map next scatterlist entry.
455 * In case of such mapping error, s5p_aes_complete() should be called.
456 *
457 * If there is no more data in tx scatter list, call s5p_aes_complete()
458 * and schedule new tasklet.
459 */
460 status = SSS_READ(dev, FCINTSTAT);
461 if (status & SSS_FCINTSTAT_BRDMAINT)
462 err_dma_rx = s5p_aes_rx(dev);
463
464 if (status & SSS_FCINTSTAT_BTDMAINT) {
465 if (sg_is_last(dev->sg_dst))
466 tx_end = true;
467 err_dma_tx = s5p_aes_tx(dev);
468 }
469
470 SSS_WRITE(dev, FCINTPEND, status);
471
472 if (err_dma_rx < 0) {
473 err = err_dma_rx;
474 goto error;
475 }
476 if (err_dma_tx < 0) {
477 err = err_dma_tx;
478 goto error;
479 }
480
481 if (tx_end) {
482 s5p_sg_done(dev);
483
484 spin_unlock_irqrestore(&dev->lock, flags);
485
486 s5p_aes_complete(dev, 0);
487 /* Device is still busy */
488 tasklet_schedule(&dev->tasklet);
489 } else {
490 /*
491 * Writing length of DMA block (either receiving or
492 * transmitting) will start the operation immediately, so this
493 * should be done at the end (even after clearing pending
494 * interrupts to not miss the interrupt).
495 */
496 if (err_dma_tx == 1)
497 s5p_set_dma_outdata(dev, dev->sg_dst);
498 if (err_dma_rx == 1)
499 s5p_set_dma_indata(dev, dev->sg_src);
500
501 spin_unlock_irqrestore(&dev->lock, flags);
502 }
503
504 return IRQ_HANDLED;
505
506 error:
507 s5p_sg_done(dev);
508 dev->busy = false;
509 spin_unlock_irqrestore(&dev->lock, flags);
510 s5p_aes_complete(dev, err);
511
512 return IRQ_HANDLED;
513 }
514
s5p_set_aes(struct s5p_aes_dev * dev,uint8_t * key,uint8_t * iv,unsigned int keylen)515 static void s5p_set_aes(struct s5p_aes_dev *dev,
516 uint8_t *key, uint8_t *iv, unsigned int keylen)
517 {
518 void __iomem *keystart;
519
520 if (iv)
521 memcpy_toio(dev->aes_ioaddr + SSS_REG_AES_IV_DATA(0), iv, 0x10);
522
523 if (keylen == AES_KEYSIZE_256)
524 keystart = dev->aes_ioaddr + SSS_REG_AES_KEY_DATA(0);
525 else if (keylen == AES_KEYSIZE_192)
526 keystart = dev->aes_ioaddr + SSS_REG_AES_KEY_DATA(2);
527 else
528 keystart = dev->aes_ioaddr + SSS_REG_AES_KEY_DATA(4);
529
530 memcpy_toio(keystart, key, keylen);
531 }
532
s5p_is_sg_aligned(struct scatterlist * sg)533 static bool s5p_is_sg_aligned(struct scatterlist *sg)
534 {
535 while (sg) {
536 if (!IS_ALIGNED(sg->length, AES_BLOCK_SIZE))
537 return false;
538 sg = sg_next(sg);
539 }
540
541 return true;
542 }
543
s5p_set_indata_start(struct s5p_aes_dev * dev,struct ablkcipher_request * req)544 static int s5p_set_indata_start(struct s5p_aes_dev *dev,
545 struct ablkcipher_request *req)
546 {
547 struct scatterlist *sg;
548 int err;
549
550 dev->sg_src_cpy = NULL;
551 sg = req->src;
552 if (!s5p_is_sg_aligned(sg)) {
553 dev_dbg(dev->dev,
554 "At least one unaligned source scatter list, making a copy\n");
555 err = s5p_make_sg_cpy(dev, sg, &dev->sg_src_cpy);
556 if (err)
557 return err;
558
559 sg = dev->sg_src_cpy;
560 }
561
562 err = s5p_set_indata(dev, sg);
563 if (err) {
564 s5p_free_sg_cpy(dev, &dev->sg_src_cpy);
565 return err;
566 }
567
568 return 0;
569 }
570
s5p_set_outdata_start(struct s5p_aes_dev * dev,struct ablkcipher_request * req)571 static int s5p_set_outdata_start(struct s5p_aes_dev *dev,
572 struct ablkcipher_request *req)
573 {
574 struct scatterlist *sg;
575 int err;
576
577 dev->sg_dst_cpy = NULL;
578 sg = req->dst;
579 if (!s5p_is_sg_aligned(sg)) {
580 dev_dbg(dev->dev,
581 "At least one unaligned dest scatter list, making a copy\n");
582 err = s5p_make_sg_cpy(dev, sg, &dev->sg_dst_cpy);
583 if (err)
584 return err;
585
586 sg = dev->sg_dst_cpy;
587 }
588
589 err = s5p_set_outdata(dev, sg);
590 if (err) {
591 s5p_free_sg_cpy(dev, &dev->sg_dst_cpy);
592 return err;
593 }
594
595 return 0;
596 }
597
s5p_aes_crypt_start(struct s5p_aes_dev * dev,unsigned long mode)598 static void s5p_aes_crypt_start(struct s5p_aes_dev *dev, unsigned long mode)
599 {
600 struct ablkcipher_request *req = dev->req;
601 uint32_t aes_control;
602 unsigned long flags;
603 int err;
604 u8 *iv;
605
606 aes_control = SSS_AES_KEY_CHANGE_MODE;
607 if (mode & FLAGS_AES_DECRYPT)
608 aes_control |= SSS_AES_MODE_DECRYPT;
609
610 if ((mode & FLAGS_AES_MODE_MASK) == FLAGS_AES_CBC) {
611 aes_control |= SSS_AES_CHAIN_MODE_CBC;
612 iv = req->info;
613 } else if ((mode & FLAGS_AES_MODE_MASK) == FLAGS_AES_CTR) {
614 aes_control |= SSS_AES_CHAIN_MODE_CTR;
615 iv = req->info;
616 } else {
617 iv = NULL; /* AES_ECB */
618 }
619
620 if (dev->ctx->keylen == AES_KEYSIZE_192)
621 aes_control |= SSS_AES_KEY_SIZE_192;
622 else if (dev->ctx->keylen == AES_KEYSIZE_256)
623 aes_control |= SSS_AES_KEY_SIZE_256;
624
625 aes_control |= SSS_AES_FIFO_MODE;
626
627 /* as a variant it is possible to use byte swapping on DMA side */
628 aes_control |= SSS_AES_BYTESWAP_DI
629 | SSS_AES_BYTESWAP_DO
630 | SSS_AES_BYTESWAP_IV
631 | SSS_AES_BYTESWAP_KEY
632 | SSS_AES_BYTESWAP_CNT;
633
634 spin_lock_irqsave(&dev->lock, flags);
635
636 SSS_WRITE(dev, FCINTENCLR,
637 SSS_FCINTENCLR_BTDMAINTENCLR | SSS_FCINTENCLR_BRDMAINTENCLR);
638 SSS_WRITE(dev, FCFIFOCTRL, 0x00);
639
640 err = s5p_set_indata_start(dev, req);
641 if (err)
642 goto indata_error;
643
644 err = s5p_set_outdata_start(dev, req);
645 if (err)
646 goto outdata_error;
647
648 SSS_AES_WRITE(dev, AES_CONTROL, aes_control);
649 s5p_set_aes(dev, dev->ctx->aes_key, iv, dev->ctx->keylen);
650
651 s5p_set_dma_indata(dev, dev->sg_src);
652 s5p_set_dma_outdata(dev, dev->sg_dst);
653
654 SSS_WRITE(dev, FCINTENSET,
655 SSS_FCINTENSET_BTDMAINTENSET | SSS_FCINTENSET_BRDMAINTENSET);
656
657 spin_unlock_irqrestore(&dev->lock, flags);
658
659 return;
660
661 outdata_error:
662 s5p_unset_indata(dev);
663
664 indata_error:
665 s5p_sg_done(dev);
666 dev->busy = false;
667 spin_unlock_irqrestore(&dev->lock, flags);
668 s5p_aes_complete(dev, err);
669 }
670
s5p_tasklet_cb(unsigned long data)671 static void s5p_tasklet_cb(unsigned long data)
672 {
673 struct s5p_aes_dev *dev = (struct s5p_aes_dev *)data;
674 struct crypto_async_request *async_req, *backlog;
675 struct s5p_aes_reqctx *reqctx;
676 unsigned long flags;
677
678 spin_lock_irqsave(&dev->lock, flags);
679 backlog = crypto_get_backlog(&dev->queue);
680 async_req = crypto_dequeue_request(&dev->queue);
681
682 if (!async_req) {
683 dev->busy = false;
684 spin_unlock_irqrestore(&dev->lock, flags);
685 return;
686 }
687 spin_unlock_irqrestore(&dev->lock, flags);
688
689 if (backlog)
690 backlog->complete(backlog, -EINPROGRESS);
691
692 dev->req = ablkcipher_request_cast(async_req);
693 dev->ctx = crypto_tfm_ctx(dev->req->base.tfm);
694 reqctx = ablkcipher_request_ctx(dev->req);
695
696 s5p_aes_crypt_start(dev, reqctx->mode);
697 }
698
s5p_aes_handle_req(struct s5p_aes_dev * dev,struct ablkcipher_request * req)699 static int s5p_aes_handle_req(struct s5p_aes_dev *dev,
700 struct ablkcipher_request *req)
701 {
702 unsigned long flags;
703 int err;
704
705 spin_lock_irqsave(&dev->lock, flags);
706 err = ablkcipher_enqueue_request(&dev->queue, req);
707 if (dev->busy) {
708 spin_unlock_irqrestore(&dev->lock, flags);
709 goto exit;
710 }
711 dev->busy = true;
712
713 spin_unlock_irqrestore(&dev->lock, flags);
714
715 tasklet_schedule(&dev->tasklet);
716
717 exit:
718 return err;
719 }
720
s5p_aes_crypt(struct ablkcipher_request * req,unsigned long mode)721 static int s5p_aes_crypt(struct ablkcipher_request *req, unsigned long mode)
722 {
723 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
724 struct s5p_aes_reqctx *reqctx = ablkcipher_request_ctx(req);
725 struct s5p_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm);
726 struct s5p_aes_dev *dev = ctx->dev;
727
728 if (!IS_ALIGNED(req->nbytes, AES_BLOCK_SIZE)) {
729 dev_err(dev->dev, "request size is not exact amount of AES blocks\n");
730 return -EINVAL;
731 }
732
733 reqctx->mode = mode;
734
735 return s5p_aes_handle_req(dev, req);
736 }
737
s5p_aes_setkey(struct crypto_ablkcipher * cipher,const uint8_t * key,unsigned int keylen)738 static int s5p_aes_setkey(struct crypto_ablkcipher *cipher,
739 const uint8_t *key, unsigned int keylen)
740 {
741 struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
742 struct s5p_aes_ctx *ctx = crypto_tfm_ctx(tfm);
743
744 if (keylen != AES_KEYSIZE_128 &&
745 keylen != AES_KEYSIZE_192 &&
746 keylen != AES_KEYSIZE_256)
747 return -EINVAL;
748
749 memcpy(ctx->aes_key, key, keylen);
750 ctx->keylen = keylen;
751
752 return 0;
753 }
754
s5p_aes_ecb_encrypt(struct ablkcipher_request * req)755 static int s5p_aes_ecb_encrypt(struct ablkcipher_request *req)
756 {
757 return s5p_aes_crypt(req, 0);
758 }
759
s5p_aes_ecb_decrypt(struct ablkcipher_request * req)760 static int s5p_aes_ecb_decrypt(struct ablkcipher_request *req)
761 {
762 return s5p_aes_crypt(req, FLAGS_AES_DECRYPT);
763 }
764
s5p_aes_cbc_encrypt(struct ablkcipher_request * req)765 static int s5p_aes_cbc_encrypt(struct ablkcipher_request *req)
766 {
767 return s5p_aes_crypt(req, FLAGS_AES_CBC);
768 }
769
s5p_aes_cbc_decrypt(struct ablkcipher_request * req)770 static int s5p_aes_cbc_decrypt(struct ablkcipher_request *req)
771 {
772 return s5p_aes_crypt(req, FLAGS_AES_DECRYPT | FLAGS_AES_CBC);
773 }
774
s5p_aes_cra_init(struct crypto_tfm * tfm)775 static int s5p_aes_cra_init(struct crypto_tfm *tfm)
776 {
777 struct s5p_aes_ctx *ctx = crypto_tfm_ctx(tfm);
778
779 ctx->dev = s5p_dev;
780 tfm->crt_ablkcipher.reqsize = sizeof(struct s5p_aes_reqctx);
781
782 return 0;
783 }
784
785 static struct crypto_alg algs[] = {
786 {
787 .cra_name = "ecb(aes)",
788 .cra_driver_name = "ecb-aes-s5p",
789 .cra_priority = 100,
790 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
791 CRYPTO_ALG_ASYNC |
792 CRYPTO_ALG_KERN_DRIVER_ONLY,
793 .cra_blocksize = AES_BLOCK_SIZE,
794 .cra_ctxsize = sizeof(struct s5p_aes_ctx),
795 .cra_alignmask = 0x0f,
796 .cra_type = &crypto_ablkcipher_type,
797 .cra_module = THIS_MODULE,
798 .cra_init = s5p_aes_cra_init,
799 .cra_u.ablkcipher = {
800 .min_keysize = AES_MIN_KEY_SIZE,
801 .max_keysize = AES_MAX_KEY_SIZE,
802 .setkey = s5p_aes_setkey,
803 .encrypt = s5p_aes_ecb_encrypt,
804 .decrypt = s5p_aes_ecb_decrypt,
805 }
806 },
807 {
808 .cra_name = "cbc(aes)",
809 .cra_driver_name = "cbc-aes-s5p",
810 .cra_priority = 100,
811 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
812 CRYPTO_ALG_ASYNC |
813 CRYPTO_ALG_KERN_DRIVER_ONLY,
814 .cra_blocksize = AES_BLOCK_SIZE,
815 .cra_ctxsize = sizeof(struct s5p_aes_ctx),
816 .cra_alignmask = 0x0f,
817 .cra_type = &crypto_ablkcipher_type,
818 .cra_module = THIS_MODULE,
819 .cra_init = s5p_aes_cra_init,
820 .cra_u.ablkcipher = {
821 .min_keysize = AES_MIN_KEY_SIZE,
822 .max_keysize = AES_MAX_KEY_SIZE,
823 .ivsize = AES_BLOCK_SIZE,
824 .setkey = s5p_aes_setkey,
825 .encrypt = s5p_aes_cbc_encrypt,
826 .decrypt = s5p_aes_cbc_decrypt,
827 }
828 },
829 };
830
s5p_aes_probe(struct platform_device * pdev)831 static int s5p_aes_probe(struct platform_device *pdev)
832 {
833 struct device *dev = &pdev->dev;
834 int i, j, err = -ENODEV;
835 struct samsung_aes_variant *variant;
836 struct s5p_aes_dev *pdata;
837 struct resource *res;
838
839 if (s5p_dev)
840 return -EEXIST;
841
842 pdata = devm_kzalloc(dev, sizeof(*pdata), GFP_KERNEL);
843 if (!pdata)
844 return -ENOMEM;
845
846 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
847 pdata->ioaddr = devm_ioremap_resource(&pdev->dev, res);
848 if (IS_ERR(pdata->ioaddr))
849 return PTR_ERR(pdata->ioaddr);
850
851 variant = find_s5p_sss_version(pdev);
852
853 pdata->clk = devm_clk_get(dev, "secss");
854 if (IS_ERR(pdata->clk)) {
855 dev_err(dev, "failed to find secss clock source\n");
856 return -ENOENT;
857 }
858
859 err = clk_prepare_enable(pdata->clk);
860 if (err < 0) {
861 dev_err(dev, "Enabling SSS clk failed, err %d\n", err);
862 return err;
863 }
864
865 spin_lock_init(&pdata->lock);
866
867 pdata->aes_ioaddr = pdata->ioaddr + variant->aes_offset;
868
869 pdata->irq_fc = platform_get_irq(pdev, 0);
870 if (pdata->irq_fc < 0) {
871 err = pdata->irq_fc;
872 dev_warn(dev, "feed control interrupt is not available.\n");
873 goto err_irq;
874 }
875 err = devm_request_threaded_irq(dev, pdata->irq_fc, NULL,
876 s5p_aes_interrupt, IRQF_ONESHOT,
877 pdev->name, pdev);
878 if (err < 0) {
879 dev_warn(dev, "feed control interrupt is not available.\n");
880 goto err_irq;
881 }
882
883 pdata->busy = false;
884 pdata->dev = dev;
885 platform_set_drvdata(pdev, pdata);
886 s5p_dev = pdata;
887
888 tasklet_init(&pdata->tasklet, s5p_tasklet_cb, (unsigned long)pdata);
889 crypto_init_queue(&pdata->queue, CRYPTO_QUEUE_LEN);
890
891 for (i = 0; i < ARRAY_SIZE(algs); i++) {
892 err = crypto_register_alg(&algs[i]);
893 if (err)
894 goto err_algs;
895 }
896
897 dev_info(dev, "s5p-sss driver registered\n");
898
899 return 0;
900
901 err_algs:
902 dev_err(dev, "can't register '%s': %d\n", algs[i].cra_name, err);
903
904 for (j = 0; j < i; j++)
905 crypto_unregister_alg(&algs[j]);
906
907 tasklet_kill(&pdata->tasklet);
908
909 err_irq:
910 clk_disable_unprepare(pdata->clk);
911
912 s5p_dev = NULL;
913
914 return err;
915 }
916
s5p_aes_remove(struct platform_device * pdev)917 static int s5p_aes_remove(struct platform_device *pdev)
918 {
919 struct s5p_aes_dev *pdata = platform_get_drvdata(pdev);
920 int i;
921
922 if (!pdata)
923 return -ENODEV;
924
925 for (i = 0; i < ARRAY_SIZE(algs); i++)
926 crypto_unregister_alg(&algs[i]);
927
928 tasklet_kill(&pdata->tasklet);
929
930 clk_disable_unprepare(pdata->clk);
931
932 s5p_dev = NULL;
933
934 return 0;
935 }
936
937 static struct platform_driver s5p_aes_crypto = {
938 .probe = s5p_aes_probe,
939 .remove = s5p_aes_remove,
940 .driver = {
941 .name = "s5p-secss",
942 .of_match_table = s5p_sss_dt_match,
943 },
944 };
945
946 module_platform_driver(s5p_aes_crypto);
947
948 MODULE_DESCRIPTION("S5PV210 AES hw acceleration support.");
949 MODULE_LICENSE("GPL v2");
950 MODULE_AUTHOR("Vladimir Zapolskiy <vzapolskiy@gmail.com>");
951