1 /*
2 * AMD Cryptographic Coprocessor (CCP) driver
3 *
4 * Copyright (C) 2013,2016 Advanced Micro Devices, Inc.
5 *
6 * Author: Tom Lendacky <thomas.lendacky@amd.com>
7 * Author: Gary R Hook <gary.hook@amd.com>
8 *
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License version 2 as
11 * published by the Free Software Foundation.
12 */
13
14 #include <linux/module.h>
15 #include <linux/kernel.h>
16 #include <linux/pci.h>
17 #include <linux/interrupt.h>
18 #include <crypto/scatterwalk.h>
19 #include <linux/ccp.h>
20
21 #include "ccp-dev.h"
22
23 /* SHA initial context values */
24 static const __be32 ccp_sha1_init[SHA1_DIGEST_SIZE / sizeof(__be32)] = {
25 cpu_to_be32(SHA1_H0), cpu_to_be32(SHA1_H1),
26 cpu_to_be32(SHA1_H2), cpu_to_be32(SHA1_H3),
27 cpu_to_be32(SHA1_H4),
28 };
29
30 static const __be32 ccp_sha224_init[SHA256_DIGEST_SIZE / sizeof(__be32)] = {
31 cpu_to_be32(SHA224_H0), cpu_to_be32(SHA224_H1),
32 cpu_to_be32(SHA224_H2), cpu_to_be32(SHA224_H3),
33 cpu_to_be32(SHA224_H4), cpu_to_be32(SHA224_H5),
34 cpu_to_be32(SHA224_H6), cpu_to_be32(SHA224_H7),
35 };
36
37 static const __be32 ccp_sha256_init[SHA256_DIGEST_SIZE / sizeof(__be32)] = {
38 cpu_to_be32(SHA256_H0), cpu_to_be32(SHA256_H1),
39 cpu_to_be32(SHA256_H2), cpu_to_be32(SHA256_H3),
40 cpu_to_be32(SHA256_H4), cpu_to_be32(SHA256_H5),
41 cpu_to_be32(SHA256_H6), cpu_to_be32(SHA256_H7),
42 };
43
44 #define CCP_NEW_JOBID(ccp) ((ccp->vdata->version == CCP_VERSION(3, 0)) ? \
45 ccp_gen_jobid(ccp) : 0)
46
ccp_gen_jobid(struct ccp_device * ccp)47 static u32 ccp_gen_jobid(struct ccp_device *ccp)
48 {
49 return atomic_inc_return(&ccp->current_id) & CCP_JOBID_MASK;
50 }
51
ccp_sg_free(struct ccp_sg_workarea * wa)52 static void ccp_sg_free(struct ccp_sg_workarea *wa)
53 {
54 if (wa->dma_count)
55 dma_unmap_sg(wa->dma_dev, wa->dma_sg, wa->nents, wa->dma_dir);
56
57 wa->dma_count = 0;
58 }
59
ccp_init_sg_workarea(struct ccp_sg_workarea * wa,struct device * dev,struct scatterlist * sg,u64 len,enum dma_data_direction dma_dir)60 static int ccp_init_sg_workarea(struct ccp_sg_workarea *wa, struct device *dev,
61 struct scatterlist *sg, u64 len,
62 enum dma_data_direction dma_dir)
63 {
64 memset(wa, 0, sizeof(*wa));
65
66 wa->sg = sg;
67 if (!sg)
68 return 0;
69
70 wa->nents = sg_nents_for_len(sg, len);
71 if (wa->nents < 0)
72 return wa->nents;
73
74 wa->bytes_left = len;
75 wa->sg_used = 0;
76
77 if (len == 0)
78 return 0;
79
80 if (dma_dir == DMA_NONE)
81 return 0;
82
83 wa->dma_sg = sg;
84 wa->dma_dev = dev;
85 wa->dma_dir = dma_dir;
86 wa->dma_count = dma_map_sg(dev, sg, wa->nents, dma_dir);
87 if (!wa->dma_count)
88 return -ENOMEM;
89
90 return 0;
91 }
92
ccp_update_sg_workarea(struct ccp_sg_workarea * wa,unsigned int len)93 static void ccp_update_sg_workarea(struct ccp_sg_workarea *wa, unsigned int len)
94 {
95 unsigned int nbytes = min_t(u64, len, wa->bytes_left);
96
97 if (!wa->sg)
98 return;
99
100 wa->sg_used += nbytes;
101 wa->bytes_left -= nbytes;
102 if (wa->sg_used == wa->sg->length) {
103 wa->sg = sg_next(wa->sg);
104 wa->sg_used = 0;
105 }
106 }
107
ccp_dm_free(struct ccp_dm_workarea * wa)108 static void ccp_dm_free(struct ccp_dm_workarea *wa)
109 {
110 if (wa->length <= CCP_DMAPOOL_MAX_SIZE) {
111 if (wa->address)
112 dma_pool_free(wa->dma_pool, wa->address,
113 wa->dma.address);
114 } else {
115 if (wa->dma.address)
116 dma_unmap_single(wa->dev, wa->dma.address, wa->length,
117 wa->dma.dir);
118 kfree(wa->address);
119 }
120
121 wa->address = NULL;
122 wa->dma.address = 0;
123 }
124
ccp_init_dm_workarea(struct ccp_dm_workarea * wa,struct ccp_cmd_queue * cmd_q,unsigned int len,enum dma_data_direction dir)125 static int ccp_init_dm_workarea(struct ccp_dm_workarea *wa,
126 struct ccp_cmd_queue *cmd_q,
127 unsigned int len,
128 enum dma_data_direction dir)
129 {
130 memset(wa, 0, sizeof(*wa));
131
132 if (!len)
133 return 0;
134
135 wa->dev = cmd_q->ccp->dev;
136 wa->length = len;
137
138 if (len <= CCP_DMAPOOL_MAX_SIZE) {
139 wa->dma_pool = cmd_q->dma_pool;
140
141 wa->address = dma_pool_alloc(wa->dma_pool, GFP_KERNEL,
142 &wa->dma.address);
143 if (!wa->address)
144 return -ENOMEM;
145
146 wa->dma.length = CCP_DMAPOOL_MAX_SIZE;
147
148 memset(wa->address, 0, CCP_DMAPOOL_MAX_SIZE);
149 } else {
150 wa->address = kzalloc(len, GFP_KERNEL);
151 if (!wa->address)
152 return -ENOMEM;
153
154 wa->dma.address = dma_map_single(wa->dev, wa->address, len,
155 dir);
156 if (!wa->dma.address)
157 return -ENOMEM;
158
159 wa->dma.length = len;
160 }
161 wa->dma.dir = dir;
162
163 return 0;
164 }
165
ccp_set_dm_area(struct ccp_dm_workarea * wa,unsigned int wa_offset,struct scatterlist * sg,unsigned int sg_offset,unsigned int len)166 static void ccp_set_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset,
167 struct scatterlist *sg, unsigned int sg_offset,
168 unsigned int len)
169 {
170 WARN_ON(!wa->address);
171
172 scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len,
173 0);
174 }
175
ccp_get_dm_area(struct ccp_dm_workarea * wa,unsigned int wa_offset,struct scatterlist * sg,unsigned int sg_offset,unsigned int len)176 static void ccp_get_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset,
177 struct scatterlist *sg, unsigned int sg_offset,
178 unsigned int len)
179 {
180 WARN_ON(!wa->address);
181
182 scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len,
183 1);
184 }
185
ccp_reverse_set_dm_area(struct ccp_dm_workarea * wa,struct scatterlist * sg,unsigned int len,unsigned int se_len,bool sign_extend)186 static int ccp_reverse_set_dm_area(struct ccp_dm_workarea *wa,
187 struct scatterlist *sg,
188 unsigned int len, unsigned int se_len,
189 bool sign_extend)
190 {
191 unsigned int nbytes, sg_offset, dm_offset, sb_len, i;
192 u8 buffer[CCP_REVERSE_BUF_SIZE];
193
194 if (WARN_ON(se_len > sizeof(buffer)))
195 return -EINVAL;
196
197 sg_offset = len;
198 dm_offset = 0;
199 nbytes = len;
200 while (nbytes) {
201 sb_len = min_t(unsigned int, nbytes, se_len);
202 sg_offset -= sb_len;
203
204 scatterwalk_map_and_copy(buffer, sg, sg_offset, sb_len, 0);
205 for (i = 0; i < sb_len; i++)
206 wa->address[dm_offset + i] = buffer[sb_len - i - 1];
207
208 dm_offset += sb_len;
209 nbytes -= sb_len;
210
211 if ((sb_len != se_len) && sign_extend) {
212 /* Must sign-extend to nearest sign-extend length */
213 if (wa->address[dm_offset - 1] & 0x80)
214 memset(wa->address + dm_offset, 0xff,
215 se_len - sb_len);
216 }
217 }
218
219 return 0;
220 }
221
ccp_reverse_get_dm_area(struct ccp_dm_workarea * wa,struct scatterlist * sg,unsigned int len)222 static void ccp_reverse_get_dm_area(struct ccp_dm_workarea *wa,
223 struct scatterlist *sg,
224 unsigned int len)
225 {
226 unsigned int nbytes, sg_offset, dm_offset, sb_len, i;
227 u8 buffer[CCP_REVERSE_BUF_SIZE];
228
229 sg_offset = 0;
230 dm_offset = len;
231 nbytes = len;
232 while (nbytes) {
233 sb_len = min_t(unsigned int, nbytes, sizeof(buffer));
234 dm_offset -= sb_len;
235
236 for (i = 0; i < sb_len; i++)
237 buffer[sb_len - i - 1] = wa->address[dm_offset + i];
238 scatterwalk_map_and_copy(buffer, sg, sg_offset, sb_len, 1);
239
240 sg_offset += sb_len;
241 nbytes -= sb_len;
242 }
243 }
244
ccp_free_data(struct ccp_data * data,struct ccp_cmd_queue * cmd_q)245 static void ccp_free_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q)
246 {
247 ccp_dm_free(&data->dm_wa);
248 ccp_sg_free(&data->sg_wa);
249 }
250
ccp_init_data(struct ccp_data * data,struct ccp_cmd_queue * cmd_q,struct scatterlist * sg,u64 sg_len,unsigned int dm_len,enum dma_data_direction dir)251 static int ccp_init_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q,
252 struct scatterlist *sg, u64 sg_len,
253 unsigned int dm_len,
254 enum dma_data_direction dir)
255 {
256 int ret;
257
258 memset(data, 0, sizeof(*data));
259
260 ret = ccp_init_sg_workarea(&data->sg_wa, cmd_q->ccp->dev, sg, sg_len,
261 dir);
262 if (ret)
263 goto e_err;
264
265 ret = ccp_init_dm_workarea(&data->dm_wa, cmd_q, dm_len, dir);
266 if (ret)
267 goto e_err;
268
269 return 0;
270
271 e_err:
272 ccp_free_data(data, cmd_q);
273
274 return ret;
275 }
276
ccp_queue_buf(struct ccp_data * data,unsigned int from)277 static unsigned int ccp_queue_buf(struct ccp_data *data, unsigned int from)
278 {
279 struct ccp_sg_workarea *sg_wa = &data->sg_wa;
280 struct ccp_dm_workarea *dm_wa = &data->dm_wa;
281 unsigned int buf_count, nbytes;
282
283 /* Clear the buffer if setting it */
284 if (!from)
285 memset(dm_wa->address, 0, dm_wa->length);
286
287 if (!sg_wa->sg)
288 return 0;
289
290 /* Perform the copy operation
291 * nbytes will always be <= UINT_MAX because dm_wa->length is
292 * an unsigned int
293 */
294 nbytes = min_t(u64, sg_wa->bytes_left, dm_wa->length);
295 scatterwalk_map_and_copy(dm_wa->address, sg_wa->sg, sg_wa->sg_used,
296 nbytes, from);
297
298 /* Update the structures and generate the count */
299 buf_count = 0;
300 while (sg_wa->bytes_left && (buf_count < dm_wa->length)) {
301 nbytes = min(sg_wa->sg->length - sg_wa->sg_used,
302 dm_wa->length - buf_count);
303 nbytes = min_t(u64, sg_wa->bytes_left, nbytes);
304
305 buf_count += nbytes;
306 ccp_update_sg_workarea(sg_wa, nbytes);
307 }
308
309 return buf_count;
310 }
311
ccp_fill_queue_buf(struct ccp_data * data)312 static unsigned int ccp_fill_queue_buf(struct ccp_data *data)
313 {
314 return ccp_queue_buf(data, 0);
315 }
316
ccp_empty_queue_buf(struct ccp_data * data)317 static unsigned int ccp_empty_queue_buf(struct ccp_data *data)
318 {
319 return ccp_queue_buf(data, 1);
320 }
321
ccp_prepare_data(struct ccp_data * src,struct ccp_data * dst,struct ccp_op * op,unsigned int block_size,bool blocksize_op)322 static void ccp_prepare_data(struct ccp_data *src, struct ccp_data *dst,
323 struct ccp_op *op, unsigned int block_size,
324 bool blocksize_op)
325 {
326 unsigned int sg_src_len, sg_dst_len, op_len;
327
328 /* The CCP can only DMA from/to one address each per operation. This
329 * requires that we find the smallest DMA area between the source
330 * and destination. The resulting len values will always be <= UINT_MAX
331 * because the dma length is an unsigned int.
332 */
333 sg_src_len = sg_dma_len(src->sg_wa.sg) - src->sg_wa.sg_used;
334 sg_src_len = min_t(u64, src->sg_wa.bytes_left, sg_src_len);
335
336 if (dst) {
337 sg_dst_len = sg_dma_len(dst->sg_wa.sg) - dst->sg_wa.sg_used;
338 sg_dst_len = min_t(u64, src->sg_wa.bytes_left, sg_dst_len);
339 op_len = min(sg_src_len, sg_dst_len);
340 } else {
341 op_len = sg_src_len;
342 }
343
344 /* The data operation length will be at least block_size in length
345 * or the smaller of available sg room remaining for the source or
346 * the destination
347 */
348 op_len = max(op_len, block_size);
349
350 /* Unless we have to buffer data, there's no reason to wait */
351 op->soc = 0;
352
353 if (sg_src_len < block_size) {
354 /* Not enough data in the sg element, so it
355 * needs to be buffered into a blocksize chunk
356 */
357 int cp_len = ccp_fill_queue_buf(src);
358
359 op->soc = 1;
360 op->src.u.dma.address = src->dm_wa.dma.address;
361 op->src.u.dma.offset = 0;
362 op->src.u.dma.length = (blocksize_op) ? block_size : cp_len;
363 } else {
364 /* Enough data in the sg element, but we need to
365 * adjust for any previously copied data
366 */
367 op->src.u.dma.address = sg_dma_address(src->sg_wa.sg);
368 op->src.u.dma.offset = src->sg_wa.sg_used;
369 op->src.u.dma.length = op_len & ~(block_size - 1);
370
371 ccp_update_sg_workarea(&src->sg_wa, op->src.u.dma.length);
372 }
373
374 if (dst) {
375 if (sg_dst_len < block_size) {
376 /* Not enough room in the sg element or we're on the
377 * last piece of data (when using padding), so the
378 * output needs to be buffered into a blocksize chunk
379 */
380 op->soc = 1;
381 op->dst.u.dma.address = dst->dm_wa.dma.address;
382 op->dst.u.dma.offset = 0;
383 op->dst.u.dma.length = op->src.u.dma.length;
384 } else {
385 /* Enough room in the sg element, but we need to
386 * adjust for any previously used area
387 */
388 op->dst.u.dma.address = sg_dma_address(dst->sg_wa.sg);
389 op->dst.u.dma.offset = dst->sg_wa.sg_used;
390 op->dst.u.dma.length = op->src.u.dma.length;
391 }
392 }
393 }
394
ccp_process_data(struct ccp_data * src,struct ccp_data * dst,struct ccp_op * op)395 static void ccp_process_data(struct ccp_data *src, struct ccp_data *dst,
396 struct ccp_op *op)
397 {
398 op->init = 0;
399
400 if (dst) {
401 if (op->dst.u.dma.address == dst->dm_wa.dma.address)
402 ccp_empty_queue_buf(dst);
403 else
404 ccp_update_sg_workarea(&dst->sg_wa,
405 op->dst.u.dma.length);
406 }
407 }
408
ccp_copy_to_from_sb(struct ccp_cmd_queue * cmd_q,struct ccp_dm_workarea * wa,u32 jobid,u32 sb,u32 byte_swap,bool from)409 static int ccp_copy_to_from_sb(struct ccp_cmd_queue *cmd_q,
410 struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
411 u32 byte_swap, bool from)
412 {
413 struct ccp_op op;
414
415 memset(&op, 0, sizeof(op));
416
417 op.cmd_q = cmd_q;
418 op.jobid = jobid;
419 op.eom = 1;
420
421 if (from) {
422 op.soc = 1;
423 op.src.type = CCP_MEMTYPE_SB;
424 op.src.u.sb = sb;
425 op.dst.type = CCP_MEMTYPE_SYSTEM;
426 op.dst.u.dma.address = wa->dma.address;
427 op.dst.u.dma.length = wa->length;
428 } else {
429 op.src.type = CCP_MEMTYPE_SYSTEM;
430 op.src.u.dma.address = wa->dma.address;
431 op.src.u.dma.length = wa->length;
432 op.dst.type = CCP_MEMTYPE_SB;
433 op.dst.u.sb = sb;
434 }
435
436 op.u.passthru.byte_swap = byte_swap;
437
438 return cmd_q->ccp->vdata->perform->passthru(&op);
439 }
440
ccp_copy_to_sb(struct ccp_cmd_queue * cmd_q,struct ccp_dm_workarea * wa,u32 jobid,u32 sb,u32 byte_swap)441 static int ccp_copy_to_sb(struct ccp_cmd_queue *cmd_q,
442 struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
443 u32 byte_swap)
444 {
445 return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, false);
446 }
447
ccp_copy_from_sb(struct ccp_cmd_queue * cmd_q,struct ccp_dm_workarea * wa,u32 jobid,u32 sb,u32 byte_swap)448 static int ccp_copy_from_sb(struct ccp_cmd_queue *cmd_q,
449 struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
450 u32 byte_swap)
451 {
452 return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, true);
453 }
454
ccp_run_aes_cmac_cmd(struct ccp_cmd_queue * cmd_q,struct ccp_cmd * cmd)455 static int ccp_run_aes_cmac_cmd(struct ccp_cmd_queue *cmd_q,
456 struct ccp_cmd *cmd)
457 {
458 struct ccp_aes_engine *aes = &cmd->u.aes;
459 struct ccp_dm_workarea key, ctx;
460 struct ccp_data src;
461 struct ccp_op op;
462 unsigned int dm_offset;
463 int ret;
464
465 if (!((aes->key_len == AES_KEYSIZE_128) ||
466 (aes->key_len == AES_KEYSIZE_192) ||
467 (aes->key_len == AES_KEYSIZE_256)))
468 return -EINVAL;
469
470 if (aes->src_len & (AES_BLOCK_SIZE - 1))
471 return -EINVAL;
472
473 if (aes->iv_len != AES_BLOCK_SIZE)
474 return -EINVAL;
475
476 if (!aes->key || !aes->iv || !aes->src)
477 return -EINVAL;
478
479 if (aes->cmac_final) {
480 if (aes->cmac_key_len != AES_BLOCK_SIZE)
481 return -EINVAL;
482
483 if (!aes->cmac_key)
484 return -EINVAL;
485 }
486
487 BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1);
488 BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1);
489
490 ret = -EIO;
491 memset(&op, 0, sizeof(op));
492 op.cmd_q = cmd_q;
493 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
494 op.sb_key = cmd_q->sb_key;
495 op.sb_ctx = cmd_q->sb_ctx;
496 op.init = 1;
497 op.u.aes.type = aes->type;
498 op.u.aes.mode = aes->mode;
499 op.u.aes.action = aes->action;
500
501 /* All supported key sizes fit in a single (32-byte) SB entry
502 * and must be in little endian format. Use the 256-bit byte
503 * swap passthru option to convert from big endian to little
504 * endian.
505 */
506 ret = ccp_init_dm_workarea(&key, cmd_q,
507 CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES,
508 DMA_TO_DEVICE);
509 if (ret)
510 return ret;
511
512 dm_offset = CCP_SB_BYTES - aes->key_len;
513 ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
514 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
515 CCP_PASSTHRU_BYTESWAP_256BIT);
516 if (ret) {
517 cmd->engine_error = cmd_q->cmd_error;
518 goto e_key;
519 }
520
521 /* The AES context fits in a single (32-byte) SB entry and
522 * must be in little endian format. Use the 256-bit byte swap
523 * passthru option to convert from big endian to little endian.
524 */
525 ret = ccp_init_dm_workarea(&ctx, cmd_q,
526 CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
527 DMA_BIDIRECTIONAL);
528 if (ret)
529 goto e_key;
530
531 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
532 ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
533 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
534 CCP_PASSTHRU_BYTESWAP_256BIT);
535 if (ret) {
536 cmd->engine_error = cmd_q->cmd_error;
537 goto e_ctx;
538 }
539
540 /* Send data to the CCP AES engine */
541 ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,
542 AES_BLOCK_SIZE, DMA_TO_DEVICE);
543 if (ret)
544 goto e_ctx;
545
546 while (src.sg_wa.bytes_left) {
547 ccp_prepare_data(&src, NULL, &op, AES_BLOCK_SIZE, true);
548 if (aes->cmac_final && !src.sg_wa.bytes_left) {
549 op.eom = 1;
550
551 /* Push the K1/K2 key to the CCP now */
552 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid,
553 op.sb_ctx,
554 CCP_PASSTHRU_BYTESWAP_256BIT);
555 if (ret) {
556 cmd->engine_error = cmd_q->cmd_error;
557 goto e_src;
558 }
559
560 ccp_set_dm_area(&ctx, 0, aes->cmac_key, 0,
561 aes->cmac_key_len);
562 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
563 CCP_PASSTHRU_BYTESWAP_256BIT);
564 if (ret) {
565 cmd->engine_error = cmd_q->cmd_error;
566 goto e_src;
567 }
568 }
569
570 ret = cmd_q->ccp->vdata->perform->aes(&op);
571 if (ret) {
572 cmd->engine_error = cmd_q->cmd_error;
573 goto e_src;
574 }
575
576 ccp_process_data(&src, NULL, &op);
577 }
578
579 /* Retrieve the AES context - convert from LE to BE using
580 * 32-byte (256-bit) byteswapping
581 */
582 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
583 CCP_PASSTHRU_BYTESWAP_256BIT);
584 if (ret) {
585 cmd->engine_error = cmd_q->cmd_error;
586 goto e_src;
587 }
588
589 /* ...but we only need AES_BLOCK_SIZE bytes */
590 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
591 ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
592
593 e_src:
594 ccp_free_data(&src, cmd_q);
595
596 e_ctx:
597 ccp_dm_free(&ctx);
598
599 e_key:
600 ccp_dm_free(&key);
601
602 return ret;
603 }
604
ccp_run_aes_cmd(struct ccp_cmd_queue * cmd_q,struct ccp_cmd * cmd)605 static int ccp_run_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
606 {
607 struct ccp_aes_engine *aes = &cmd->u.aes;
608 struct ccp_dm_workarea key, ctx;
609 struct ccp_data src, dst;
610 struct ccp_op op;
611 unsigned int dm_offset;
612 bool in_place = false;
613 int ret;
614
615 if (aes->mode == CCP_AES_MODE_CMAC)
616 return ccp_run_aes_cmac_cmd(cmd_q, cmd);
617
618 if (!((aes->key_len == AES_KEYSIZE_128) ||
619 (aes->key_len == AES_KEYSIZE_192) ||
620 (aes->key_len == AES_KEYSIZE_256)))
621 return -EINVAL;
622
623 if (((aes->mode == CCP_AES_MODE_ECB) ||
624 (aes->mode == CCP_AES_MODE_CBC) ||
625 (aes->mode == CCP_AES_MODE_CFB)) &&
626 (aes->src_len & (AES_BLOCK_SIZE - 1)))
627 return -EINVAL;
628
629 if (!aes->key || !aes->src || !aes->dst)
630 return -EINVAL;
631
632 if (aes->mode != CCP_AES_MODE_ECB) {
633 if (aes->iv_len != AES_BLOCK_SIZE)
634 return -EINVAL;
635
636 if (!aes->iv)
637 return -EINVAL;
638 }
639
640 BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1);
641 BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1);
642
643 ret = -EIO;
644 memset(&op, 0, sizeof(op));
645 op.cmd_q = cmd_q;
646 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
647 op.sb_key = cmd_q->sb_key;
648 op.sb_ctx = cmd_q->sb_ctx;
649 op.init = (aes->mode == CCP_AES_MODE_ECB) ? 0 : 1;
650 op.u.aes.type = aes->type;
651 op.u.aes.mode = aes->mode;
652 op.u.aes.action = aes->action;
653
654 /* All supported key sizes fit in a single (32-byte) SB entry
655 * and must be in little endian format. Use the 256-bit byte
656 * swap passthru option to convert from big endian to little
657 * endian.
658 */
659 ret = ccp_init_dm_workarea(&key, cmd_q,
660 CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES,
661 DMA_TO_DEVICE);
662 if (ret)
663 return ret;
664
665 dm_offset = CCP_SB_BYTES - aes->key_len;
666 ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
667 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
668 CCP_PASSTHRU_BYTESWAP_256BIT);
669 if (ret) {
670 cmd->engine_error = cmd_q->cmd_error;
671 goto e_key;
672 }
673
674 /* The AES context fits in a single (32-byte) SB entry and
675 * must be in little endian format. Use the 256-bit byte swap
676 * passthru option to convert from big endian to little endian.
677 */
678 ret = ccp_init_dm_workarea(&ctx, cmd_q,
679 CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
680 DMA_BIDIRECTIONAL);
681 if (ret)
682 goto e_key;
683
684 if (aes->mode != CCP_AES_MODE_ECB) {
685 /* Load the AES context - convert to LE */
686 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
687 ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
688 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
689 CCP_PASSTHRU_BYTESWAP_256BIT);
690 if (ret) {
691 cmd->engine_error = cmd_q->cmd_error;
692 goto e_ctx;
693 }
694 }
695 switch (aes->mode) {
696 case CCP_AES_MODE_CFB: /* CFB128 only */
697 case CCP_AES_MODE_CTR:
698 op.u.aes.size = AES_BLOCK_SIZE * BITS_PER_BYTE - 1;
699 break;
700 default:
701 op.u.aes.size = 0;
702 }
703
704 /* Prepare the input and output data workareas. For in-place
705 * operations we need to set the dma direction to BIDIRECTIONAL
706 * and copy the src workarea to the dst workarea.
707 */
708 if (sg_virt(aes->src) == sg_virt(aes->dst))
709 in_place = true;
710
711 ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,
712 AES_BLOCK_SIZE,
713 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
714 if (ret)
715 goto e_ctx;
716
717 if (in_place) {
718 dst = src;
719 } else {
720 ret = ccp_init_data(&dst, cmd_q, aes->dst, aes->src_len,
721 AES_BLOCK_SIZE, DMA_FROM_DEVICE);
722 if (ret)
723 goto e_src;
724 }
725
726 /* Send data to the CCP AES engine */
727 while (src.sg_wa.bytes_left) {
728 ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true);
729 if (!src.sg_wa.bytes_left) {
730 op.eom = 1;
731
732 /* Since we don't retrieve the AES context in ECB
733 * mode we have to wait for the operation to complete
734 * on the last piece of data
735 */
736 if (aes->mode == CCP_AES_MODE_ECB)
737 op.soc = 1;
738 }
739
740 ret = cmd_q->ccp->vdata->perform->aes(&op);
741 if (ret) {
742 cmd->engine_error = cmd_q->cmd_error;
743 goto e_dst;
744 }
745
746 ccp_process_data(&src, &dst, &op);
747 }
748
749 if (aes->mode != CCP_AES_MODE_ECB) {
750 /* Retrieve the AES context - convert from LE to BE using
751 * 32-byte (256-bit) byteswapping
752 */
753 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
754 CCP_PASSTHRU_BYTESWAP_256BIT);
755 if (ret) {
756 cmd->engine_error = cmd_q->cmd_error;
757 goto e_dst;
758 }
759
760 /* ...but we only need AES_BLOCK_SIZE bytes */
761 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
762 ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
763 }
764
765 e_dst:
766 if (!in_place)
767 ccp_free_data(&dst, cmd_q);
768
769 e_src:
770 ccp_free_data(&src, cmd_q);
771
772 e_ctx:
773 ccp_dm_free(&ctx);
774
775 e_key:
776 ccp_dm_free(&key);
777
778 return ret;
779 }
780
ccp_run_xts_aes_cmd(struct ccp_cmd_queue * cmd_q,struct ccp_cmd * cmd)781 static int ccp_run_xts_aes_cmd(struct ccp_cmd_queue *cmd_q,
782 struct ccp_cmd *cmd)
783 {
784 struct ccp_xts_aes_engine *xts = &cmd->u.xts;
785 struct ccp_dm_workarea key, ctx;
786 struct ccp_data src, dst;
787 struct ccp_op op;
788 unsigned int unit_size, dm_offset;
789 bool in_place = false;
790 unsigned int sb_count;
791 enum ccp_aes_type aestype;
792 int ret;
793
794 switch (xts->unit_size) {
795 case CCP_XTS_AES_UNIT_SIZE_16:
796 unit_size = 16;
797 break;
798 case CCP_XTS_AES_UNIT_SIZE_512:
799 unit_size = 512;
800 break;
801 case CCP_XTS_AES_UNIT_SIZE_1024:
802 unit_size = 1024;
803 break;
804 case CCP_XTS_AES_UNIT_SIZE_2048:
805 unit_size = 2048;
806 break;
807 case CCP_XTS_AES_UNIT_SIZE_4096:
808 unit_size = 4096;
809 break;
810
811 default:
812 return -EINVAL;
813 }
814
815 if (xts->key_len == AES_KEYSIZE_128)
816 aestype = CCP_AES_TYPE_128;
817 else
818 return -EINVAL;
819
820 if (!xts->final && (xts->src_len & (AES_BLOCK_SIZE - 1)))
821 return -EINVAL;
822
823 if (xts->iv_len != AES_BLOCK_SIZE)
824 return -EINVAL;
825
826 if (!xts->key || !xts->iv || !xts->src || !xts->dst)
827 return -EINVAL;
828
829 BUILD_BUG_ON(CCP_XTS_AES_KEY_SB_COUNT != 1);
830 BUILD_BUG_ON(CCP_XTS_AES_CTX_SB_COUNT != 1);
831
832 ret = -EIO;
833 memset(&op, 0, sizeof(op));
834 op.cmd_q = cmd_q;
835 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
836 op.sb_key = cmd_q->sb_key;
837 op.sb_ctx = cmd_q->sb_ctx;
838 op.init = 1;
839 op.u.xts.type = aestype;
840 op.u.xts.action = xts->action;
841 op.u.xts.unit_size = xts->unit_size;
842
843 /* A version 3 device only supports 128-bit keys, which fits into a
844 * single SB entry. A version 5 device uses a 512-bit vector, so two
845 * SB entries.
846 */
847 if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0))
848 sb_count = CCP_XTS_AES_KEY_SB_COUNT;
849 else
850 sb_count = CCP5_XTS_AES_KEY_SB_COUNT;
851 ret = ccp_init_dm_workarea(&key, cmd_q,
852 sb_count * CCP_SB_BYTES,
853 DMA_TO_DEVICE);
854 if (ret)
855 return ret;
856
857 if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) {
858 /* All supported key sizes must be in little endian format.
859 * Use the 256-bit byte swap passthru option to convert from
860 * big endian to little endian.
861 */
862 dm_offset = CCP_SB_BYTES - AES_KEYSIZE_128;
863 ccp_set_dm_area(&key, dm_offset, xts->key, 0, xts->key_len);
864 ccp_set_dm_area(&key, 0, xts->key, xts->key_len, xts->key_len);
865 } else {
866 /* Version 5 CCPs use a 512-bit space for the key: each portion
867 * occupies 256 bits, or one entire slot, and is zero-padded.
868 */
869 unsigned int pad;
870
871 dm_offset = CCP_SB_BYTES;
872 pad = dm_offset - xts->key_len;
873 ccp_set_dm_area(&key, pad, xts->key, 0, xts->key_len);
874 ccp_set_dm_area(&key, dm_offset + pad, xts->key, xts->key_len,
875 xts->key_len);
876 }
877 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
878 CCP_PASSTHRU_BYTESWAP_256BIT);
879 if (ret) {
880 cmd->engine_error = cmd_q->cmd_error;
881 goto e_key;
882 }
883
884 /* The AES context fits in a single (32-byte) SB entry and
885 * for XTS is already in little endian format so no byte swapping
886 * is needed.
887 */
888 ret = ccp_init_dm_workarea(&ctx, cmd_q,
889 CCP_XTS_AES_CTX_SB_COUNT * CCP_SB_BYTES,
890 DMA_BIDIRECTIONAL);
891 if (ret)
892 goto e_key;
893
894 ccp_set_dm_area(&ctx, 0, xts->iv, 0, xts->iv_len);
895 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
896 CCP_PASSTHRU_BYTESWAP_NOOP);
897 if (ret) {
898 cmd->engine_error = cmd_q->cmd_error;
899 goto e_ctx;
900 }
901
902 /* Prepare the input and output data workareas. For in-place
903 * operations we need to set the dma direction to BIDIRECTIONAL
904 * and copy the src workarea to the dst workarea.
905 */
906 if (sg_virt(xts->src) == sg_virt(xts->dst))
907 in_place = true;
908
909 ret = ccp_init_data(&src, cmd_q, xts->src, xts->src_len,
910 unit_size,
911 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
912 if (ret)
913 goto e_ctx;
914
915 if (in_place) {
916 dst = src;
917 } else {
918 ret = ccp_init_data(&dst, cmd_q, xts->dst, xts->src_len,
919 unit_size, DMA_FROM_DEVICE);
920 if (ret)
921 goto e_src;
922 }
923
924 /* Send data to the CCP AES engine */
925 while (src.sg_wa.bytes_left) {
926 ccp_prepare_data(&src, &dst, &op, unit_size, true);
927 if (!src.sg_wa.bytes_left)
928 op.eom = 1;
929
930 ret = cmd_q->ccp->vdata->perform->xts_aes(&op);
931 if (ret) {
932 cmd->engine_error = cmd_q->cmd_error;
933 goto e_dst;
934 }
935
936 ccp_process_data(&src, &dst, &op);
937 }
938
939 /* Retrieve the AES context - convert from LE to BE using
940 * 32-byte (256-bit) byteswapping
941 */
942 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
943 CCP_PASSTHRU_BYTESWAP_256BIT);
944 if (ret) {
945 cmd->engine_error = cmd_q->cmd_error;
946 goto e_dst;
947 }
948
949 /* ...but we only need AES_BLOCK_SIZE bytes */
950 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
951 ccp_get_dm_area(&ctx, dm_offset, xts->iv, 0, xts->iv_len);
952
953 e_dst:
954 if (!in_place)
955 ccp_free_data(&dst, cmd_q);
956
957 e_src:
958 ccp_free_data(&src, cmd_q);
959
960 e_ctx:
961 ccp_dm_free(&ctx);
962
963 e_key:
964 ccp_dm_free(&key);
965
966 return ret;
967 }
968
ccp_run_sha_cmd(struct ccp_cmd_queue * cmd_q,struct ccp_cmd * cmd)969 static int ccp_run_sha_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
970 {
971 struct ccp_sha_engine *sha = &cmd->u.sha;
972 struct ccp_dm_workarea ctx;
973 struct ccp_data src;
974 struct ccp_op op;
975 unsigned int ioffset, ooffset;
976 unsigned int digest_size;
977 int sb_count;
978 const void *init;
979 u64 block_size;
980 int ctx_size;
981 int ret;
982
983 switch (sha->type) {
984 case CCP_SHA_TYPE_1:
985 if (sha->ctx_len < SHA1_DIGEST_SIZE)
986 return -EINVAL;
987 block_size = SHA1_BLOCK_SIZE;
988 break;
989 case CCP_SHA_TYPE_224:
990 if (sha->ctx_len < SHA224_DIGEST_SIZE)
991 return -EINVAL;
992 block_size = SHA224_BLOCK_SIZE;
993 break;
994 case CCP_SHA_TYPE_256:
995 if (sha->ctx_len < SHA256_DIGEST_SIZE)
996 return -EINVAL;
997 block_size = SHA256_BLOCK_SIZE;
998 break;
999 default:
1000 return -EINVAL;
1001 }
1002
1003 if (!sha->ctx)
1004 return -EINVAL;
1005
1006 if (!sha->final && (sha->src_len & (block_size - 1)))
1007 return -EINVAL;
1008
1009 /* The version 3 device can't handle zero-length input */
1010 if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) {
1011
1012 if (!sha->src_len) {
1013 unsigned int digest_len;
1014 const u8 *sha_zero;
1015
1016 /* Not final, just return */
1017 if (!sha->final)
1018 return 0;
1019
1020 /* CCP can't do a zero length sha operation so the
1021 * caller must buffer the data.
1022 */
1023 if (sha->msg_bits)
1024 return -EINVAL;
1025
1026 /* The CCP cannot perform zero-length sha operations
1027 * so the caller is required to buffer data for the
1028 * final operation. However, a sha operation for a
1029 * message with a total length of zero is valid so
1030 * known values are required to supply the result.
1031 */
1032 switch (sha->type) {
1033 case CCP_SHA_TYPE_1:
1034 sha_zero = sha1_zero_message_hash;
1035 digest_len = SHA1_DIGEST_SIZE;
1036 break;
1037 case CCP_SHA_TYPE_224:
1038 sha_zero = sha224_zero_message_hash;
1039 digest_len = SHA224_DIGEST_SIZE;
1040 break;
1041 case CCP_SHA_TYPE_256:
1042 sha_zero = sha256_zero_message_hash;
1043 digest_len = SHA256_DIGEST_SIZE;
1044 break;
1045 default:
1046 return -EINVAL;
1047 }
1048
1049 scatterwalk_map_and_copy((void *)sha_zero, sha->ctx, 0,
1050 digest_len, 1);
1051
1052 return 0;
1053 }
1054 }
1055
1056 /* Set variables used throughout */
1057 switch (sha->type) {
1058 case CCP_SHA_TYPE_1:
1059 digest_size = SHA1_DIGEST_SIZE;
1060 init = (void *) ccp_sha1_init;
1061 ctx_size = SHA1_DIGEST_SIZE;
1062 sb_count = 1;
1063 if (cmd_q->ccp->vdata->version != CCP_VERSION(3, 0))
1064 ooffset = ioffset = CCP_SB_BYTES - SHA1_DIGEST_SIZE;
1065 else
1066 ooffset = ioffset = 0;
1067 break;
1068 case CCP_SHA_TYPE_224:
1069 digest_size = SHA224_DIGEST_SIZE;
1070 init = (void *) ccp_sha224_init;
1071 ctx_size = SHA256_DIGEST_SIZE;
1072 sb_count = 1;
1073 ioffset = 0;
1074 if (cmd_q->ccp->vdata->version != CCP_VERSION(3, 0))
1075 ooffset = CCP_SB_BYTES - SHA224_DIGEST_SIZE;
1076 else
1077 ooffset = 0;
1078 break;
1079 case CCP_SHA_TYPE_256:
1080 digest_size = SHA256_DIGEST_SIZE;
1081 init = (void *) ccp_sha256_init;
1082 ctx_size = SHA256_DIGEST_SIZE;
1083 sb_count = 1;
1084 ooffset = ioffset = 0;
1085 break;
1086 default:
1087 ret = -EINVAL;
1088 goto e_data;
1089 }
1090
1091 /* For zero-length plaintext the src pointer is ignored;
1092 * otherwise both parts must be valid
1093 */
1094 if (sha->src_len && !sha->src)
1095 return -EINVAL;
1096
1097 memset(&op, 0, sizeof(op));
1098 op.cmd_q = cmd_q;
1099 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1100 op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */
1101 op.u.sha.type = sha->type;
1102 op.u.sha.msg_bits = sha->msg_bits;
1103
1104 ret = ccp_init_dm_workarea(&ctx, cmd_q, sb_count * CCP_SB_BYTES,
1105 DMA_BIDIRECTIONAL);
1106 if (ret)
1107 return ret;
1108 if (sha->first) {
1109 switch (sha->type) {
1110 case CCP_SHA_TYPE_1:
1111 case CCP_SHA_TYPE_224:
1112 case CCP_SHA_TYPE_256:
1113 memcpy(ctx.address + ioffset, init, ctx_size);
1114 break;
1115 default:
1116 ret = -EINVAL;
1117 goto e_ctx;
1118 }
1119 } else {
1120 /* Restore the context */
1121 ccp_set_dm_area(&ctx, 0, sha->ctx, 0,
1122 sb_count * CCP_SB_BYTES);
1123 }
1124
1125 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1126 CCP_PASSTHRU_BYTESWAP_256BIT);
1127 if (ret) {
1128 cmd->engine_error = cmd_q->cmd_error;
1129 goto e_ctx;
1130 }
1131
1132 if (sha->src) {
1133 /* Send data to the CCP SHA engine; block_size is set above */
1134 ret = ccp_init_data(&src, cmd_q, sha->src, sha->src_len,
1135 block_size, DMA_TO_DEVICE);
1136 if (ret)
1137 goto e_ctx;
1138
1139 while (src.sg_wa.bytes_left) {
1140 ccp_prepare_data(&src, NULL, &op, block_size, false);
1141 if (sha->final && !src.sg_wa.bytes_left)
1142 op.eom = 1;
1143
1144 ret = cmd_q->ccp->vdata->perform->sha(&op);
1145 if (ret) {
1146 cmd->engine_error = cmd_q->cmd_error;
1147 goto e_data;
1148 }
1149
1150 ccp_process_data(&src, NULL, &op);
1151 }
1152 } else {
1153 op.eom = 1;
1154 ret = cmd_q->ccp->vdata->perform->sha(&op);
1155 if (ret) {
1156 cmd->engine_error = cmd_q->cmd_error;
1157 goto e_data;
1158 }
1159 }
1160
1161 /* Retrieve the SHA context - convert from LE to BE using
1162 * 32-byte (256-bit) byteswapping to BE
1163 */
1164 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1165 CCP_PASSTHRU_BYTESWAP_256BIT);
1166 if (ret) {
1167 cmd->engine_error = cmd_q->cmd_error;
1168 goto e_data;
1169 }
1170
1171 if (sha->final) {
1172 /* Finishing up, so get the digest */
1173 switch (sha->type) {
1174 case CCP_SHA_TYPE_1:
1175 case CCP_SHA_TYPE_224:
1176 case CCP_SHA_TYPE_256:
1177 ccp_get_dm_area(&ctx, ooffset,
1178 sha->ctx, 0,
1179 digest_size);
1180 break;
1181 default:
1182 ret = -EINVAL;
1183 goto e_ctx;
1184 }
1185 } else {
1186 /* Stash the context */
1187 ccp_get_dm_area(&ctx, 0, sha->ctx, 0,
1188 sb_count * CCP_SB_BYTES);
1189 }
1190
1191 if (sha->final && sha->opad) {
1192 /* HMAC operation, recursively perform final SHA */
1193 struct ccp_cmd hmac_cmd;
1194 struct scatterlist sg;
1195 u8 *hmac_buf;
1196
1197 if (sha->opad_len != block_size) {
1198 ret = -EINVAL;
1199 goto e_data;
1200 }
1201
1202 hmac_buf = kmalloc(block_size + digest_size, GFP_KERNEL);
1203 if (!hmac_buf) {
1204 ret = -ENOMEM;
1205 goto e_data;
1206 }
1207 sg_init_one(&sg, hmac_buf, block_size + digest_size);
1208
1209 scatterwalk_map_and_copy(hmac_buf, sha->opad, 0, block_size, 0);
1210 switch (sha->type) {
1211 case CCP_SHA_TYPE_1:
1212 case CCP_SHA_TYPE_224:
1213 case CCP_SHA_TYPE_256:
1214 memcpy(hmac_buf + block_size,
1215 ctx.address + ooffset,
1216 digest_size);
1217 break;
1218 default:
1219 ret = -EINVAL;
1220 goto e_ctx;
1221 }
1222
1223 memset(&hmac_cmd, 0, sizeof(hmac_cmd));
1224 hmac_cmd.engine = CCP_ENGINE_SHA;
1225 hmac_cmd.u.sha.type = sha->type;
1226 hmac_cmd.u.sha.ctx = sha->ctx;
1227 hmac_cmd.u.sha.ctx_len = sha->ctx_len;
1228 hmac_cmd.u.sha.src = &sg;
1229 hmac_cmd.u.sha.src_len = block_size + digest_size;
1230 hmac_cmd.u.sha.opad = NULL;
1231 hmac_cmd.u.sha.opad_len = 0;
1232 hmac_cmd.u.sha.first = 1;
1233 hmac_cmd.u.sha.final = 1;
1234 hmac_cmd.u.sha.msg_bits = (block_size + digest_size) << 3;
1235
1236 ret = ccp_run_sha_cmd(cmd_q, &hmac_cmd);
1237 if (ret)
1238 cmd->engine_error = hmac_cmd.engine_error;
1239
1240 kfree(hmac_buf);
1241 }
1242
1243 e_data:
1244 if (sha->src)
1245 ccp_free_data(&src, cmd_q);
1246
1247 e_ctx:
1248 ccp_dm_free(&ctx);
1249
1250 return ret;
1251 }
1252
ccp_run_rsa_cmd(struct ccp_cmd_queue * cmd_q,struct ccp_cmd * cmd)1253 static int ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1254 {
1255 struct ccp_rsa_engine *rsa = &cmd->u.rsa;
1256 struct ccp_dm_workarea exp, src;
1257 struct ccp_data dst;
1258 struct ccp_op op;
1259 unsigned int sb_count, i_len, o_len;
1260 int ret;
1261
1262 if (rsa->key_size > CCP_RSA_MAX_WIDTH)
1263 return -EINVAL;
1264
1265 if (!rsa->exp || !rsa->mod || !rsa->src || !rsa->dst)
1266 return -EINVAL;
1267
1268 /* The RSA modulus must precede the message being acted upon, so
1269 * it must be copied to a DMA area where the message and the
1270 * modulus can be concatenated. Therefore the input buffer
1271 * length required is twice the output buffer length (which
1272 * must be a multiple of 256-bits).
1273 */
1274 o_len = ((rsa->key_size + 255) / 256) * 32;
1275 i_len = o_len * 2;
1276
1277 sb_count = o_len / CCP_SB_BYTES;
1278
1279 memset(&op, 0, sizeof(op));
1280 op.cmd_q = cmd_q;
1281 op.jobid = ccp_gen_jobid(cmd_q->ccp);
1282 op.sb_key = cmd_q->ccp->vdata->perform->sballoc(cmd_q, sb_count);
1283
1284 if (!op.sb_key)
1285 return -EIO;
1286
1287 /* The RSA exponent may span multiple (32-byte) SB entries and must
1288 * be in little endian format. Reverse copy each 32-byte chunk
1289 * of the exponent (En chunk to E0 chunk, E(n-1) chunk to E1 chunk)
1290 * and each byte within that chunk and do not perform any byte swap
1291 * operations on the passthru operation.
1292 */
1293 ret = ccp_init_dm_workarea(&exp, cmd_q, o_len, DMA_TO_DEVICE);
1294 if (ret)
1295 goto e_sb;
1296
1297 ret = ccp_reverse_set_dm_area(&exp, rsa->exp, rsa->exp_len,
1298 CCP_SB_BYTES, false);
1299 if (ret)
1300 goto e_exp;
1301 ret = ccp_copy_to_sb(cmd_q, &exp, op.jobid, op.sb_key,
1302 CCP_PASSTHRU_BYTESWAP_NOOP);
1303 if (ret) {
1304 cmd->engine_error = cmd_q->cmd_error;
1305 goto e_exp;
1306 }
1307
1308 /* Concatenate the modulus and the message. Both the modulus and
1309 * the operands must be in little endian format. Since the input
1310 * is in big endian format it must be converted.
1311 */
1312 ret = ccp_init_dm_workarea(&src, cmd_q, i_len, DMA_TO_DEVICE);
1313 if (ret)
1314 goto e_exp;
1315
1316 ret = ccp_reverse_set_dm_area(&src, rsa->mod, rsa->mod_len,
1317 CCP_SB_BYTES, false);
1318 if (ret)
1319 goto e_src;
1320 src.address += o_len; /* Adjust the address for the copy operation */
1321 ret = ccp_reverse_set_dm_area(&src, rsa->src, rsa->src_len,
1322 CCP_SB_BYTES, false);
1323 if (ret)
1324 goto e_src;
1325 src.address -= o_len; /* Reset the address to original value */
1326
1327 /* Prepare the output area for the operation */
1328 ret = ccp_init_data(&dst, cmd_q, rsa->dst, rsa->mod_len,
1329 o_len, DMA_FROM_DEVICE);
1330 if (ret)
1331 goto e_src;
1332
1333 op.soc = 1;
1334 op.src.u.dma.address = src.dma.address;
1335 op.src.u.dma.offset = 0;
1336 op.src.u.dma.length = i_len;
1337 op.dst.u.dma.address = dst.dm_wa.dma.address;
1338 op.dst.u.dma.offset = 0;
1339 op.dst.u.dma.length = o_len;
1340
1341 op.u.rsa.mod_size = rsa->key_size;
1342 op.u.rsa.input_len = i_len;
1343
1344 ret = cmd_q->ccp->vdata->perform->rsa(&op);
1345 if (ret) {
1346 cmd->engine_error = cmd_q->cmd_error;
1347 goto e_dst;
1348 }
1349
1350 ccp_reverse_get_dm_area(&dst.dm_wa, rsa->dst, rsa->mod_len);
1351
1352 e_dst:
1353 ccp_free_data(&dst, cmd_q);
1354
1355 e_src:
1356 ccp_dm_free(&src);
1357
1358 e_exp:
1359 ccp_dm_free(&exp);
1360
1361 e_sb:
1362 cmd_q->ccp->vdata->perform->sbfree(cmd_q, op.sb_key, sb_count);
1363
1364 return ret;
1365 }
1366
ccp_run_passthru_cmd(struct ccp_cmd_queue * cmd_q,struct ccp_cmd * cmd)1367 static int ccp_run_passthru_cmd(struct ccp_cmd_queue *cmd_q,
1368 struct ccp_cmd *cmd)
1369 {
1370 struct ccp_passthru_engine *pt = &cmd->u.passthru;
1371 struct ccp_dm_workarea mask;
1372 struct ccp_data src, dst;
1373 struct ccp_op op;
1374 bool in_place = false;
1375 unsigned int i;
1376 int ret = 0;
1377
1378 if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1)))
1379 return -EINVAL;
1380
1381 if (!pt->src || !pt->dst)
1382 return -EINVAL;
1383
1384 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
1385 if (pt->mask_len != CCP_PASSTHRU_MASKSIZE)
1386 return -EINVAL;
1387 if (!pt->mask)
1388 return -EINVAL;
1389 }
1390
1391 BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1);
1392
1393 memset(&op, 0, sizeof(op));
1394 op.cmd_q = cmd_q;
1395 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1396
1397 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
1398 /* Load the mask */
1399 op.sb_key = cmd_q->sb_key;
1400
1401 ret = ccp_init_dm_workarea(&mask, cmd_q,
1402 CCP_PASSTHRU_SB_COUNT *
1403 CCP_SB_BYTES,
1404 DMA_TO_DEVICE);
1405 if (ret)
1406 return ret;
1407
1408 ccp_set_dm_area(&mask, 0, pt->mask, 0, pt->mask_len);
1409 ret = ccp_copy_to_sb(cmd_q, &mask, op.jobid, op.sb_key,
1410 CCP_PASSTHRU_BYTESWAP_NOOP);
1411 if (ret) {
1412 cmd->engine_error = cmd_q->cmd_error;
1413 goto e_mask;
1414 }
1415 }
1416
1417 /* Prepare the input and output data workareas. For in-place
1418 * operations we need to set the dma direction to BIDIRECTIONAL
1419 * and copy the src workarea to the dst workarea.
1420 */
1421 if (sg_virt(pt->src) == sg_virt(pt->dst))
1422 in_place = true;
1423
1424 ret = ccp_init_data(&src, cmd_q, pt->src, pt->src_len,
1425 CCP_PASSTHRU_MASKSIZE,
1426 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1427 if (ret)
1428 goto e_mask;
1429
1430 if (in_place) {
1431 dst = src;
1432 } else {
1433 ret = ccp_init_data(&dst, cmd_q, pt->dst, pt->src_len,
1434 CCP_PASSTHRU_MASKSIZE, DMA_FROM_DEVICE);
1435 if (ret)
1436 goto e_src;
1437 }
1438
1439 /* Send data to the CCP Passthru engine
1440 * Because the CCP engine works on a single source and destination
1441 * dma address at a time, each entry in the source scatterlist
1442 * (after the dma_map_sg call) must be less than or equal to the
1443 * (remaining) length in the destination scatterlist entry and the
1444 * length must be a multiple of CCP_PASSTHRU_BLOCKSIZE
1445 */
1446 dst.sg_wa.sg_used = 0;
1447 for (i = 1; i <= src.sg_wa.dma_count; i++) {
1448 if (!dst.sg_wa.sg ||
1449 (dst.sg_wa.sg->length < src.sg_wa.sg->length)) {
1450 ret = -EINVAL;
1451 goto e_dst;
1452 }
1453
1454 if (i == src.sg_wa.dma_count) {
1455 op.eom = 1;
1456 op.soc = 1;
1457 }
1458
1459 op.src.type = CCP_MEMTYPE_SYSTEM;
1460 op.src.u.dma.address = sg_dma_address(src.sg_wa.sg);
1461 op.src.u.dma.offset = 0;
1462 op.src.u.dma.length = sg_dma_len(src.sg_wa.sg);
1463
1464 op.dst.type = CCP_MEMTYPE_SYSTEM;
1465 op.dst.u.dma.address = sg_dma_address(dst.sg_wa.sg);
1466 op.dst.u.dma.offset = dst.sg_wa.sg_used;
1467 op.dst.u.dma.length = op.src.u.dma.length;
1468
1469 ret = cmd_q->ccp->vdata->perform->passthru(&op);
1470 if (ret) {
1471 cmd->engine_error = cmd_q->cmd_error;
1472 goto e_dst;
1473 }
1474
1475 dst.sg_wa.sg_used += src.sg_wa.sg->length;
1476 if (dst.sg_wa.sg_used == dst.sg_wa.sg->length) {
1477 dst.sg_wa.sg = sg_next(dst.sg_wa.sg);
1478 dst.sg_wa.sg_used = 0;
1479 }
1480 src.sg_wa.sg = sg_next(src.sg_wa.sg);
1481 }
1482
1483 e_dst:
1484 if (!in_place)
1485 ccp_free_data(&dst, cmd_q);
1486
1487 e_src:
1488 ccp_free_data(&src, cmd_q);
1489
1490 e_mask:
1491 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP)
1492 ccp_dm_free(&mask);
1493
1494 return ret;
1495 }
1496
ccp_run_passthru_nomap_cmd(struct ccp_cmd_queue * cmd_q,struct ccp_cmd * cmd)1497 static int ccp_run_passthru_nomap_cmd(struct ccp_cmd_queue *cmd_q,
1498 struct ccp_cmd *cmd)
1499 {
1500 struct ccp_passthru_nomap_engine *pt = &cmd->u.passthru_nomap;
1501 struct ccp_dm_workarea mask;
1502 struct ccp_op op;
1503 int ret;
1504
1505 if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1)))
1506 return -EINVAL;
1507
1508 if (!pt->src_dma || !pt->dst_dma)
1509 return -EINVAL;
1510
1511 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
1512 if (pt->mask_len != CCP_PASSTHRU_MASKSIZE)
1513 return -EINVAL;
1514 if (!pt->mask)
1515 return -EINVAL;
1516 }
1517
1518 BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1);
1519
1520 memset(&op, 0, sizeof(op));
1521 op.cmd_q = cmd_q;
1522 op.jobid = ccp_gen_jobid(cmd_q->ccp);
1523
1524 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
1525 /* Load the mask */
1526 op.sb_key = cmd_q->sb_key;
1527
1528 mask.length = pt->mask_len;
1529 mask.dma.address = pt->mask;
1530 mask.dma.length = pt->mask_len;
1531
1532 ret = ccp_copy_to_sb(cmd_q, &mask, op.jobid, op.sb_key,
1533 CCP_PASSTHRU_BYTESWAP_NOOP);
1534 if (ret) {
1535 cmd->engine_error = cmd_q->cmd_error;
1536 return ret;
1537 }
1538 }
1539
1540 /* Send data to the CCP Passthru engine */
1541 op.eom = 1;
1542 op.soc = 1;
1543
1544 op.src.type = CCP_MEMTYPE_SYSTEM;
1545 op.src.u.dma.address = pt->src_dma;
1546 op.src.u.dma.offset = 0;
1547 op.src.u.dma.length = pt->src_len;
1548
1549 op.dst.type = CCP_MEMTYPE_SYSTEM;
1550 op.dst.u.dma.address = pt->dst_dma;
1551 op.dst.u.dma.offset = 0;
1552 op.dst.u.dma.length = pt->src_len;
1553
1554 ret = cmd_q->ccp->vdata->perform->passthru(&op);
1555 if (ret)
1556 cmd->engine_error = cmd_q->cmd_error;
1557
1558 return ret;
1559 }
1560
ccp_run_ecc_mm_cmd(struct ccp_cmd_queue * cmd_q,struct ccp_cmd * cmd)1561 static int ccp_run_ecc_mm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1562 {
1563 struct ccp_ecc_engine *ecc = &cmd->u.ecc;
1564 struct ccp_dm_workarea src, dst;
1565 struct ccp_op op;
1566 int ret;
1567 u8 *save;
1568
1569 if (!ecc->u.mm.operand_1 ||
1570 (ecc->u.mm.operand_1_len > CCP_ECC_MODULUS_BYTES))
1571 return -EINVAL;
1572
1573 if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT)
1574 if (!ecc->u.mm.operand_2 ||
1575 (ecc->u.mm.operand_2_len > CCP_ECC_MODULUS_BYTES))
1576 return -EINVAL;
1577
1578 if (!ecc->u.mm.result ||
1579 (ecc->u.mm.result_len < CCP_ECC_MODULUS_BYTES))
1580 return -EINVAL;
1581
1582 memset(&op, 0, sizeof(op));
1583 op.cmd_q = cmd_q;
1584 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1585
1586 /* Concatenate the modulus and the operands. Both the modulus and
1587 * the operands must be in little endian format. Since the input
1588 * is in big endian format it must be converted and placed in a
1589 * fixed length buffer.
1590 */
1591 ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE,
1592 DMA_TO_DEVICE);
1593 if (ret)
1594 return ret;
1595
1596 /* Save the workarea address since it is updated in order to perform
1597 * the concatenation
1598 */
1599 save = src.address;
1600
1601 /* Copy the ECC modulus */
1602 ret = ccp_reverse_set_dm_area(&src, ecc->mod, ecc->mod_len,
1603 CCP_ECC_OPERAND_SIZE, false);
1604 if (ret)
1605 goto e_src;
1606 src.address += CCP_ECC_OPERAND_SIZE;
1607
1608 /* Copy the first operand */
1609 ret = ccp_reverse_set_dm_area(&src, ecc->u.mm.operand_1,
1610 ecc->u.mm.operand_1_len,
1611 CCP_ECC_OPERAND_SIZE, false);
1612 if (ret)
1613 goto e_src;
1614 src.address += CCP_ECC_OPERAND_SIZE;
1615
1616 if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT) {
1617 /* Copy the second operand */
1618 ret = ccp_reverse_set_dm_area(&src, ecc->u.mm.operand_2,
1619 ecc->u.mm.operand_2_len,
1620 CCP_ECC_OPERAND_SIZE, false);
1621 if (ret)
1622 goto e_src;
1623 src.address += CCP_ECC_OPERAND_SIZE;
1624 }
1625
1626 /* Restore the workarea address */
1627 src.address = save;
1628
1629 /* Prepare the output area for the operation */
1630 ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE,
1631 DMA_FROM_DEVICE);
1632 if (ret)
1633 goto e_src;
1634
1635 op.soc = 1;
1636 op.src.u.dma.address = src.dma.address;
1637 op.src.u.dma.offset = 0;
1638 op.src.u.dma.length = src.length;
1639 op.dst.u.dma.address = dst.dma.address;
1640 op.dst.u.dma.offset = 0;
1641 op.dst.u.dma.length = dst.length;
1642
1643 op.u.ecc.function = cmd->u.ecc.function;
1644
1645 ret = cmd_q->ccp->vdata->perform->ecc(&op);
1646 if (ret) {
1647 cmd->engine_error = cmd_q->cmd_error;
1648 goto e_dst;
1649 }
1650
1651 ecc->ecc_result = le16_to_cpup(
1652 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET));
1653 if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) {
1654 ret = -EIO;
1655 goto e_dst;
1656 }
1657
1658 /* Save the ECC result */
1659 ccp_reverse_get_dm_area(&dst, ecc->u.mm.result, CCP_ECC_MODULUS_BYTES);
1660
1661 e_dst:
1662 ccp_dm_free(&dst);
1663
1664 e_src:
1665 ccp_dm_free(&src);
1666
1667 return ret;
1668 }
1669
ccp_run_ecc_pm_cmd(struct ccp_cmd_queue * cmd_q,struct ccp_cmd * cmd)1670 static int ccp_run_ecc_pm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1671 {
1672 struct ccp_ecc_engine *ecc = &cmd->u.ecc;
1673 struct ccp_dm_workarea src, dst;
1674 struct ccp_op op;
1675 int ret;
1676 u8 *save;
1677
1678 if (!ecc->u.pm.point_1.x ||
1679 (ecc->u.pm.point_1.x_len > CCP_ECC_MODULUS_BYTES) ||
1680 !ecc->u.pm.point_1.y ||
1681 (ecc->u.pm.point_1.y_len > CCP_ECC_MODULUS_BYTES))
1682 return -EINVAL;
1683
1684 if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) {
1685 if (!ecc->u.pm.point_2.x ||
1686 (ecc->u.pm.point_2.x_len > CCP_ECC_MODULUS_BYTES) ||
1687 !ecc->u.pm.point_2.y ||
1688 (ecc->u.pm.point_2.y_len > CCP_ECC_MODULUS_BYTES))
1689 return -EINVAL;
1690 } else {
1691 if (!ecc->u.pm.domain_a ||
1692 (ecc->u.pm.domain_a_len > CCP_ECC_MODULUS_BYTES))
1693 return -EINVAL;
1694
1695 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT)
1696 if (!ecc->u.pm.scalar ||
1697 (ecc->u.pm.scalar_len > CCP_ECC_MODULUS_BYTES))
1698 return -EINVAL;
1699 }
1700
1701 if (!ecc->u.pm.result.x ||
1702 (ecc->u.pm.result.x_len < CCP_ECC_MODULUS_BYTES) ||
1703 !ecc->u.pm.result.y ||
1704 (ecc->u.pm.result.y_len < CCP_ECC_MODULUS_BYTES))
1705 return -EINVAL;
1706
1707 memset(&op, 0, sizeof(op));
1708 op.cmd_q = cmd_q;
1709 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1710
1711 /* Concatenate the modulus and the operands. Both the modulus and
1712 * the operands must be in little endian format. Since the input
1713 * is in big endian format it must be converted and placed in a
1714 * fixed length buffer.
1715 */
1716 ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE,
1717 DMA_TO_DEVICE);
1718 if (ret)
1719 return ret;
1720
1721 /* Save the workarea address since it is updated in order to perform
1722 * the concatenation
1723 */
1724 save = src.address;
1725
1726 /* Copy the ECC modulus */
1727 ret = ccp_reverse_set_dm_area(&src, ecc->mod, ecc->mod_len,
1728 CCP_ECC_OPERAND_SIZE, false);
1729 if (ret)
1730 goto e_src;
1731 src.address += CCP_ECC_OPERAND_SIZE;
1732
1733 /* Copy the first point X and Y coordinate */
1734 ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.point_1.x,
1735 ecc->u.pm.point_1.x_len,
1736 CCP_ECC_OPERAND_SIZE, false);
1737 if (ret)
1738 goto e_src;
1739 src.address += CCP_ECC_OPERAND_SIZE;
1740 ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.point_1.y,
1741 ecc->u.pm.point_1.y_len,
1742 CCP_ECC_OPERAND_SIZE, false);
1743 if (ret)
1744 goto e_src;
1745 src.address += CCP_ECC_OPERAND_SIZE;
1746
1747 /* Set the first point Z coordinate to 1 */
1748 *src.address = 0x01;
1749 src.address += CCP_ECC_OPERAND_SIZE;
1750
1751 if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) {
1752 /* Copy the second point X and Y coordinate */
1753 ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.point_2.x,
1754 ecc->u.pm.point_2.x_len,
1755 CCP_ECC_OPERAND_SIZE, false);
1756 if (ret)
1757 goto e_src;
1758 src.address += CCP_ECC_OPERAND_SIZE;
1759 ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.point_2.y,
1760 ecc->u.pm.point_2.y_len,
1761 CCP_ECC_OPERAND_SIZE, false);
1762 if (ret)
1763 goto e_src;
1764 src.address += CCP_ECC_OPERAND_SIZE;
1765
1766 /* Set the second point Z coordinate to 1 */
1767 *src.address = 0x01;
1768 src.address += CCP_ECC_OPERAND_SIZE;
1769 } else {
1770 /* Copy the Domain "a" parameter */
1771 ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.domain_a,
1772 ecc->u.pm.domain_a_len,
1773 CCP_ECC_OPERAND_SIZE, false);
1774 if (ret)
1775 goto e_src;
1776 src.address += CCP_ECC_OPERAND_SIZE;
1777
1778 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT) {
1779 /* Copy the scalar value */
1780 ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.scalar,
1781 ecc->u.pm.scalar_len,
1782 CCP_ECC_OPERAND_SIZE,
1783 false);
1784 if (ret)
1785 goto e_src;
1786 src.address += CCP_ECC_OPERAND_SIZE;
1787 }
1788 }
1789
1790 /* Restore the workarea address */
1791 src.address = save;
1792
1793 /* Prepare the output area for the operation */
1794 ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE,
1795 DMA_FROM_DEVICE);
1796 if (ret)
1797 goto e_src;
1798
1799 op.soc = 1;
1800 op.src.u.dma.address = src.dma.address;
1801 op.src.u.dma.offset = 0;
1802 op.src.u.dma.length = src.length;
1803 op.dst.u.dma.address = dst.dma.address;
1804 op.dst.u.dma.offset = 0;
1805 op.dst.u.dma.length = dst.length;
1806
1807 op.u.ecc.function = cmd->u.ecc.function;
1808
1809 ret = cmd_q->ccp->vdata->perform->ecc(&op);
1810 if (ret) {
1811 cmd->engine_error = cmd_q->cmd_error;
1812 goto e_dst;
1813 }
1814
1815 ecc->ecc_result = le16_to_cpup(
1816 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET));
1817 if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) {
1818 ret = -EIO;
1819 goto e_dst;
1820 }
1821
1822 /* Save the workarea address since it is updated as we walk through
1823 * to copy the point math result
1824 */
1825 save = dst.address;
1826
1827 /* Save the ECC result X and Y coordinates */
1828 ccp_reverse_get_dm_area(&dst, ecc->u.pm.result.x,
1829 CCP_ECC_MODULUS_BYTES);
1830 dst.address += CCP_ECC_OUTPUT_SIZE;
1831 ccp_reverse_get_dm_area(&dst, ecc->u.pm.result.y,
1832 CCP_ECC_MODULUS_BYTES);
1833 dst.address += CCP_ECC_OUTPUT_SIZE;
1834
1835 /* Restore the workarea address */
1836 dst.address = save;
1837
1838 e_dst:
1839 ccp_dm_free(&dst);
1840
1841 e_src:
1842 ccp_dm_free(&src);
1843
1844 return ret;
1845 }
1846
ccp_run_ecc_cmd(struct ccp_cmd_queue * cmd_q,struct ccp_cmd * cmd)1847 static int ccp_run_ecc_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1848 {
1849 struct ccp_ecc_engine *ecc = &cmd->u.ecc;
1850
1851 ecc->ecc_result = 0;
1852
1853 if (!ecc->mod ||
1854 (ecc->mod_len > CCP_ECC_MODULUS_BYTES))
1855 return -EINVAL;
1856
1857 switch (ecc->function) {
1858 case CCP_ECC_FUNCTION_MMUL_384BIT:
1859 case CCP_ECC_FUNCTION_MADD_384BIT:
1860 case CCP_ECC_FUNCTION_MINV_384BIT:
1861 return ccp_run_ecc_mm_cmd(cmd_q, cmd);
1862
1863 case CCP_ECC_FUNCTION_PADD_384BIT:
1864 case CCP_ECC_FUNCTION_PMUL_384BIT:
1865 case CCP_ECC_FUNCTION_PDBL_384BIT:
1866 return ccp_run_ecc_pm_cmd(cmd_q, cmd);
1867
1868 default:
1869 return -EINVAL;
1870 }
1871 }
1872
ccp_run_cmd(struct ccp_cmd_queue * cmd_q,struct ccp_cmd * cmd)1873 int ccp_run_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1874 {
1875 int ret;
1876
1877 cmd->engine_error = 0;
1878 cmd_q->cmd_error = 0;
1879 cmd_q->int_rcvd = 0;
1880 cmd_q->free_slots = cmd_q->ccp->vdata->perform->get_free_slots(cmd_q);
1881
1882 switch (cmd->engine) {
1883 case CCP_ENGINE_AES:
1884 ret = ccp_run_aes_cmd(cmd_q, cmd);
1885 break;
1886 case CCP_ENGINE_XTS_AES_128:
1887 ret = ccp_run_xts_aes_cmd(cmd_q, cmd);
1888 break;
1889 case CCP_ENGINE_SHA:
1890 ret = ccp_run_sha_cmd(cmd_q, cmd);
1891 break;
1892 case CCP_ENGINE_RSA:
1893 ret = ccp_run_rsa_cmd(cmd_q, cmd);
1894 break;
1895 case CCP_ENGINE_PASSTHRU:
1896 if (cmd->flags & CCP_CMD_PASSTHRU_NO_DMA_MAP)
1897 ret = ccp_run_passthru_nomap_cmd(cmd_q, cmd);
1898 else
1899 ret = ccp_run_passthru_cmd(cmd_q, cmd);
1900 break;
1901 case CCP_ENGINE_ECC:
1902 ret = ccp_run_ecc_cmd(cmd_q, cmd);
1903 break;
1904 default:
1905 ret = -EINVAL;
1906 }
1907
1908 return ret;
1909 }
1910