1 /*
2 * AMD Cryptographic Coprocessor (CCP) driver
3 *
4 * Copyright (C) 2013,2017 Advanced Micro Devices, Inc.
5 *
6 * Author: Tom Lendacky <thomas.lendacky@amd.com>
7 * Author: Gary R Hook <gary.hook@amd.com>
8 *
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License version 2 as
11 * published by the Free Software Foundation.
12 */
13
14 #include <linux/module.h>
15 #include <linux/kernel.h>
16 #include <linux/pci.h>
17 #include <linux/interrupt.h>
18 #include <crypto/scatterwalk.h>
19 #include <crypto/des.h>
20 #include <linux/ccp.h>
21
22 #include "ccp-dev.h"
23
24 /* SHA initial context values */
25 static const __be32 ccp_sha1_init[SHA1_DIGEST_SIZE / sizeof(__be32)] = {
26 cpu_to_be32(SHA1_H0), cpu_to_be32(SHA1_H1),
27 cpu_to_be32(SHA1_H2), cpu_to_be32(SHA1_H3),
28 cpu_to_be32(SHA1_H4),
29 };
30
31 static const __be32 ccp_sha224_init[SHA256_DIGEST_SIZE / sizeof(__be32)] = {
32 cpu_to_be32(SHA224_H0), cpu_to_be32(SHA224_H1),
33 cpu_to_be32(SHA224_H2), cpu_to_be32(SHA224_H3),
34 cpu_to_be32(SHA224_H4), cpu_to_be32(SHA224_H5),
35 cpu_to_be32(SHA224_H6), cpu_to_be32(SHA224_H7),
36 };
37
38 static const __be32 ccp_sha256_init[SHA256_DIGEST_SIZE / sizeof(__be32)] = {
39 cpu_to_be32(SHA256_H0), cpu_to_be32(SHA256_H1),
40 cpu_to_be32(SHA256_H2), cpu_to_be32(SHA256_H3),
41 cpu_to_be32(SHA256_H4), cpu_to_be32(SHA256_H5),
42 cpu_to_be32(SHA256_H6), cpu_to_be32(SHA256_H7),
43 };
44
45 static const __be64 ccp_sha384_init[SHA512_DIGEST_SIZE / sizeof(__be64)] = {
46 cpu_to_be64(SHA384_H0), cpu_to_be64(SHA384_H1),
47 cpu_to_be64(SHA384_H2), cpu_to_be64(SHA384_H3),
48 cpu_to_be64(SHA384_H4), cpu_to_be64(SHA384_H5),
49 cpu_to_be64(SHA384_H6), cpu_to_be64(SHA384_H7),
50 };
51
52 static const __be64 ccp_sha512_init[SHA512_DIGEST_SIZE / sizeof(__be64)] = {
53 cpu_to_be64(SHA512_H0), cpu_to_be64(SHA512_H1),
54 cpu_to_be64(SHA512_H2), cpu_to_be64(SHA512_H3),
55 cpu_to_be64(SHA512_H4), cpu_to_be64(SHA512_H5),
56 cpu_to_be64(SHA512_H6), cpu_to_be64(SHA512_H7),
57 };
58
59 #define CCP_NEW_JOBID(ccp) ((ccp->vdata->version == CCP_VERSION(3, 0)) ? \
60 ccp_gen_jobid(ccp) : 0)
61
ccp_gen_jobid(struct ccp_device * ccp)62 static u32 ccp_gen_jobid(struct ccp_device *ccp)
63 {
64 return atomic_inc_return(&ccp->current_id) & CCP_JOBID_MASK;
65 }
66
ccp_sg_free(struct ccp_sg_workarea * wa)67 static void ccp_sg_free(struct ccp_sg_workarea *wa)
68 {
69 if (wa->dma_count)
70 dma_unmap_sg(wa->dma_dev, wa->dma_sg_head, wa->nents, wa->dma_dir);
71
72 wa->dma_count = 0;
73 }
74
ccp_init_sg_workarea(struct ccp_sg_workarea * wa,struct device * dev,struct scatterlist * sg,u64 len,enum dma_data_direction dma_dir)75 static int ccp_init_sg_workarea(struct ccp_sg_workarea *wa, struct device *dev,
76 struct scatterlist *sg, u64 len,
77 enum dma_data_direction dma_dir)
78 {
79 memset(wa, 0, sizeof(*wa));
80
81 wa->sg = sg;
82 if (!sg)
83 return 0;
84
85 wa->nents = sg_nents_for_len(sg, len);
86 if (wa->nents < 0)
87 return wa->nents;
88
89 wa->bytes_left = len;
90 wa->sg_used = 0;
91
92 if (len == 0)
93 return 0;
94
95 if (dma_dir == DMA_NONE)
96 return 0;
97
98 wa->dma_sg = sg;
99 wa->dma_sg_head = sg;
100 wa->dma_dev = dev;
101 wa->dma_dir = dma_dir;
102 wa->dma_count = dma_map_sg(dev, sg, wa->nents, dma_dir);
103 if (!wa->dma_count)
104 return -ENOMEM;
105
106 return 0;
107 }
108
ccp_update_sg_workarea(struct ccp_sg_workarea * wa,unsigned int len)109 static void ccp_update_sg_workarea(struct ccp_sg_workarea *wa, unsigned int len)
110 {
111 unsigned int nbytes = min_t(u64, len, wa->bytes_left);
112 unsigned int sg_combined_len = 0;
113
114 if (!wa->sg)
115 return;
116
117 wa->sg_used += nbytes;
118 wa->bytes_left -= nbytes;
119 if (wa->sg_used == sg_dma_len(wa->dma_sg)) {
120 /* Advance to the next DMA scatterlist entry */
121 wa->dma_sg = sg_next(wa->dma_sg);
122
123 /* In the case that the DMA mapped scatterlist has entries
124 * that have been merged, the non-DMA mapped scatterlist
125 * must be advanced multiple times for each merged entry.
126 * This ensures that the current non-DMA mapped entry
127 * corresponds to the current DMA mapped entry.
128 */
129 do {
130 sg_combined_len += wa->sg->length;
131 wa->sg = sg_next(wa->sg);
132 } while (wa->sg_used > sg_combined_len);
133
134 wa->sg_used = 0;
135 }
136 }
137
ccp_dm_free(struct ccp_dm_workarea * wa)138 static void ccp_dm_free(struct ccp_dm_workarea *wa)
139 {
140 if (wa->length <= CCP_DMAPOOL_MAX_SIZE) {
141 if (wa->address)
142 dma_pool_free(wa->dma_pool, wa->address,
143 wa->dma.address);
144 } else {
145 if (wa->dma.address)
146 dma_unmap_single(wa->dev, wa->dma.address, wa->length,
147 wa->dma.dir);
148 kfree(wa->address);
149 }
150
151 wa->address = NULL;
152 wa->dma.address = 0;
153 }
154
ccp_init_dm_workarea(struct ccp_dm_workarea * wa,struct ccp_cmd_queue * cmd_q,unsigned int len,enum dma_data_direction dir)155 static int ccp_init_dm_workarea(struct ccp_dm_workarea *wa,
156 struct ccp_cmd_queue *cmd_q,
157 unsigned int len,
158 enum dma_data_direction dir)
159 {
160 memset(wa, 0, sizeof(*wa));
161
162 if (!len)
163 return 0;
164
165 wa->dev = cmd_q->ccp->dev;
166 wa->length = len;
167
168 if (len <= CCP_DMAPOOL_MAX_SIZE) {
169 wa->dma_pool = cmd_q->dma_pool;
170
171 wa->address = dma_pool_alloc(wa->dma_pool, GFP_KERNEL,
172 &wa->dma.address);
173 if (!wa->address)
174 return -ENOMEM;
175
176 wa->dma.length = CCP_DMAPOOL_MAX_SIZE;
177
178 memset(wa->address, 0, CCP_DMAPOOL_MAX_SIZE);
179 } else {
180 wa->address = kzalloc(len, GFP_KERNEL);
181 if (!wa->address)
182 return -ENOMEM;
183
184 wa->dma.address = dma_map_single(wa->dev, wa->address, len,
185 dir);
186 if (dma_mapping_error(wa->dev, wa->dma.address))
187 return -ENOMEM;
188
189 wa->dma.length = len;
190 }
191 wa->dma.dir = dir;
192
193 return 0;
194 }
195
ccp_set_dm_area(struct ccp_dm_workarea * wa,unsigned int wa_offset,struct scatterlist * sg,unsigned int sg_offset,unsigned int len)196 static int ccp_set_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset,
197 struct scatterlist *sg, unsigned int sg_offset,
198 unsigned int len)
199 {
200 WARN_ON(!wa->address);
201
202 if (len > (wa->length - wa_offset))
203 return -EINVAL;
204
205 scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len,
206 0);
207 return 0;
208 }
209
ccp_get_dm_area(struct ccp_dm_workarea * wa,unsigned int wa_offset,struct scatterlist * sg,unsigned int sg_offset,unsigned int len)210 static void ccp_get_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset,
211 struct scatterlist *sg, unsigned int sg_offset,
212 unsigned int len)
213 {
214 WARN_ON(!wa->address);
215
216 scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len,
217 1);
218 }
219
ccp_reverse_set_dm_area(struct ccp_dm_workarea * wa,unsigned int wa_offset,struct scatterlist * sg,unsigned int sg_offset,unsigned int len)220 static int ccp_reverse_set_dm_area(struct ccp_dm_workarea *wa,
221 unsigned int wa_offset,
222 struct scatterlist *sg,
223 unsigned int sg_offset,
224 unsigned int len)
225 {
226 u8 *p, *q;
227 int rc;
228
229 rc = ccp_set_dm_area(wa, wa_offset, sg, sg_offset, len);
230 if (rc)
231 return rc;
232
233 p = wa->address + wa_offset;
234 q = p + len - 1;
235 while (p < q) {
236 *p = *p ^ *q;
237 *q = *p ^ *q;
238 *p = *p ^ *q;
239 p++;
240 q--;
241 }
242 return 0;
243 }
244
ccp_reverse_get_dm_area(struct ccp_dm_workarea * wa,unsigned int wa_offset,struct scatterlist * sg,unsigned int sg_offset,unsigned int len)245 static void ccp_reverse_get_dm_area(struct ccp_dm_workarea *wa,
246 unsigned int wa_offset,
247 struct scatterlist *sg,
248 unsigned int sg_offset,
249 unsigned int len)
250 {
251 u8 *p, *q;
252
253 p = wa->address + wa_offset;
254 q = p + len - 1;
255 while (p < q) {
256 *p = *p ^ *q;
257 *q = *p ^ *q;
258 *p = *p ^ *q;
259 p++;
260 q--;
261 }
262
263 ccp_get_dm_area(wa, wa_offset, sg, sg_offset, len);
264 }
265
ccp_free_data(struct ccp_data * data,struct ccp_cmd_queue * cmd_q)266 static void ccp_free_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q)
267 {
268 ccp_dm_free(&data->dm_wa);
269 ccp_sg_free(&data->sg_wa);
270 }
271
ccp_init_data(struct ccp_data * data,struct ccp_cmd_queue * cmd_q,struct scatterlist * sg,u64 sg_len,unsigned int dm_len,enum dma_data_direction dir)272 static int ccp_init_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q,
273 struct scatterlist *sg, u64 sg_len,
274 unsigned int dm_len,
275 enum dma_data_direction dir)
276 {
277 int ret;
278
279 memset(data, 0, sizeof(*data));
280
281 ret = ccp_init_sg_workarea(&data->sg_wa, cmd_q->ccp->dev, sg, sg_len,
282 dir);
283 if (ret)
284 goto e_err;
285
286 ret = ccp_init_dm_workarea(&data->dm_wa, cmd_q, dm_len, dir);
287 if (ret)
288 goto e_err;
289
290 return 0;
291
292 e_err:
293 ccp_free_data(data, cmd_q);
294
295 return ret;
296 }
297
ccp_queue_buf(struct ccp_data * data,unsigned int from)298 static unsigned int ccp_queue_buf(struct ccp_data *data, unsigned int from)
299 {
300 struct ccp_sg_workarea *sg_wa = &data->sg_wa;
301 struct ccp_dm_workarea *dm_wa = &data->dm_wa;
302 unsigned int buf_count, nbytes;
303
304 /* Clear the buffer if setting it */
305 if (!from)
306 memset(dm_wa->address, 0, dm_wa->length);
307
308 if (!sg_wa->sg)
309 return 0;
310
311 /* Perform the copy operation
312 * nbytes will always be <= UINT_MAX because dm_wa->length is
313 * an unsigned int
314 */
315 nbytes = min_t(u64, sg_wa->bytes_left, dm_wa->length);
316 scatterwalk_map_and_copy(dm_wa->address, sg_wa->sg, sg_wa->sg_used,
317 nbytes, from);
318
319 /* Update the structures and generate the count */
320 buf_count = 0;
321 while (sg_wa->bytes_left && (buf_count < dm_wa->length)) {
322 nbytes = min(sg_dma_len(sg_wa->dma_sg) - sg_wa->sg_used,
323 dm_wa->length - buf_count);
324 nbytes = min_t(u64, sg_wa->bytes_left, nbytes);
325
326 buf_count += nbytes;
327 ccp_update_sg_workarea(sg_wa, nbytes);
328 }
329
330 return buf_count;
331 }
332
ccp_fill_queue_buf(struct ccp_data * data)333 static unsigned int ccp_fill_queue_buf(struct ccp_data *data)
334 {
335 return ccp_queue_buf(data, 0);
336 }
337
ccp_empty_queue_buf(struct ccp_data * data)338 static unsigned int ccp_empty_queue_buf(struct ccp_data *data)
339 {
340 return ccp_queue_buf(data, 1);
341 }
342
ccp_prepare_data(struct ccp_data * src,struct ccp_data * dst,struct ccp_op * op,unsigned int block_size,bool blocksize_op)343 static void ccp_prepare_data(struct ccp_data *src, struct ccp_data *dst,
344 struct ccp_op *op, unsigned int block_size,
345 bool blocksize_op)
346 {
347 unsigned int sg_src_len, sg_dst_len, op_len;
348
349 /* The CCP can only DMA from/to one address each per operation. This
350 * requires that we find the smallest DMA area between the source
351 * and destination. The resulting len values will always be <= UINT_MAX
352 * because the dma length is an unsigned int.
353 */
354 sg_src_len = sg_dma_len(src->sg_wa.dma_sg) - src->sg_wa.sg_used;
355 sg_src_len = min_t(u64, src->sg_wa.bytes_left, sg_src_len);
356
357 if (dst) {
358 sg_dst_len = sg_dma_len(dst->sg_wa.dma_sg) - dst->sg_wa.sg_used;
359 sg_dst_len = min_t(u64, src->sg_wa.bytes_left, sg_dst_len);
360 op_len = min(sg_src_len, sg_dst_len);
361 } else {
362 op_len = sg_src_len;
363 }
364
365 /* The data operation length will be at least block_size in length
366 * or the smaller of available sg room remaining for the source or
367 * the destination
368 */
369 op_len = max(op_len, block_size);
370
371 /* Unless we have to buffer data, there's no reason to wait */
372 op->soc = 0;
373
374 if (sg_src_len < block_size) {
375 /* Not enough data in the sg element, so it
376 * needs to be buffered into a blocksize chunk
377 */
378 int cp_len = ccp_fill_queue_buf(src);
379
380 op->soc = 1;
381 op->src.u.dma.address = src->dm_wa.dma.address;
382 op->src.u.dma.offset = 0;
383 op->src.u.dma.length = (blocksize_op) ? block_size : cp_len;
384 } else {
385 /* Enough data in the sg element, but we need to
386 * adjust for any previously copied data
387 */
388 op->src.u.dma.address = sg_dma_address(src->sg_wa.dma_sg);
389 op->src.u.dma.offset = src->sg_wa.sg_used;
390 op->src.u.dma.length = op_len & ~(block_size - 1);
391
392 ccp_update_sg_workarea(&src->sg_wa, op->src.u.dma.length);
393 }
394
395 if (dst) {
396 if (sg_dst_len < block_size) {
397 /* Not enough room in the sg element or we're on the
398 * last piece of data (when using padding), so the
399 * output needs to be buffered into a blocksize chunk
400 */
401 op->soc = 1;
402 op->dst.u.dma.address = dst->dm_wa.dma.address;
403 op->dst.u.dma.offset = 0;
404 op->dst.u.dma.length = op->src.u.dma.length;
405 } else {
406 /* Enough room in the sg element, but we need to
407 * adjust for any previously used area
408 */
409 op->dst.u.dma.address = sg_dma_address(dst->sg_wa.dma_sg);
410 op->dst.u.dma.offset = dst->sg_wa.sg_used;
411 op->dst.u.dma.length = op->src.u.dma.length;
412 }
413 }
414 }
415
ccp_process_data(struct ccp_data * src,struct ccp_data * dst,struct ccp_op * op)416 static void ccp_process_data(struct ccp_data *src, struct ccp_data *dst,
417 struct ccp_op *op)
418 {
419 op->init = 0;
420
421 if (dst) {
422 if (op->dst.u.dma.address == dst->dm_wa.dma.address)
423 ccp_empty_queue_buf(dst);
424 else
425 ccp_update_sg_workarea(&dst->sg_wa,
426 op->dst.u.dma.length);
427 }
428 }
429
ccp_copy_to_from_sb(struct ccp_cmd_queue * cmd_q,struct ccp_dm_workarea * wa,u32 jobid,u32 sb,u32 byte_swap,bool from)430 static int ccp_copy_to_from_sb(struct ccp_cmd_queue *cmd_q,
431 struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
432 u32 byte_swap, bool from)
433 {
434 struct ccp_op op;
435
436 memset(&op, 0, sizeof(op));
437
438 op.cmd_q = cmd_q;
439 op.jobid = jobid;
440 op.eom = 1;
441
442 if (from) {
443 op.soc = 1;
444 op.src.type = CCP_MEMTYPE_SB;
445 op.src.u.sb = sb;
446 op.dst.type = CCP_MEMTYPE_SYSTEM;
447 op.dst.u.dma.address = wa->dma.address;
448 op.dst.u.dma.length = wa->length;
449 } else {
450 op.src.type = CCP_MEMTYPE_SYSTEM;
451 op.src.u.dma.address = wa->dma.address;
452 op.src.u.dma.length = wa->length;
453 op.dst.type = CCP_MEMTYPE_SB;
454 op.dst.u.sb = sb;
455 }
456
457 op.u.passthru.byte_swap = byte_swap;
458
459 return cmd_q->ccp->vdata->perform->passthru(&op);
460 }
461
ccp_copy_to_sb(struct ccp_cmd_queue * cmd_q,struct ccp_dm_workarea * wa,u32 jobid,u32 sb,u32 byte_swap)462 static int ccp_copy_to_sb(struct ccp_cmd_queue *cmd_q,
463 struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
464 u32 byte_swap)
465 {
466 return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, false);
467 }
468
ccp_copy_from_sb(struct ccp_cmd_queue * cmd_q,struct ccp_dm_workarea * wa,u32 jobid,u32 sb,u32 byte_swap)469 static int ccp_copy_from_sb(struct ccp_cmd_queue *cmd_q,
470 struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
471 u32 byte_swap)
472 {
473 return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, true);
474 }
475
476 static noinline_for_stack int
ccp_run_aes_cmac_cmd(struct ccp_cmd_queue * cmd_q,struct ccp_cmd * cmd)477 ccp_run_aes_cmac_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
478 {
479 struct ccp_aes_engine *aes = &cmd->u.aes;
480 struct ccp_dm_workarea key, ctx;
481 struct ccp_data src;
482 struct ccp_op op;
483 unsigned int dm_offset;
484 int ret;
485
486 if (!((aes->key_len == AES_KEYSIZE_128) ||
487 (aes->key_len == AES_KEYSIZE_192) ||
488 (aes->key_len == AES_KEYSIZE_256)))
489 return -EINVAL;
490
491 if (aes->src_len & (AES_BLOCK_SIZE - 1))
492 return -EINVAL;
493
494 if (aes->iv_len != AES_BLOCK_SIZE)
495 return -EINVAL;
496
497 if (!aes->key || !aes->iv || !aes->src)
498 return -EINVAL;
499
500 if (aes->cmac_final) {
501 if (aes->cmac_key_len != AES_BLOCK_SIZE)
502 return -EINVAL;
503
504 if (!aes->cmac_key)
505 return -EINVAL;
506 }
507
508 BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1);
509 BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1);
510
511 ret = -EIO;
512 memset(&op, 0, sizeof(op));
513 op.cmd_q = cmd_q;
514 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
515 op.sb_key = cmd_q->sb_key;
516 op.sb_ctx = cmd_q->sb_ctx;
517 op.init = 1;
518 op.u.aes.type = aes->type;
519 op.u.aes.mode = aes->mode;
520 op.u.aes.action = aes->action;
521
522 /* All supported key sizes fit in a single (32-byte) SB entry
523 * and must be in little endian format. Use the 256-bit byte
524 * swap passthru option to convert from big endian to little
525 * endian.
526 */
527 ret = ccp_init_dm_workarea(&key, cmd_q,
528 CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES,
529 DMA_TO_DEVICE);
530 if (ret)
531 return ret;
532
533 dm_offset = CCP_SB_BYTES - aes->key_len;
534 ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
535 if (ret)
536 goto e_key;
537 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
538 CCP_PASSTHRU_BYTESWAP_256BIT);
539 if (ret) {
540 cmd->engine_error = cmd_q->cmd_error;
541 goto e_key;
542 }
543
544 /* The AES context fits in a single (32-byte) SB entry and
545 * must be in little endian format. Use the 256-bit byte swap
546 * passthru option to convert from big endian to little endian.
547 */
548 ret = ccp_init_dm_workarea(&ctx, cmd_q,
549 CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
550 DMA_BIDIRECTIONAL);
551 if (ret)
552 goto e_key;
553
554 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
555 ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
556 if (ret)
557 goto e_ctx;
558 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
559 CCP_PASSTHRU_BYTESWAP_256BIT);
560 if (ret) {
561 cmd->engine_error = cmd_q->cmd_error;
562 goto e_ctx;
563 }
564
565 /* Send data to the CCP AES engine */
566 ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,
567 AES_BLOCK_SIZE, DMA_TO_DEVICE);
568 if (ret)
569 goto e_ctx;
570
571 while (src.sg_wa.bytes_left) {
572 ccp_prepare_data(&src, NULL, &op, AES_BLOCK_SIZE, true);
573 if (aes->cmac_final && !src.sg_wa.bytes_left) {
574 op.eom = 1;
575
576 /* Push the K1/K2 key to the CCP now */
577 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid,
578 op.sb_ctx,
579 CCP_PASSTHRU_BYTESWAP_256BIT);
580 if (ret) {
581 cmd->engine_error = cmd_q->cmd_error;
582 goto e_src;
583 }
584
585 ret = ccp_set_dm_area(&ctx, 0, aes->cmac_key, 0,
586 aes->cmac_key_len);
587 if (ret)
588 goto e_src;
589 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
590 CCP_PASSTHRU_BYTESWAP_256BIT);
591 if (ret) {
592 cmd->engine_error = cmd_q->cmd_error;
593 goto e_src;
594 }
595 }
596
597 ret = cmd_q->ccp->vdata->perform->aes(&op);
598 if (ret) {
599 cmd->engine_error = cmd_q->cmd_error;
600 goto e_src;
601 }
602
603 ccp_process_data(&src, NULL, &op);
604 }
605
606 /* Retrieve the AES context - convert from LE to BE using
607 * 32-byte (256-bit) byteswapping
608 */
609 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
610 CCP_PASSTHRU_BYTESWAP_256BIT);
611 if (ret) {
612 cmd->engine_error = cmd_q->cmd_error;
613 goto e_src;
614 }
615
616 /* ...but we only need AES_BLOCK_SIZE bytes */
617 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
618 ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
619
620 e_src:
621 ccp_free_data(&src, cmd_q);
622
623 e_ctx:
624 ccp_dm_free(&ctx);
625
626 e_key:
627 ccp_dm_free(&key);
628
629 return ret;
630 }
631
632 static noinline_for_stack int
ccp_run_aes_gcm_cmd(struct ccp_cmd_queue * cmd_q,struct ccp_cmd * cmd)633 ccp_run_aes_gcm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
634 {
635 struct ccp_aes_engine *aes = &cmd->u.aes;
636 struct ccp_dm_workarea key, ctx, final_wa, tag;
637 struct ccp_data src, dst;
638 struct ccp_data aad;
639 struct ccp_op op;
640
641 unsigned long long *final;
642 unsigned int dm_offset;
643 unsigned int authsize;
644 unsigned int jobid;
645 unsigned int ilen;
646 bool in_place = true; /* Default value */
647 int ret;
648
649 struct scatterlist *p_inp, sg_inp[2];
650 struct scatterlist *p_tag, sg_tag[2];
651 struct scatterlist *p_outp, sg_outp[2];
652 struct scatterlist *p_aad;
653
654 if (!aes->iv)
655 return -EINVAL;
656
657 if (!((aes->key_len == AES_KEYSIZE_128) ||
658 (aes->key_len == AES_KEYSIZE_192) ||
659 (aes->key_len == AES_KEYSIZE_256)))
660 return -EINVAL;
661
662 if (!aes->key) /* Gotta have a key SGL */
663 return -EINVAL;
664
665 /* Zero defaults to 16 bytes, the maximum size */
666 authsize = aes->authsize ? aes->authsize : AES_BLOCK_SIZE;
667 switch (authsize) {
668 case 16:
669 case 15:
670 case 14:
671 case 13:
672 case 12:
673 case 8:
674 case 4:
675 break;
676 default:
677 return -EINVAL;
678 }
679
680 /* First, decompose the source buffer into AAD & PT,
681 * and the destination buffer into AAD, CT & tag, or
682 * the input into CT & tag.
683 * It is expected that the input and output SGs will
684 * be valid, even if the AAD and input lengths are 0.
685 */
686 p_aad = aes->src;
687 p_inp = scatterwalk_ffwd(sg_inp, aes->src, aes->aad_len);
688 p_outp = scatterwalk_ffwd(sg_outp, aes->dst, aes->aad_len);
689 if (aes->action == CCP_AES_ACTION_ENCRYPT) {
690 ilen = aes->src_len;
691 p_tag = scatterwalk_ffwd(sg_tag, p_outp, ilen);
692 } else {
693 /* Input length for decryption includes tag */
694 ilen = aes->src_len - authsize;
695 p_tag = scatterwalk_ffwd(sg_tag, p_inp, ilen);
696 }
697
698 jobid = CCP_NEW_JOBID(cmd_q->ccp);
699
700 memset(&op, 0, sizeof(op));
701 op.cmd_q = cmd_q;
702 op.jobid = jobid;
703 op.sb_key = cmd_q->sb_key; /* Pre-allocated */
704 op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */
705 op.init = 1;
706 op.u.aes.type = aes->type;
707
708 /* Copy the key to the LSB */
709 ret = ccp_init_dm_workarea(&key, cmd_q,
710 CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
711 DMA_TO_DEVICE);
712 if (ret)
713 return ret;
714
715 dm_offset = CCP_SB_BYTES - aes->key_len;
716 ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
717 if (ret)
718 goto e_key;
719 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
720 CCP_PASSTHRU_BYTESWAP_256BIT);
721 if (ret) {
722 cmd->engine_error = cmd_q->cmd_error;
723 goto e_key;
724 }
725
726 /* Copy the context (IV) to the LSB.
727 * There is an assumption here that the IV is 96 bits in length, plus
728 * a nonce of 32 bits. If no IV is present, use a zeroed buffer.
729 */
730 ret = ccp_init_dm_workarea(&ctx, cmd_q,
731 CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
732 DMA_BIDIRECTIONAL);
733 if (ret)
734 goto e_key;
735
736 dm_offset = CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES - aes->iv_len;
737 ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
738 if (ret)
739 goto e_ctx;
740
741 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
742 CCP_PASSTHRU_BYTESWAP_256BIT);
743 if (ret) {
744 cmd->engine_error = cmd_q->cmd_error;
745 goto e_ctx;
746 }
747
748 op.init = 1;
749 if (aes->aad_len > 0) {
750 /* Step 1: Run a GHASH over the Additional Authenticated Data */
751 ret = ccp_init_data(&aad, cmd_q, p_aad, aes->aad_len,
752 AES_BLOCK_SIZE,
753 DMA_TO_DEVICE);
754 if (ret)
755 goto e_ctx;
756
757 op.u.aes.mode = CCP_AES_MODE_GHASH;
758 op.u.aes.action = CCP_AES_GHASHAAD;
759
760 while (aad.sg_wa.bytes_left) {
761 ccp_prepare_data(&aad, NULL, &op, AES_BLOCK_SIZE, true);
762
763 ret = cmd_q->ccp->vdata->perform->aes(&op);
764 if (ret) {
765 cmd->engine_error = cmd_q->cmd_error;
766 goto e_aad;
767 }
768
769 ccp_process_data(&aad, NULL, &op);
770 op.init = 0;
771 }
772 }
773
774 op.u.aes.mode = CCP_AES_MODE_GCTR;
775 op.u.aes.action = aes->action;
776
777 if (ilen > 0) {
778 /* Step 2: Run a GCTR over the plaintext */
779 in_place = (sg_virt(p_inp) == sg_virt(p_outp)) ? true : false;
780
781 ret = ccp_init_data(&src, cmd_q, p_inp, ilen,
782 AES_BLOCK_SIZE,
783 in_place ? DMA_BIDIRECTIONAL
784 : DMA_TO_DEVICE);
785 if (ret)
786 goto e_aad;
787
788 if (in_place) {
789 dst = src;
790 } else {
791 ret = ccp_init_data(&dst, cmd_q, p_outp, ilen,
792 AES_BLOCK_SIZE, DMA_FROM_DEVICE);
793 if (ret)
794 goto e_src;
795 }
796
797 op.soc = 0;
798 op.eom = 0;
799 op.init = 1;
800 while (src.sg_wa.bytes_left) {
801 ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true);
802 if (!src.sg_wa.bytes_left) {
803 unsigned int nbytes = ilen % AES_BLOCK_SIZE;
804
805 if (nbytes) {
806 op.eom = 1;
807 op.u.aes.size = (nbytes * 8) - 1;
808 }
809 }
810
811 ret = cmd_q->ccp->vdata->perform->aes(&op);
812 if (ret) {
813 cmd->engine_error = cmd_q->cmd_error;
814 goto e_dst;
815 }
816
817 ccp_process_data(&src, &dst, &op);
818 op.init = 0;
819 }
820 }
821
822 /* Step 3: Update the IV portion of the context with the original IV */
823 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
824 CCP_PASSTHRU_BYTESWAP_256BIT);
825 if (ret) {
826 cmd->engine_error = cmd_q->cmd_error;
827 goto e_dst;
828 }
829
830 ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
831 if (ret)
832 goto e_dst;
833
834 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
835 CCP_PASSTHRU_BYTESWAP_256BIT);
836 if (ret) {
837 cmd->engine_error = cmd_q->cmd_error;
838 goto e_dst;
839 }
840
841 /* Step 4: Concatenate the lengths of the AAD and source, and
842 * hash that 16 byte buffer.
843 */
844 ret = ccp_init_dm_workarea(&final_wa, cmd_q, AES_BLOCK_SIZE,
845 DMA_BIDIRECTIONAL);
846 if (ret)
847 goto e_dst;
848 final = (unsigned long long *) final_wa.address;
849 final[0] = cpu_to_be64(aes->aad_len * 8);
850 final[1] = cpu_to_be64(ilen * 8);
851
852 memset(&op, 0, sizeof(op));
853 op.cmd_q = cmd_q;
854 op.jobid = jobid;
855 op.sb_key = cmd_q->sb_key; /* Pre-allocated */
856 op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */
857 op.init = 1;
858 op.u.aes.type = aes->type;
859 op.u.aes.mode = CCP_AES_MODE_GHASH;
860 op.u.aes.action = CCP_AES_GHASHFINAL;
861 op.src.type = CCP_MEMTYPE_SYSTEM;
862 op.src.u.dma.address = final_wa.dma.address;
863 op.src.u.dma.length = AES_BLOCK_SIZE;
864 op.dst.type = CCP_MEMTYPE_SYSTEM;
865 op.dst.u.dma.address = final_wa.dma.address;
866 op.dst.u.dma.length = AES_BLOCK_SIZE;
867 op.eom = 1;
868 op.u.aes.size = 0;
869 ret = cmd_q->ccp->vdata->perform->aes(&op);
870 if (ret)
871 goto e_final_wa;
872
873 if (aes->action == CCP_AES_ACTION_ENCRYPT) {
874 /* Put the ciphered tag after the ciphertext. */
875 ccp_get_dm_area(&final_wa, 0, p_tag, 0, authsize);
876 } else {
877 /* Does this ciphered tag match the input? */
878 ret = ccp_init_dm_workarea(&tag, cmd_q, authsize,
879 DMA_BIDIRECTIONAL);
880 if (ret)
881 goto e_final_wa;
882 ret = ccp_set_dm_area(&tag, 0, p_tag, 0, authsize);
883 if (ret) {
884 ccp_dm_free(&tag);
885 goto e_final_wa;
886 }
887
888 ret = crypto_memneq(tag.address, final_wa.address,
889 authsize) ? -EBADMSG : 0;
890 ccp_dm_free(&tag);
891 }
892
893 e_final_wa:
894 ccp_dm_free(&final_wa);
895
896 e_dst:
897 if (ilen > 0 && !in_place)
898 ccp_free_data(&dst, cmd_q);
899
900 e_src:
901 if (ilen > 0)
902 ccp_free_data(&src, cmd_q);
903
904 e_aad:
905 if (aes->aad_len)
906 ccp_free_data(&aad, cmd_q);
907
908 e_ctx:
909 ccp_dm_free(&ctx);
910
911 e_key:
912 ccp_dm_free(&key);
913
914 return ret;
915 }
916
917 static noinline_for_stack int
ccp_run_aes_cmd(struct ccp_cmd_queue * cmd_q,struct ccp_cmd * cmd)918 ccp_run_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
919 {
920 struct ccp_aes_engine *aes = &cmd->u.aes;
921 struct ccp_dm_workarea key, ctx;
922 struct ccp_data src, dst;
923 struct ccp_op op;
924 unsigned int dm_offset;
925 bool in_place = false;
926 int ret;
927
928 if (!((aes->key_len == AES_KEYSIZE_128) ||
929 (aes->key_len == AES_KEYSIZE_192) ||
930 (aes->key_len == AES_KEYSIZE_256)))
931 return -EINVAL;
932
933 if (((aes->mode == CCP_AES_MODE_ECB) ||
934 (aes->mode == CCP_AES_MODE_CBC) ||
935 (aes->mode == CCP_AES_MODE_CFB)) &&
936 (aes->src_len & (AES_BLOCK_SIZE - 1)))
937 return -EINVAL;
938
939 if (!aes->key || !aes->src || !aes->dst)
940 return -EINVAL;
941
942 if (aes->mode != CCP_AES_MODE_ECB) {
943 if (aes->iv_len != AES_BLOCK_SIZE)
944 return -EINVAL;
945
946 if (!aes->iv)
947 return -EINVAL;
948 }
949
950 BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1);
951 BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1);
952
953 ret = -EIO;
954 memset(&op, 0, sizeof(op));
955 op.cmd_q = cmd_q;
956 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
957 op.sb_key = cmd_q->sb_key;
958 op.sb_ctx = cmd_q->sb_ctx;
959 op.init = (aes->mode == CCP_AES_MODE_ECB) ? 0 : 1;
960 op.u.aes.type = aes->type;
961 op.u.aes.mode = aes->mode;
962 op.u.aes.action = aes->action;
963
964 /* All supported key sizes fit in a single (32-byte) SB entry
965 * and must be in little endian format. Use the 256-bit byte
966 * swap passthru option to convert from big endian to little
967 * endian.
968 */
969 ret = ccp_init_dm_workarea(&key, cmd_q,
970 CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES,
971 DMA_TO_DEVICE);
972 if (ret)
973 return ret;
974
975 dm_offset = CCP_SB_BYTES - aes->key_len;
976 ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
977 if (ret)
978 goto e_key;
979 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
980 CCP_PASSTHRU_BYTESWAP_256BIT);
981 if (ret) {
982 cmd->engine_error = cmd_q->cmd_error;
983 goto e_key;
984 }
985
986 /* The AES context fits in a single (32-byte) SB entry and
987 * must be in little endian format. Use the 256-bit byte swap
988 * passthru option to convert from big endian to little endian.
989 */
990 ret = ccp_init_dm_workarea(&ctx, cmd_q,
991 CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
992 DMA_BIDIRECTIONAL);
993 if (ret)
994 goto e_key;
995
996 if (aes->mode != CCP_AES_MODE_ECB) {
997 /* Load the AES context - convert to LE */
998 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
999 ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
1000 if (ret)
1001 goto e_ctx;
1002 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1003 CCP_PASSTHRU_BYTESWAP_256BIT);
1004 if (ret) {
1005 cmd->engine_error = cmd_q->cmd_error;
1006 goto e_ctx;
1007 }
1008 }
1009 switch (aes->mode) {
1010 case CCP_AES_MODE_CFB: /* CFB128 only */
1011 case CCP_AES_MODE_CTR:
1012 op.u.aes.size = AES_BLOCK_SIZE * BITS_PER_BYTE - 1;
1013 break;
1014 default:
1015 op.u.aes.size = 0;
1016 }
1017
1018 /* Prepare the input and output data workareas. For in-place
1019 * operations we need to set the dma direction to BIDIRECTIONAL
1020 * and copy the src workarea to the dst workarea.
1021 */
1022 if (sg_virt(aes->src) == sg_virt(aes->dst))
1023 in_place = true;
1024
1025 ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,
1026 AES_BLOCK_SIZE,
1027 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1028 if (ret)
1029 goto e_ctx;
1030
1031 if (in_place) {
1032 dst = src;
1033 } else {
1034 ret = ccp_init_data(&dst, cmd_q, aes->dst, aes->src_len,
1035 AES_BLOCK_SIZE, DMA_FROM_DEVICE);
1036 if (ret)
1037 goto e_src;
1038 }
1039
1040 /* Send data to the CCP AES engine */
1041 while (src.sg_wa.bytes_left) {
1042 ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true);
1043 if (!src.sg_wa.bytes_left) {
1044 op.eom = 1;
1045
1046 /* Since we don't retrieve the AES context in ECB
1047 * mode we have to wait for the operation to complete
1048 * on the last piece of data
1049 */
1050 if (aes->mode == CCP_AES_MODE_ECB)
1051 op.soc = 1;
1052 }
1053
1054 ret = cmd_q->ccp->vdata->perform->aes(&op);
1055 if (ret) {
1056 cmd->engine_error = cmd_q->cmd_error;
1057 goto e_dst;
1058 }
1059
1060 ccp_process_data(&src, &dst, &op);
1061 }
1062
1063 if (aes->mode != CCP_AES_MODE_ECB) {
1064 /* Retrieve the AES context - convert from LE to BE using
1065 * 32-byte (256-bit) byteswapping
1066 */
1067 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1068 CCP_PASSTHRU_BYTESWAP_256BIT);
1069 if (ret) {
1070 cmd->engine_error = cmd_q->cmd_error;
1071 goto e_dst;
1072 }
1073
1074 /* ...but we only need AES_BLOCK_SIZE bytes */
1075 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
1076 ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
1077 }
1078
1079 e_dst:
1080 if (!in_place)
1081 ccp_free_data(&dst, cmd_q);
1082
1083 e_src:
1084 ccp_free_data(&src, cmd_q);
1085
1086 e_ctx:
1087 ccp_dm_free(&ctx);
1088
1089 e_key:
1090 ccp_dm_free(&key);
1091
1092 return ret;
1093 }
1094
1095 static noinline_for_stack int
ccp_run_xts_aes_cmd(struct ccp_cmd_queue * cmd_q,struct ccp_cmd * cmd)1096 ccp_run_xts_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1097 {
1098 struct ccp_xts_aes_engine *xts = &cmd->u.xts;
1099 struct ccp_dm_workarea key, ctx;
1100 struct ccp_data src, dst;
1101 struct ccp_op op;
1102 unsigned int unit_size, dm_offset;
1103 bool in_place = false;
1104 unsigned int sb_count;
1105 enum ccp_aes_type aestype;
1106 int ret;
1107
1108 switch (xts->unit_size) {
1109 case CCP_XTS_AES_UNIT_SIZE_16:
1110 unit_size = 16;
1111 break;
1112 case CCP_XTS_AES_UNIT_SIZE_512:
1113 unit_size = 512;
1114 break;
1115 case CCP_XTS_AES_UNIT_SIZE_1024:
1116 unit_size = 1024;
1117 break;
1118 case CCP_XTS_AES_UNIT_SIZE_2048:
1119 unit_size = 2048;
1120 break;
1121 case CCP_XTS_AES_UNIT_SIZE_4096:
1122 unit_size = 4096;
1123 break;
1124
1125 default:
1126 return -EINVAL;
1127 }
1128
1129 if (xts->key_len == AES_KEYSIZE_128)
1130 aestype = CCP_AES_TYPE_128;
1131 else if (xts->key_len == AES_KEYSIZE_256)
1132 aestype = CCP_AES_TYPE_256;
1133 else
1134 return -EINVAL;
1135
1136 if (!xts->final && (xts->src_len & (AES_BLOCK_SIZE - 1)))
1137 return -EINVAL;
1138
1139 if (xts->iv_len != AES_BLOCK_SIZE)
1140 return -EINVAL;
1141
1142 if (!xts->key || !xts->iv || !xts->src || !xts->dst)
1143 return -EINVAL;
1144
1145 BUILD_BUG_ON(CCP_XTS_AES_KEY_SB_COUNT != 1);
1146 BUILD_BUG_ON(CCP_XTS_AES_CTX_SB_COUNT != 1);
1147
1148 ret = -EIO;
1149 memset(&op, 0, sizeof(op));
1150 op.cmd_q = cmd_q;
1151 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1152 op.sb_key = cmd_q->sb_key;
1153 op.sb_ctx = cmd_q->sb_ctx;
1154 op.init = 1;
1155 op.u.xts.type = aestype;
1156 op.u.xts.action = xts->action;
1157 op.u.xts.unit_size = xts->unit_size;
1158
1159 /* A version 3 device only supports 128-bit keys, which fits into a
1160 * single SB entry. A version 5 device uses a 512-bit vector, so two
1161 * SB entries.
1162 */
1163 if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0))
1164 sb_count = CCP_XTS_AES_KEY_SB_COUNT;
1165 else
1166 sb_count = CCP5_XTS_AES_KEY_SB_COUNT;
1167 ret = ccp_init_dm_workarea(&key, cmd_q,
1168 sb_count * CCP_SB_BYTES,
1169 DMA_TO_DEVICE);
1170 if (ret)
1171 return ret;
1172
1173 if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) {
1174 /* All supported key sizes must be in little endian format.
1175 * Use the 256-bit byte swap passthru option to convert from
1176 * big endian to little endian.
1177 */
1178 dm_offset = CCP_SB_BYTES - AES_KEYSIZE_128;
1179 ret = ccp_set_dm_area(&key, dm_offset, xts->key, 0, xts->key_len);
1180 if (ret)
1181 goto e_key;
1182 ret = ccp_set_dm_area(&key, 0, xts->key, xts->key_len, xts->key_len);
1183 if (ret)
1184 goto e_key;
1185 } else {
1186 /* Version 5 CCPs use a 512-bit space for the key: each portion
1187 * occupies 256 bits, or one entire slot, and is zero-padded.
1188 */
1189 unsigned int pad;
1190
1191 dm_offset = CCP_SB_BYTES;
1192 pad = dm_offset - xts->key_len;
1193 ret = ccp_set_dm_area(&key, pad, xts->key, 0, xts->key_len);
1194 if (ret)
1195 goto e_key;
1196 ret = ccp_set_dm_area(&key, dm_offset + pad, xts->key,
1197 xts->key_len, xts->key_len);
1198 if (ret)
1199 goto e_key;
1200 }
1201 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
1202 CCP_PASSTHRU_BYTESWAP_256BIT);
1203 if (ret) {
1204 cmd->engine_error = cmd_q->cmd_error;
1205 goto e_key;
1206 }
1207
1208 /* The AES context fits in a single (32-byte) SB entry and
1209 * for XTS is already in little endian format so no byte swapping
1210 * is needed.
1211 */
1212 ret = ccp_init_dm_workarea(&ctx, cmd_q,
1213 CCP_XTS_AES_CTX_SB_COUNT * CCP_SB_BYTES,
1214 DMA_BIDIRECTIONAL);
1215 if (ret)
1216 goto e_key;
1217
1218 ret = ccp_set_dm_area(&ctx, 0, xts->iv, 0, xts->iv_len);
1219 if (ret)
1220 goto e_ctx;
1221 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1222 CCP_PASSTHRU_BYTESWAP_NOOP);
1223 if (ret) {
1224 cmd->engine_error = cmd_q->cmd_error;
1225 goto e_ctx;
1226 }
1227
1228 /* Prepare the input and output data workareas. For in-place
1229 * operations we need to set the dma direction to BIDIRECTIONAL
1230 * and copy the src workarea to the dst workarea.
1231 */
1232 if (sg_virt(xts->src) == sg_virt(xts->dst))
1233 in_place = true;
1234
1235 ret = ccp_init_data(&src, cmd_q, xts->src, xts->src_len,
1236 unit_size,
1237 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1238 if (ret)
1239 goto e_ctx;
1240
1241 if (in_place) {
1242 dst = src;
1243 } else {
1244 ret = ccp_init_data(&dst, cmd_q, xts->dst, xts->src_len,
1245 unit_size, DMA_FROM_DEVICE);
1246 if (ret)
1247 goto e_src;
1248 }
1249
1250 /* Send data to the CCP AES engine */
1251 while (src.sg_wa.bytes_left) {
1252 ccp_prepare_data(&src, &dst, &op, unit_size, true);
1253 if (!src.sg_wa.bytes_left)
1254 op.eom = 1;
1255
1256 ret = cmd_q->ccp->vdata->perform->xts_aes(&op);
1257 if (ret) {
1258 cmd->engine_error = cmd_q->cmd_error;
1259 goto e_dst;
1260 }
1261
1262 ccp_process_data(&src, &dst, &op);
1263 }
1264
1265 /* Retrieve the AES context - convert from LE to BE using
1266 * 32-byte (256-bit) byteswapping
1267 */
1268 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1269 CCP_PASSTHRU_BYTESWAP_256BIT);
1270 if (ret) {
1271 cmd->engine_error = cmd_q->cmd_error;
1272 goto e_dst;
1273 }
1274
1275 /* ...but we only need AES_BLOCK_SIZE bytes */
1276 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
1277 ccp_get_dm_area(&ctx, dm_offset, xts->iv, 0, xts->iv_len);
1278
1279 e_dst:
1280 if (!in_place)
1281 ccp_free_data(&dst, cmd_q);
1282
1283 e_src:
1284 ccp_free_data(&src, cmd_q);
1285
1286 e_ctx:
1287 ccp_dm_free(&ctx);
1288
1289 e_key:
1290 ccp_dm_free(&key);
1291
1292 return ret;
1293 }
1294
1295 static noinline_for_stack int
ccp_run_des3_cmd(struct ccp_cmd_queue * cmd_q,struct ccp_cmd * cmd)1296 ccp_run_des3_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1297 {
1298 struct ccp_des3_engine *des3 = &cmd->u.des3;
1299
1300 struct ccp_dm_workarea key, ctx;
1301 struct ccp_data src, dst;
1302 struct ccp_op op;
1303 unsigned int dm_offset;
1304 unsigned int len_singlekey;
1305 bool in_place = false;
1306 int ret;
1307
1308 /* Error checks */
1309 if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0))
1310 return -EINVAL;
1311
1312 if (!cmd_q->ccp->vdata->perform->des3)
1313 return -EINVAL;
1314
1315 if (des3->key_len != DES3_EDE_KEY_SIZE)
1316 return -EINVAL;
1317
1318 if (((des3->mode == CCP_DES3_MODE_ECB) ||
1319 (des3->mode == CCP_DES3_MODE_CBC)) &&
1320 (des3->src_len & (DES3_EDE_BLOCK_SIZE - 1)))
1321 return -EINVAL;
1322
1323 if (!des3->key || !des3->src || !des3->dst)
1324 return -EINVAL;
1325
1326 if (des3->mode != CCP_DES3_MODE_ECB) {
1327 if (des3->iv_len != DES3_EDE_BLOCK_SIZE)
1328 return -EINVAL;
1329
1330 if (!des3->iv)
1331 return -EINVAL;
1332 }
1333
1334 ret = -EIO;
1335 /* Zero out all the fields of the command desc */
1336 memset(&op, 0, sizeof(op));
1337
1338 /* Set up the Function field */
1339 op.cmd_q = cmd_q;
1340 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1341 op.sb_key = cmd_q->sb_key;
1342
1343 op.init = (des3->mode == CCP_DES3_MODE_ECB) ? 0 : 1;
1344 op.u.des3.type = des3->type;
1345 op.u.des3.mode = des3->mode;
1346 op.u.des3.action = des3->action;
1347
1348 /*
1349 * All supported key sizes fit in a single (32-byte) KSB entry and
1350 * (like AES) must be in little endian format. Use the 256-bit byte
1351 * swap passthru option to convert from big endian to little endian.
1352 */
1353 ret = ccp_init_dm_workarea(&key, cmd_q,
1354 CCP_DES3_KEY_SB_COUNT * CCP_SB_BYTES,
1355 DMA_TO_DEVICE);
1356 if (ret)
1357 return ret;
1358
1359 /*
1360 * The contents of the key triplet are in the reverse order of what
1361 * is required by the engine. Copy the 3 pieces individually to put
1362 * them where they belong.
1363 */
1364 dm_offset = CCP_SB_BYTES - des3->key_len; /* Basic offset */
1365
1366 len_singlekey = des3->key_len / 3;
1367 ret = ccp_set_dm_area(&key, dm_offset + 2 * len_singlekey,
1368 des3->key, 0, len_singlekey);
1369 if (ret)
1370 goto e_key;
1371 ret = ccp_set_dm_area(&key, dm_offset + len_singlekey,
1372 des3->key, len_singlekey, len_singlekey);
1373 if (ret)
1374 goto e_key;
1375 ret = ccp_set_dm_area(&key, dm_offset,
1376 des3->key, 2 * len_singlekey, len_singlekey);
1377 if (ret)
1378 goto e_key;
1379
1380 /* Copy the key to the SB */
1381 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
1382 CCP_PASSTHRU_BYTESWAP_256BIT);
1383 if (ret) {
1384 cmd->engine_error = cmd_q->cmd_error;
1385 goto e_key;
1386 }
1387
1388 /*
1389 * The DES3 context fits in a single (32-byte) KSB entry and
1390 * must be in little endian format. Use the 256-bit byte swap
1391 * passthru option to convert from big endian to little endian.
1392 */
1393 if (des3->mode != CCP_DES3_MODE_ECB) {
1394 op.sb_ctx = cmd_q->sb_ctx;
1395
1396 ret = ccp_init_dm_workarea(&ctx, cmd_q,
1397 CCP_DES3_CTX_SB_COUNT * CCP_SB_BYTES,
1398 DMA_BIDIRECTIONAL);
1399 if (ret)
1400 goto e_key;
1401
1402 /* Load the context into the LSB */
1403 dm_offset = CCP_SB_BYTES - des3->iv_len;
1404 ret = ccp_set_dm_area(&ctx, dm_offset, des3->iv, 0,
1405 des3->iv_len);
1406 if (ret)
1407 goto e_ctx;
1408
1409 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1410 CCP_PASSTHRU_BYTESWAP_256BIT);
1411 if (ret) {
1412 cmd->engine_error = cmd_q->cmd_error;
1413 goto e_ctx;
1414 }
1415 }
1416
1417 /*
1418 * Prepare the input and output data workareas. For in-place
1419 * operations we need to set the dma direction to BIDIRECTIONAL
1420 * and copy the src workarea to the dst workarea.
1421 */
1422 if (sg_virt(des3->src) == sg_virt(des3->dst))
1423 in_place = true;
1424
1425 ret = ccp_init_data(&src, cmd_q, des3->src, des3->src_len,
1426 DES3_EDE_BLOCK_SIZE,
1427 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1428 if (ret)
1429 goto e_ctx;
1430
1431 if (in_place)
1432 dst = src;
1433 else {
1434 ret = ccp_init_data(&dst, cmd_q, des3->dst, des3->src_len,
1435 DES3_EDE_BLOCK_SIZE, DMA_FROM_DEVICE);
1436 if (ret)
1437 goto e_src;
1438 }
1439
1440 /* Send data to the CCP DES3 engine */
1441 while (src.sg_wa.bytes_left) {
1442 ccp_prepare_data(&src, &dst, &op, DES3_EDE_BLOCK_SIZE, true);
1443 if (!src.sg_wa.bytes_left) {
1444 op.eom = 1;
1445
1446 /* Since we don't retrieve the context in ECB mode
1447 * we have to wait for the operation to complete
1448 * on the last piece of data
1449 */
1450 op.soc = 0;
1451 }
1452
1453 ret = cmd_q->ccp->vdata->perform->des3(&op);
1454 if (ret) {
1455 cmd->engine_error = cmd_q->cmd_error;
1456 goto e_dst;
1457 }
1458
1459 ccp_process_data(&src, &dst, &op);
1460 }
1461
1462 if (des3->mode != CCP_DES3_MODE_ECB) {
1463 /* Retrieve the context and make BE */
1464 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1465 CCP_PASSTHRU_BYTESWAP_256BIT);
1466 if (ret) {
1467 cmd->engine_error = cmd_q->cmd_error;
1468 goto e_dst;
1469 }
1470
1471 /* ...but we only need the last DES3_EDE_BLOCK_SIZE bytes */
1472 ccp_get_dm_area(&ctx, dm_offset, des3->iv, 0,
1473 DES3_EDE_BLOCK_SIZE);
1474 }
1475 e_dst:
1476 if (!in_place)
1477 ccp_free_data(&dst, cmd_q);
1478
1479 e_src:
1480 ccp_free_data(&src, cmd_q);
1481
1482 e_ctx:
1483 if (des3->mode != CCP_DES3_MODE_ECB)
1484 ccp_dm_free(&ctx);
1485
1486 e_key:
1487 ccp_dm_free(&key);
1488
1489 return ret;
1490 }
1491
1492 static noinline_for_stack int
ccp_run_sha_cmd(struct ccp_cmd_queue * cmd_q,struct ccp_cmd * cmd)1493 ccp_run_sha_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1494 {
1495 struct ccp_sha_engine *sha = &cmd->u.sha;
1496 struct ccp_dm_workarea ctx;
1497 struct ccp_data src;
1498 struct ccp_op op;
1499 unsigned int ioffset, ooffset;
1500 unsigned int digest_size;
1501 int sb_count;
1502 const void *init;
1503 u64 block_size;
1504 int ctx_size;
1505 int ret;
1506
1507 switch (sha->type) {
1508 case CCP_SHA_TYPE_1:
1509 if (sha->ctx_len < SHA1_DIGEST_SIZE)
1510 return -EINVAL;
1511 block_size = SHA1_BLOCK_SIZE;
1512 break;
1513 case CCP_SHA_TYPE_224:
1514 if (sha->ctx_len < SHA224_DIGEST_SIZE)
1515 return -EINVAL;
1516 block_size = SHA224_BLOCK_SIZE;
1517 break;
1518 case CCP_SHA_TYPE_256:
1519 if (sha->ctx_len < SHA256_DIGEST_SIZE)
1520 return -EINVAL;
1521 block_size = SHA256_BLOCK_SIZE;
1522 break;
1523 case CCP_SHA_TYPE_384:
1524 if (cmd_q->ccp->vdata->version < CCP_VERSION(4, 0)
1525 || sha->ctx_len < SHA384_DIGEST_SIZE)
1526 return -EINVAL;
1527 block_size = SHA384_BLOCK_SIZE;
1528 break;
1529 case CCP_SHA_TYPE_512:
1530 if (cmd_q->ccp->vdata->version < CCP_VERSION(4, 0)
1531 || sha->ctx_len < SHA512_DIGEST_SIZE)
1532 return -EINVAL;
1533 block_size = SHA512_BLOCK_SIZE;
1534 break;
1535 default:
1536 return -EINVAL;
1537 }
1538
1539 if (!sha->ctx)
1540 return -EINVAL;
1541
1542 if (!sha->final && (sha->src_len & (block_size - 1)))
1543 return -EINVAL;
1544
1545 /* The version 3 device can't handle zero-length input */
1546 if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) {
1547
1548 if (!sha->src_len) {
1549 unsigned int digest_len;
1550 const u8 *sha_zero;
1551
1552 /* Not final, just return */
1553 if (!sha->final)
1554 return 0;
1555
1556 /* CCP can't do a zero length sha operation so the
1557 * caller must buffer the data.
1558 */
1559 if (sha->msg_bits)
1560 return -EINVAL;
1561
1562 /* The CCP cannot perform zero-length sha operations
1563 * so the caller is required to buffer data for the
1564 * final operation. However, a sha operation for a
1565 * message with a total length of zero is valid so
1566 * known values are required to supply the result.
1567 */
1568 switch (sha->type) {
1569 case CCP_SHA_TYPE_1:
1570 sha_zero = sha1_zero_message_hash;
1571 digest_len = SHA1_DIGEST_SIZE;
1572 break;
1573 case CCP_SHA_TYPE_224:
1574 sha_zero = sha224_zero_message_hash;
1575 digest_len = SHA224_DIGEST_SIZE;
1576 break;
1577 case CCP_SHA_TYPE_256:
1578 sha_zero = sha256_zero_message_hash;
1579 digest_len = SHA256_DIGEST_SIZE;
1580 break;
1581 default:
1582 return -EINVAL;
1583 }
1584
1585 scatterwalk_map_and_copy((void *)sha_zero, sha->ctx, 0,
1586 digest_len, 1);
1587
1588 return 0;
1589 }
1590 }
1591
1592 /* Set variables used throughout */
1593 switch (sha->type) {
1594 case CCP_SHA_TYPE_1:
1595 digest_size = SHA1_DIGEST_SIZE;
1596 init = (void *) ccp_sha1_init;
1597 ctx_size = SHA1_DIGEST_SIZE;
1598 sb_count = 1;
1599 if (cmd_q->ccp->vdata->version != CCP_VERSION(3, 0))
1600 ooffset = ioffset = CCP_SB_BYTES - SHA1_DIGEST_SIZE;
1601 else
1602 ooffset = ioffset = 0;
1603 break;
1604 case CCP_SHA_TYPE_224:
1605 digest_size = SHA224_DIGEST_SIZE;
1606 init = (void *) ccp_sha224_init;
1607 ctx_size = SHA256_DIGEST_SIZE;
1608 sb_count = 1;
1609 ioffset = 0;
1610 if (cmd_q->ccp->vdata->version != CCP_VERSION(3, 0))
1611 ooffset = CCP_SB_BYTES - SHA224_DIGEST_SIZE;
1612 else
1613 ooffset = 0;
1614 break;
1615 case CCP_SHA_TYPE_256:
1616 digest_size = SHA256_DIGEST_SIZE;
1617 init = (void *) ccp_sha256_init;
1618 ctx_size = SHA256_DIGEST_SIZE;
1619 sb_count = 1;
1620 ooffset = ioffset = 0;
1621 break;
1622 case CCP_SHA_TYPE_384:
1623 digest_size = SHA384_DIGEST_SIZE;
1624 init = (void *) ccp_sha384_init;
1625 ctx_size = SHA512_DIGEST_SIZE;
1626 sb_count = 2;
1627 ioffset = 0;
1628 ooffset = 2 * CCP_SB_BYTES - SHA384_DIGEST_SIZE;
1629 break;
1630 case CCP_SHA_TYPE_512:
1631 digest_size = SHA512_DIGEST_SIZE;
1632 init = (void *) ccp_sha512_init;
1633 ctx_size = SHA512_DIGEST_SIZE;
1634 sb_count = 2;
1635 ooffset = ioffset = 0;
1636 break;
1637 default:
1638 ret = -EINVAL;
1639 goto e_data;
1640 }
1641
1642 /* For zero-length plaintext the src pointer is ignored;
1643 * otherwise both parts must be valid
1644 */
1645 if (sha->src_len && !sha->src)
1646 return -EINVAL;
1647
1648 memset(&op, 0, sizeof(op));
1649 op.cmd_q = cmd_q;
1650 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1651 op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */
1652 op.u.sha.type = sha->type;
1653 op.u.sha.msg_bits = sha->msg_bits;
1654
1655 /* For SHA1/224/256 the context fits in a single (32-byte) SB entry;
1656 * SHA384/512 require 2 adjacent SB slots, with the right half in the
1657 * first slot, and the left half in the second. Each portion must then
1658 * be in little endian format: use the 256-bit byte swap option.
1659 */
1660 ret = ccp_init_dm_workarea(&ctx, cmd_q, sb_count * CCP_SB_BYTES,
1661 DMA_BIDIRECTIONAL);
1662 if (ret)
1663 return ret;
1664 if (sha->first) {
1665 switch (sha->type) {
1666 case CCP_SHA_TYPE_1:
1667 case CCP_SHA_TYPE_224:
1668 case CCP_SHA_TYPE_256:
1669 memcpy(ctx.address + ioffset, init, ctx_size);
1670 break;
1671 case CCP_SHA_TYPE_384:
1672 case CCP_SHA_TYPE_512:
1673 memcpy(ctx.address + ctx_size / 2, init,
1674 ctx_size / 2);
1675 memcpy(ctx.address, init + ctx_size / 2,
1676 ctx_size / 2);
1677 break;
1678 default:
1679 ret = -EINVAL;
1680 goto e_ctx;
1681 }
1682 } else {
1683 /* Restore the context */
1684 ret = ccp_set_dm_area(&ctx, 0, sha->ctx, 0,
1685 sb_count * CCP_SB_BYTES);
1686 if (ret)
1687 goto e_ctx;
1688 }
1689
1690 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1691 CCP_PASSTHRU_BYTESWAP_256BIT);
1692 if (ret) {
1693 cmd->engine_error = cmd_q->cmd_error;
1694 goto e_ctx;
1695 }
1696
1697 if (sha->src) {
1698 /* Send data to the CCP SHA engine; block_size is set above */
1699 ret = ccp_init_data(&src, cmd_q, sha->src, sha->src_len,
1700 block_size, DMA_TO_DEVICE);
1701 if (ret)
1702 goto e_ctx;
1703
1704 while (src.sg_wa.bytes_left) {
1705 ccp_prepare_data(&src, NULL, &op, block_size, false);
1706 if (sha->final && !src.sg_wa.bytes_left)
1707 op.eom = 1;
1708
1709 ret = cmd_q->ccp->vdata->perform->sha(&op);
1710 if (ret) {
1711 cmd->engine_error = cmd_q->cmd_error;
1712 goto e_data;
1713 }
1714
1715 ccp_process_data(&src, NULL, &op);
1716 }
1717 } else {
1718 op.eom = 1;
1719 ret = cmd_q->ccp->vdata->perform->sha(&op);
1720 if (ret) {
1721 cmd->engine_error = cmd_q->cmd_error;
1722 goto e_data;
1723 }
1724 }
1725
1726 /* Retrieve the SHA context - convert from LE to BE using
1727 * 32-byte (256-bit) byteswapping to BE
1728 */
1729 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1730 CCP_PASSTHRU_BYTESWAP_256BIT);
1731 if (ret) {
1732 cmd->engine_error = cmd_q->cmd_error;
1733 goto e_data;
1734 }
1735
1736 if (sha->final) {
1737 /* Finishing up, so get the digest */
1738 switch (sha->type) {
1739 case CCP_SHA_TYPE_1:
1740 case CCP_SHA_TYPE_224:
1741 case CCP_SHA_TYPE_256:
1742 ccp_get_dm_area(&ctx, ooffset,
1743 sha->ctx, 0,
1744 digest_size);
1745 break;
1746 case CCP_SHA_TYPE_384:
1747 case CCP_SHA_TYPE_512:
1748 ccp_get_dm_area(&ctx, 0,
1749 sha->ctx, LSB_ITEM_SIZE - ooffset,
1750 LSB_ITEM_SIZE);
1751 ccp_get_dm_area(&ctx, LSB_ITEM_SIZE + ooffset,
1752 sha->ctx, 0,
1753 LSB_ITEM_SIZE - ooffset);
1754 break;
1755 default:
1756 ret = -EINVAL;
1757 goto e_data;
1758 }
1759 } else {
1760 /* Stash the context */
1761 ccp_get_dm_area(&ctx, 0, sha->ctx, 0,
1762 sb_count * CCP_SB_BYTES);
1763 }
1764
1765 if (sha->final && sha->opad) {
1766 /* HMAC operation, recursively perform final SHA */
1767 struct ccp_cmd hmac_cmd;
1768 struct scatterlist sg;
1769 u8 *hmac_buf;
1770
1771 if (sha->opad_len != block_size) {
1772 ret = -EINVAL;
1773 goto e_data;
1774 }
1775
1776 hmac_buf = kmalloc(block_size + digest_size, GFP_KERNEL);
1777 if (!hmac_buf) {
1778 ret = -ENOMEM;
1779 goto e_data;
1780 }
1781 sg_init_one(&sg, hmac_buf, block_size + digest_size);
1782
1783 scatterwalk_map_and_copy(hmac_buf, sha->opad, 0, block_size, 0);
1784 switch (sha->type) {
1785 case CCP_SHA_TYPE_1:
1786 case CCP_SHA_TYPE_224:
1787 case CCP_SHA_TYPE_256:
1788 memcpy(hmac_buf + block_size,
1789 ctx.address + ooffset,
1790 digest_size);
1791 break;
1792 case CCP_SHA_TYPE_384:
1793 case CCP_SHA_TYPE_512:
1794 memcpy(hmac_buf + block_size,
1795 ctx.address + LSB_ITEM_SIZE + ooffset,
1796 LSB_ITEM_SIZE);
1797 memcpy(hmac_buf + block_size +
1798 (LSB_ITEM_SIZE - ooffset),
1799 ctx.address,
1800 LSB_ITEM_SIZE);
1801 break;
1802 default:
1803 kfree(hmac_buf);
1804 ret = -EINVAL;
1805 goto e_data;
1806 }
1807
1808 memset(&hmac_cmd, 0, sizeof(hmac_cmd));
1809 hmac_cmd.engine = CCP_ENGINE_SHA;
1810 hmac_cmd.u.sha.type = sha->type;
1811 hmac_cmd.u.sha.ctx = sha->ctx;
1812 hmac_cmd.u.sha.ctx_len = sha->ctx_len;
1813 hmac_cmd.u.sha.src = &sg;
1814 hmac_cmd.u.sha.src_len = block_size + digest_size;
1815 hmac_cmd.u.sha.opad = NULL;
1816 hmac_cmd.u.sha.opad_len = 0;
1817 hmac_cmd.u.sha.first = 1;
1818 hmac_cmd.u.sha.final = 1;
1819 hmac_cmd.u.sha.msg_bits = (block_size + digest_size) << 3;
1820
1821 ret = ccp_run_sha_cmd(cmd_q, &hmac_cmd);
1822 if (ret)
1823 cmd->engine_error = hmac_cmd.engine_error;
1824
1825 kfree(hmac_buf);
1826 }
1827
1828 e_data:
1829 if (sha->src)
1830 ccp_free_data(&src, cmd_q);
1831
1832 e_ctx:
1833 ccp_dm_free(&ctx);
1834
1835 return ret;
1836 }
1837
1838 static noinline_for_stack int
ccp_run_rsa_cmd(struct ccp_cmd_queue * cmd_q,struct ccp_cmd * cmd)1839 ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1840 {
1841 struct ccp_rsa_engine *rsa = &cmd->u.rsa;
1842 struct ccp_dm_workarea exp, src, dst;
1843 struct ccp_op op;
1844 unsigned int sb_count, i_len, o_len;
1845 int ret;
1846
1847 /* Check against the maximum allowable size, in bits */
1848 if (rsa->key_size > cmd_q->ccp->vdata->rsamax)
1849 return -EINVAL;
1850
1851 if (!rsa->exp || !rsa->mod || !rsa->src || !rsa->dst)
1852 return -EINVAL;
1853
1854 memset(&op, 0, sizeof(op));
1855 op.cmd_q = cmd_q;
1856 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1857
1858 /* The RSA modulus must precede the message being acted upon, so
1859 * it must be copied to a DMA area where the message and the
1860 * modulus can be concatenated. Therefore the input buffer
1861 * length required is twice the output buffer length (which
1862 * must be a multiple of 256-bits). Compute o_len, i_len in bytes.
1863 * Buffer sizes must be a multiple of 32 bytes; rounding up may be
1864 * required.
1865 */
1866 o_len = 32 * ((rsa->key_size + 255) / 256);
1867 i_len = o_len * 2;
1868
1869 sb_count = 0;
1870 if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0)) {
1871 /* sb_count is the number of storage block slots required
1872 * for the modulus.
1873 */
1874 sb_count = o_len / CCP_SB_BYTES;
1875 op.sb_key = cmd_q->ccp->vdata->perform->sballoc(cmd_q,
1876 sb_count);
1877 if (!op.sb_key)
1878 return -EIO;
1879 } else {
1880 /* A version 5 device allows a modulus size that will not fit
1881 * in the LSB, so the command will transfer it from memory.
1882 * Set the sb key to the default, even though it's not used.
1883 */
1884 op.sb_key = cmd_q->sb_key;
1885 }
1886
1887 /* The RSA exponent must be in little endian format. Reverse its
1888 * byte order.
1889 */
1890 ret = ccp_init_dm_workarea(&exp, cmd_q, o_len, DMA_TO_DEVICE);
1891 if (ret)
1892 goto e_sb;
1893
1894 ret = ccp_reverse_set_dm_area(&exp, 0, rsa->exp, 0, rsa->exp_len);
1895 if (ret)
1896 goto e_exp;
1897
1898 if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0)) {
1899 /* Copy the exponent to the local storage block, using
1900 * as many 32-byte blocks as were allocated above. It's
1901 * already little endian, so no further change is required.
1902 */
1903 ret = ccp_copy_to_sb(cmd_q, &exp, op.jobid, op.sb_key,
1904 CCP_PASSTHRU_BYTESWAP_NOOP);
1905 if (ret) {
1906 cmd->engine_error = cmd_q->cmd_error;
1907 goto e_exp;
1908 }
1909 } else {
1910 /* The exponent can be retrieved from memory via DMA. */
1911 op.exp.u.dma.address = exp.dma.address;
1912 op.exp.u.dma.offset = 0;
1913 }
1914
1915 /* Concatenate the modulus and the message. Both the modulus and
1916 * the operands must be in little endian format. Since the input
1917 * is in big endian format it must be converted.
1918 */
1919 ret = ccp_init_dm_workarea(&src, cmd_q, i_len, DMA_TO_DEVICE);
1920 if (ret)
1921 goto e_exp;
1922
1923 ret = ccp_reverse_set_dm_area(&src, 0, rsa->mod, 0, rsa->mod_len);
1924 if (ret)
1925 goto e_src;
1926 ret = ccp_reverse_set_dm_area(&src, o_len, rsa->src, 0, rsa->src_len);
1927 if (ret)
1928 goto e_src;
1929
1930 /* Prepare the output area for the operation */
1931 ret = ccp_init_dm_workarea(&dst, cmd_q, o_len, DMA_FROM_DEVICE);
1932 if (ret)
1933 goto e_src;
1934
1935 op.soc = 1;
1936 op.src.u.dma.address = src.dma.address;
1937 op.src.u.dma.offset = 0;
1938 op.src.u.dma.length = i_len;
1939 op.dst.u.dma.address = dst.dma.address;
1940 op.dst.u.dma.offset = 0;
1941 op.dst.u.dma.length = o_len;
1942
1943 op.u.rsa.mod_size = rsa->key_size;
1944 op.u.rsa.input_len = i_len;
1945
1946 ret = cmd_q->ccp->vdata->perform->rsa(&op);
1947 if (ret) {
1948 cmd->engine_error = cmd_q->cmd_error;
1949 goto e_dst;
1950 }
1951
1952 ccp_reverse_get_dm_area(&dst, 0, rsa->dst, 0, rsa->mod_len);
1953
1954 e_dst:
1955 ccp_dm_free(&dst);
1956
1957 e_src:
1958 ccp_dm_free(&src);
1959
1960 e_exp:
1961 ccp_dm_free(&exp);
1962
1963 e_sb:
1964 if (sb_count)
1965 cmd_q->ccp->vdata->perform->sbfree(cmd_q, op.sb_key, sb_count);
1966
1967 return ret;
1968 }
1969
1970 static noinline_for_stack int
ccp_run_passthru_cmd(struct ccp_cmd_queue * cmd_q,struct ccp_cmd * cmd)1971 ccp_run_passthru_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1972 {
1973 struct ccp_passthru_engine *pt = &cmd->u.passthru;
1974 struct ccp_dm_workarea mask;
1975 struct ccp_data src, dst;
1976 struct ccp_op op;
1977 bool in_place = false;
1978 unsigned int i;
1979 int ret = 0;
1980
1981 if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1)))
1982 return -EINVAL;
1983
1984 if (!pt->src || !pt->dst)
1985 return -EINVAL;
1986
1987 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
1988 if (pt->mask_len != CCP_PASSTHRU_MASKSIZE)
1989 return -EINVAL;
1990 if (!pt->mask)
1991 return -EINVAL;
1992 }
1993
1994 BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1);
1995
1996 memset(&op, 0, sizeof(op));
1997 op.cmd_q = cmd_q;
1998 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1999
2000 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
2001 /* Load the mask */
2002 op.sb_key = cmd_q->sb_key;
2003
2004 ret = ccp_init_dm_workarea(&mask, cmd_q,
2005 CCP_PASSTHRU_SB_COUNT *
2006 CCP_SB_BYTES,
2007 DMA_TO_DEVICE);
2008 if (ret)
2009 return ret;
2010
2011 ret = ccp_set_dm_area(&mask, 0, pt->mask, 0, pt->mask_len);
2012 if (ret)
2013 goto e_mask;
2014 ret = ccp_copy_to_sb(cmd_q, &mask, op.jobid, op.sb_key,
2015 CCP_PASSTHRU_BYTESWAP_NOOP);
2016 if (ret) {
2017 cmd->engine_error = cmd_q->cmd_error;
2018 goto e_mask;
2019 }
2020 }
2021
2022 /* Prepare the input and output data workareas. For in-place
2023 * operations we need to set the dma direction to BIDIRECTIONAL
2024 * and copy the src workarea to the dst workarea.
2025 */
2026 if (sg_virt(pt->src) == sg_virt(pt->dst))
2027 in_place = true;
2028
2029 ret = ccp_init_data(&src, cmd_q, pt->src, pt->src_len,
2030 CCP_PASSTHRU_MASKSIZE,
2031 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
2032 if (ret)
2033 goto e_mask;
2034
2035 if (in_place) {
2036 dst = src;
2037 } else {
2038 ret = ccp_init_data(&dst, cmd_q, pt->dst, pt->src_len,
2039 CCP_PASSTHRU_MASKSIZE, DMA_FROM_DEVICE);
2040 if (ret)
2041 goto e_src;
2042 }
2043
2044 /* Send data to the CCP Passthru engine
2045 * Because the CCP engine works on a single source and destination
2046 * dma address at a time, each entry in the source scatterlist
2047 * (after the dma_map_sg call) must be less than or equal to the
2048 * (remaining) length in the destination scatterlist entry and the
2049 * length must be a multiple of CCP_PASSTHRU_BLOCKSIZE
2050 */
2051 dst.sg_wa.sg_used = 0;
2052 for (i = 1; i <= src.sg_wa.dma_count; i++) {
2053 if (!dst.sg_wa.sg ||
2054 (sg_dma_len(dst.sg_wa.sg) < sg_dma_len(src.sg_wa.sg))) {
2055 ret = -EINVAL;
2056 goto e_dst;
2057 }
2058
2059 if (i == src.sg_wa.dma_count) {
2060 op.eom = 1;
2061 op.soc = 1;
2062 }
2063
2064 op.src.type = CCP_MEMTYPE_SYSTEM;
2065 op.src.u.dma.address = sg_dma_address(src.sg_wa.sg);
2066 op.src.u.dma.offset = 0;
2067 op.src.u.dma.length = sg_dma_len(src.sg_wa.sg);
2068
2069 op.dst.type = CCP_MEMTYPE_SYSTEM;
2070 op.dst.u.dma.address = sg_dma_address(dst.sg_wa.sg);
2071 op.dst.u.dma.offset = dst.sg_wa.sg_used;
2072 op.dst.u.dma.length = op.src.u.dma.length;
2073
2074 ret = cmd_q->ccp->vdata->perform->passthru(&op);
2075 if (ret) {
2076 cmd->engine_error = cmd_q->cmd_error;
2077 goto e_dst;
2078 }
2079
2080 dst.sg_wa.sg_used += sg_dma_len(src.sg_wa.sg);
2081 if (dst.sg_wa.sg_used == sg_dma_len(dst.sg_wa.sg)) {
2082 dst.sg_wa.sg = sg_next(dst.sg_wa.sg);
2083 dst.sg_wa.sg_used = 0;
2084 }
2085 src.sg_wa.sg = sg_next(src.sg_wa.sg);
2086 }
2087
2088 e_dst:
2089 if (!in_place)
2090 ccp_free_data(&dst, cmd_q);
2091
2092 e_src:
2093 ccp_free_data(&src, cmd_q);
2094
2095 e_mask:
2096 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP)
2097 ccp_dm_free(&mask);
2098
2099 return ret;
2100 }
2101
2102 static noinline_for_stack int
ccp_run_passthru_nomap_cmd(struct ccp_cmd_queue * cmd_q,struct ccp_cmd * cmd)2103 ccp_run_passthru_nomap_cmd(struct ccp_cmd_queue *cmd_q,
2104 struct ccp_cmd *cmd)
2105 {
2106 struct ccp_passthru_nomap_engine *pt = &cmd->u.passthru_nomap;
2107 struct ccp_dm_workarea mask;
2108 struct ccp_op op;
2109 int ret;
2110
2111 if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1)))
2112 return -EINVAL;
2113
2114 if (!pt->src_dma || !pt->dst_dma)
2115 return -EINVAL;
2116
2117 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
2118 if (pt->mask_len != CCP_PASSTHRU_MASKSIZE)
2119 return -EINVAL;
2120 if (!pt->mask)
2121 return -EINVAL;
2122 }
2123
2124 BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1);
2125
2126 memset(&op, 0, sizeof(op));
2127 op.cmd_q = cmd_q;
2128 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
2129
2130 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
2131 /* Load the mask */
2132 op.sb_key = cmd_q->sb_key;
2133
2134 mask.length = pt->mask_len;
2135 mask.dma.address = pt->mask;
2136 mask.dma.length = pt->mask_len;
2137
2138 ret = ccp_copy_to_sb(cmd_q, &mask, op.jobid, op.sb_key,
2139 CCP_PASSTHRU_BYTESWAP_NOOP);
2140 if (ret) {
2141 cmd->engine_error = cmd_q->cmd_error;
2142 return ret;
2143 }
2144 }
2145
2146 /* Send data to the CCP Passthru engine */
2147 op.eom = 1;
2148 op.soc = 1;
2149
2150 op.src.type = CCP_MEMTYPE_SYSTEM;
2151 op.src.u.dma.address = pt->src_dma;
2152 op.src.u.dma.offset = 0;
2153 op.src.u.dma.length = pt->src_len;
2154
2155 op.dst.type = CCP_MEMTYPE_SYSTEM;
2156 op.dst.u.dma.address = pt->dst_dma;
2157 op.dst.u.dma.offset = 0;
2158 op.dst.u.dma.length = pt->src_len;
2159
2160 ret = cmd_q->ccp->vdata->perform->passthru(&op);
2161 if (ret)
2162 cmd->engine_error = cmd_q->cmd_error;
2163
2164 return ret;
2165 }
2166
ccp_run_ecc_mm_cmd(struct ccp_cmd_queue * cmd_q,struct ccp_cmd * cmd)2167 static int ccp_run_ecc_mm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2168 {
2169 struct ccp_ecc_engine *ecc = &cmd->u.ecc;
2170 struct ccp_dm_workarea src, dst;
2171 struct ccp_op op;
2172 int ret;
2173 u8 *save;
2174
2175 if (!ecc->u.mm.operand_1 ||
2176 (ecc->u.mm.operand_1_len > CCP_ECC_MODULUS_BYTES))
2177 return -EINVAL;
2178
2179 if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT)
2180 if (!ecc->u.mm.operand_2 ||
2181 (ecc->u.mm.operand_2_len > CCP_ECC_MODULUS_BYTES))
2182 return -EINVAL;
2183
2184 if (!ecc->u.mm.result ||
2185 (ecc->u.mm.result_len < CCP_ECC_MODULUS_BYTES))
2186 return -EINVAL;
2187
2188 memset(&op, 0, sizeof(op));
2189 op.cmd_q = cmd_q;
2190 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
2191
2192 /* Concatenate the modulus and the operands. Both the modulus and
2193 * the operands must be in little endian format. Since the input
2194 * is in big endian format it must be converted and placed in a
2195 * fixed length buffer.
2196 */
2197 ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE,
2198 DMA_TO_DEVICE);
2199 if (ret)
2200 return ret;
2201
2202 /* Save the workarea address since it is updated in order to perform
2203 * the concatenation
2204 */
2205 save = src.address;
2206
2207 /* Copy the ECC modulus */
2208 ret = ccp_reverse_set_dm_area(&src, 0, ecc->mod, 0, ecc->mod_len);
2209 if (ret)
2210 goto e_src;
2211 src.address += CCP_ECC_OPERAND_SIZE;
2212
2213 /* Copy the first operand */
2214 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.mm.operand_1, 0,
2215 ecc->u.mm.operand_1_len);
2216 if (ret)
2217 goto e_src;
2218 src.address += CCP_ECC_OPERAND_SIZE;
2219
2220 if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT) {
2221 /* Copy the second operand */
2222 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.mm.operand_2, 0,
2223 ecc->u.mm.operand_2_len);
2224 if (ret)
2225 goto e_src;
2226 src.address += CCP_ECC_OPERAND_SIZE;
2227 }
2228
2229 /* Restore the workarea address */
2230 src.address = save;
2231
2232 /* Prepare the output area for the operation */
2233 ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE,
2234 DMA_FROM_DEVICE);
2235 if (ret)
2236 goto e_src;
2237
2238 op.soc = 1;
2239 op.src.u.dma.address = src.dma.address;
2240 op.src.u.dma.offset = 0;
2241 op.src.u.dma.length = src.length;
2242 op.dst.u.dma.address = dst.dma.address;
2243 op.dst.u.dma.offset = 0;
2244 op.dst.u.dma.length = dst.length;
2245
2246 op.u.ecc.function = cmd->u.ecc.function;
2247
2248 ret = cmd_q->ccp->vdata->perform->ecc(&op);
2249 if (ret) {
2250 cmd->engine_error = cmd_q->cmd_error;
2251 goto e_dst;
2252 }
2253
2254 ecc->ecc_result = le16_to_cpup(
2255 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET));
2256 if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) {
2257 ret = -EIO;
2258 goto e_dst;
2259 }
2260
2261 /* Save the ECC result */
2262 ccp_reverse_get_dm_area(&dst, 0, ecc->u.mm.result, 0,
2263 CCP_ECC_MODULUS_BYTES);
2264
2265 e_dst:
2266 ccp_dm_free(&dst);
2267
2268 e_src:
2269 ccp_dm_free(&src);
2270
2271 return ret;
2272 }
2273
ccp_run_ecc_pm_cmd(struct ccp_cmd_queue * cmd_q,struct ccp_cmd * cmd)2274 static int ccp_run_ecc_pm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2275 {
2276 struct ccp_ecc_engine *ecc = &cmd->u.ecc;
2277 struct ccp_dm_workarea src, dst;
2278 struct ccp_op op;
2279 int ret;
2280 u8 *save;
2281
2282 if (!ecc->u.pm.point_1.x ||
2283 (ecc->u.pm.point_1.x_len > CCP_ECC_MODULUS_BYTES) ||
2284 !ecc->u.pm.point_1.y ||
2285 (ecc->u.pm.point_1.y_len > CCP_ECC_MODULUS_BYTES))
2286 return -EINVAL;
2287
2288 if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) {
2289 if (!ecc->u.pm.point_2.x ||
2290 (ecc->u.pm.point_2.x_len > CCP_ECC_MODULUS_BYTES) ||
2291 !ecc->u.pm.point_2.y ||
2292 (ecc->u.pm.point_2.y_len > CCP_ECC_MODULUS_BYTES))
2293 return -EINVAL;
2294 } else {
2295 if (!ecc->u.pm.domain_a ||
2296 (ecc->u.pm.domain_a_len > CCP_ECC_MODULUS_BYTES))
2297 return -EINVAL;
2298
2299 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT)
2300 if (!ecc->u.pm.scalar ||
2301 (ecc->u.pm.scalar_len > CCP_ECC_MODULUS_BYTES))
2302 return -EINVAL;
2303 }
2304
2305 if (!ecc->u.pm.result.x ||
2306 (ecc->u.pm.result.x_len < CCP_ECC_MODULUS_BYTES) ||
2307 !ecc->u.pm.result.y ||
2308 (ecc->u.pm.result.y_len < CCP_ECC_MODULUS_BYTES))
2309 return -EINVAL;
2310
2311 memset(&op, 0, sizeof(op));
2312 op.cmd_q = cmd_q;
2313 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
2314
2315 /* Concatenate the modulus and the operands. Both the modulus and
2316 * the operands must be in little endian format. Since the input
2317 * is in big endian format it must be converted and placed in a
2318 * fixed length buffer.
2319 */
2320 ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE,
2321 DMA_TO_DEVICE);
2322 if (ret)
2323 return ret;
2324
2325 /* Save the workarea address since it is updated in order to perform
2326 * the concatenation
2327 */
2328 save = src.address;
2329
2330 /* Copy the ECC modulus */
2331 ret = ccp_reverse_set_dm_area(&src, 0, ecc->mod, 0, ecc->mod_len);
2332 if (ret)
2333 goto e_src;
2334 src.address += CCP_ECC_OPERAND_SIZE;
2335
2336 /* Copy the first point X and Y coordinate */
2337 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_1.x, 0,
2338 ecc->u.pm.point_1.x_len);
2339 if (ret)
2340 goto e_src;
2341 src.address += CCP_ECC_OPERAND_SIZE;
2342 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_1.y, 0,
2343 ecc->u.pm.point_1.y_len);
2344 if (ret)
2345 goto e_src;
2346 src.address += CCP_ECC_OPERAND_SIZE;
2347
2348 /* Set the first point Z coordinate to 1 */
2349 *src.address = 0x01;
2350 src.address += CCP_ECC_OPERAND_SIZE;
2351
2352 if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) {
2353 /* Copy the second point X and Y coordinate */
2354 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_2.x, 0,
2355 ecc->u.pm.point_2.x_len);
2356 if (ret)
2357 goto e_src;
2358 src.address += CCP_ECC_OPERAND_SIZE;
2359 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_2.y, 0,
2360 ecc->u.pm.point_2.y_len);
2361 if (ret)
2362 goto e_src;
2363 src.address += CCP_ECC_OPERAND_SIZE;
2364
2365 /* Set the second point Z coordinate to 1 */
2366 *src.address = 0x01;
2367 src.address += CCP_ECC_OPERAND_SIZE;
2368 } else {
2369 /* Copy the Domain "a" parameter */
2370 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.domain_a, 0,
2371 ecc->u.pm.domain_a_len);
2372 if (ret)
2373 goto e_src;
2374 src.address += CCP_ECC_OPERAND_SIZE;
2375
2376 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT) {
2377 /* Copy the scalar value */
2378 ret = ccp_reverse_set_dm_area(&src, 0,
2379 ecc->u.pm.scalar, 0,
2380 ecc->u.pm.scalar_len);
2381 if (ret)
2382 goto e_src;
2383 src.address += CCP_ECC_OPERAND_SIZE;
2384 }
2385 }
2386
2387 /* Restore the workarea address */
2388 src.address = save;
2389
2390 /* Prepare the output area for the operation */
2391 ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE,
2392 DMA_FROM_DEVICE);
2393 if (ret)
2394 goto e_src;
2395
2396 op.soc = 1;
2397 op.src.u.dma.address = src.dma.address;
2398 op.src.u.dma.offset = 0;
2399 op.src.u.dma.length = src.length;
2400 op.dst.u.dma.address = dst.dma.address;
2401 op.dst.u.dma.offset = 0;
2402 op.dst.u.dma.length = dst.length;
2403
2404 op.u.ecc.function = cmd->u.ecc.function;
2405
2406 ret = cmd_q->ccp->vdata->perform->ecc(&op);
2407 if (ret) {
2408 cmd->engine_error = cmd_q->cmd_error;
2409 goto e_dst;
2410 }
2411
2412 ecc->ecc_result = le16_to_cpup(
2413 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET));
2414 if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) {
2415 ret = -EIO;
2416 goto e_dst;
2417 }
2418
2419 /* Save the workarea address since it is updated as we walk through
2420 * to copy the point math result
2421 */
2422 save = dst.address;
2423
2424 /* Save the ECC result X and Y coordinates */
2425 ccp_reverse_get_dm_area(&dst, 0, ecc->u.pm.result.x, 0,
2426 CCP_ECC_MODULUS_BYTES);
2427 dst.address += CCP_ECC_OUTPUT_SIZE;
2428 ccp_reverse_get_dm_area(&dst, 0, ecc->u.pm.result.y, 0,
2429 CCP_ECC_MODULUS_BYTES);
2430 dst.address += CCP_ECC_OUTPUT_SIZE;
2431
2432 /* Restore the workarea address */
2433 dst.address = save;
2434
2435 e_dst:
2436 ccp_dm_free(&dst);
2437
2438 e_src:
2439 ccp_dm_free(&src);
2440
2441 return ret;
2442 }
2443
2444 static noinline_for_stack int
ccp_run_ecc_cmd(struct ccp_cmd_queue * cmd_q,struct ccp_cmd * cmd)2445 ccp_run_ecc_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2446 {
2447 struct ccp_ecc_engine *ecc = &cmd->u.ecc;
2448
2449 ecc->ecc_result = 0;
2450
2451 if (!ecc->mod ||
2452 (ecc->mod_len > CCP_ECC_MODULUS_BYTES))
2453 return -EINVAL;
2454
2455 switch (ecc->function) {
2456 case CCP_ECC_FUNCTION_MMUL_384BIT:
2457 case CCP_ECC_FUNCTION_MADD_384BIT:
2458 case CCP_ECC_FUNCTION_MINV_384BIT:
2459 return ccp_run_ecc_mm_cmd(cmd_q, cmd);
2460
2461 case CCP_ECC_FUNCTION_PADD_384BIT:
2462 case CCP_ECC_FUNCTION_PMUL_384BIT:
2463 case CCP_ECC_FUNCTION_PDBL_384BIT:
2464 return ccp_run_ecc_pm_cmd(cmd_q, cmd);
2465
2466 default:
2467 return -EINVAL;
2468 }
2469 }
2470
ccp_run_cmd(struct ccp_cmd_queue * cmd_q,struct ccp_cmd * cmd)2471 int ccp_run_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2472 {
2473 int ret;
2474
2475 cmd->engine_error = 0;
2476 cmd_q->cmd_error = 0;
2477 cmd_q->int_rcvd = 0;
2478 cmd_q->free_slots = cmd_q->ccp->vdata->perform->get_free_slots(cmd_q);
2479
2480 switch (cmd->engine) {
2481 case CCP_ENGINE_AES:
2482 switch (cmd->u.aes.mode) {
2483 case CCP_AES_MODE_CMAC:
2484 ret = ccp_run_aes_cmac_cmd(cmd_q, cmd);
2485 break;
2486 case CCP_AES_MODE_GCM:
2487 ret = ccp_run_aes_gcm_cmd(cmd_q, cmd);
2488 break;
2489 default:
2490 ret = ccp_run_aes_cmd(cmd_q, cmd);
2491 break;
2492 }
2493 break;
2494 case CCP_ENGINE_XTS_AES_128:
2495 ret = ccp_run_xts_aes_cmd(cmd_q, cmd);
2496 break;
2497 case CCP_ENGINE_DES3:
2498 ret = ccp_run_des3_cmd(cmd_q, cmd);
2499 break;
2500 case CCP_ENGINE_SHA:
2501 ret = ccp_run_sha_cmd(cmd_q, cmd);
2502 break;
2503 case CCP_ENGINE_RSA:
2504 ret = ccp_run_rsa_cmd(cmd_q, cmd);
2505 break;
2506 case CCP_ENGINE_PASSTHRU:
2507 if (cmd->flags & CCP_CMD_PASSTHRU_NO_DMA_MAP)
2508 ret = ccp_run_passthru_nomap_cmd(cmd_q, cmd);
2509 else
2510 ret = ccp_run_passthru_cmd(cmd_q, cmd);
2511 break;
2512 case CCP_ENGINE_ECC:
2513 ret = ccp_run_ecc_cmd(cmd_q, cmd);
2514 break;
2515 default:
2516 ret = -EINVAL;
2517 }
2518
2519 return ret;
2520 }
2521