1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) 2017 Marvell
4 *
5 * Antoine Tenart <antoine.tenart@free-electrons.com>
6 */
7
8 #include <crypto/aes.h>
9 #include <crypto/hmac.h>
10 #include <crypto/md5.h>
11 #include <crypto/sha.h>
12 #include <crypto/sha3.h>
13 #include <crypto/skcipher.h>
14 #include <crypto/sm3.h>
15 #include <linux/device.h>
16 #include <linux/dma-mapping.h>
17 #include <linux/dmapool.h>
18
19 #include "safexcel.h"
20
21 struct safexcel_ahash_ctx {
22 struct safexcel_context base;
23
24 u32 alg;
25 u8 key_sz;
26 bool cbcmac;
27 bool do_fallback;
28 bool fb_init_done;
29 bool fb_do_setkey;
30
31 struct crypto_cipher *kaes;
32 struct crypto_ahash *fback;
33 struct crypto_shash *shpre;
34 struct shash_desc *shdesc;
35 };
36
37 struct safexcel_ahash_req {
38 bool last_req;
39 bool finish;
40 bool hmac;
41 bool needs_inv;
42 bool hmac_zlen;
43 bool len_is_le;
44 bool not_first;
45 bool xcbcmac;
46
47 int nents;
48 dma_addr_t result_dma;
49
50 u32 digest;
51
52 u8 state_sz; /* expected state size, only set once */
53 u8 block_sz; /* block size, only set once */
54 u8 digest_sz; /* output digest size, only set once */
55 __le32 state[SHA3_512_BLOCK_SIZE /
56 sizeof(__le32)] __aligned(sizeof(__le32));
57
58 u64 len;
59 u64 processed;
60
61 u8 cache[HASH_CACHE_SIZE] __aligned(sizeof(u32));
62 dma_addr_t cache_dma;
63 unsigned int cache_sz;
64
65 u8 cache_next[HASH_CACHE_SIZE] __aligned(sizeof(u32));
66 };
67
safexcel_queued_len(struct safexcel_ahash_req * req)68 static inline u64 safexcel_queued_len(struct safexcel_ahash_req *req)
69 {
70 return req->len - req->processed;
71 }
72
safexcel_hash_token(struct safexcel_command_desc * cdesc,u32 input_length,u32 result_length,bool cbcmac)73 static void safexcel_hash_token(struct safexcel_command_desc *cdesc,
74 u32 input_length, u32 result_length,
75 bool cbcmac)
76 {
77 struct safexcel_token *token =
78 (struct safexcel_token *)cdesc->control_data.token;
79
80 token[0].opcode = EIP197_TOKEN_OPCODE_DIRECTION;
81 token[0].packet_length = input_length;
82 token[0].instructions = EIP197_TOKEN_INS_TYPE_HASH;
83
84 input_length &= 15;
85 if (unlikely(cbcmac && input_length)) {
86 token[0].stat = 0;
87 token[1].opcode = EIP197_TOKEN_OPCODE_INSERT;
88 token[1].packet_length = 16 - input_length;
89 token[1].stat = EIP197_TOKEN_STAT_LAST_HASH;
90 token[1].instructions = EIP197_TOKEN_INS_TYPE_HASH;
91 } else {
92 token[0].stat = EIP197_TOKEN_STAT_LAST_HASH;
93 eip197_noop_token(&token[1]);
94 }
95
96 token[2].opcode = EIP197_TOKEN_OPCODE_INSERT;
97 token[2].stat = EIP197_TOKEN_STAT_LAST_HASH |
98 EIP197_TOKEN_STAT_LAST_PACKET;
99 token[2].packet_length = result_length;
100 token[2].instructions = EIP197_TOKEN_INS_TYPE_OUTPUT |
101 EIP197_TOKEN_INS_INSERT_HASH_DIGEST;
102
103 eip197_noop_token(&token[3]);
104 }
105
safexcel_context_control(struct safexcel_ahash_ctx * ctx,struct safexcel_ahash_req * req,struct safexcel_command_desc * cdesc)106 static void safexcel_context_control(struct safexcel_ahash_ctx *ctx,
107 struct safexcel_ahash_req *req,
108 struct safexcel_command_desc *cdesc)
109 {
110 struct safexcel_crypto_priv *priv = ctx->base.priv;
111 u64 count = 0;
112
113 cdesc->control_data.control0 = ctx->alg;
114 cdesc->control_data.control1 = 0;
115
116 /*
117 * Copy the input digest if needed, and setup the context
118 * fields. Do this now as we need it to setup the first command
119 * descriptor.
120 */
121 if (unlikely(req->digest == CONTEXT_CONTROL_DIGEST_XCM)) {
122 if (req->xcbcmac)
123 memcpy(ctx->base.ctxr->data, &ctx->base.ipad, ctx->key_sz);
124 else
125 memcpy(ctx->base.ctxr->data, req->state, req->state_sz);
126
127 if (!req->finish && req->xcbcmac)
128 cdesc->control_data.control0 |=
129 CONTEXT_CONTROL_DIGEST_XCM |
130 CONTEXT_CONTROL_TYPE_HASH_OUT |
131 CONTEXT_CONTROL_NO_FINISH_HASH |
132 CONTEXT_CONTROL_SIZE(req->state_sz /
133 sizeof(u32));
134 else
135 cdesc->control_data.control0 |=
136 CONTEXT_CONTROL_DIGEST_XCM |
137 CONTEXT_CONTROL_TYPE_HASH_OUT |
138 CONTEXT_CONTROL_SIZE(req->state_sz /
139 sizeof(u32));
140 return;
141 } else if (!req->processed) {
142 /* First - and possibly only - block of basic hash only */
143 if (req->finish)
144 cdesc->control_data.control0 |= req->digest |
145 CONTEXT_CONTROL_TYPE_HASH_OUT |
146 CONTEXT_CONTROL_RESTART_HASH |
147 /* ensure its not 0! */
148 CONTEXT_CONTROL_SIZE(1);
149 else
150 cdesc->control_data.control0 |= req->digest |
151 CONTEXT_CONTROL_TYPE_HASH_OUT |
152 CONTEXT_CONTROL_RESTART_HASH |
153 CONTEXT_CONTROL_NO_FINISH_HASH |
154 /* ensure its not 0! */
155 CONTEXT_CONTROL_SIZE(1);
156 return;
157 }
158
159 /* Hash continuation or HMAC, setup (inner) digest from state */
160 memcpy(ctx->base.ctxr->data, req->state, req->state_sz);
161
162 if (req->finish) {
163 /* Compute digest count for hash/HMAC finish operations */
164 if ((req->digest == CONTEXT_CONTROL_DIGEST_PRECOMPUTED) ||
165 req->hmac_zlen || (req->processed != req->block_sz)) {
166 count = req->processed / EIP197_COUNTER_BLOCK_SIZE;
167
168 /* This is a hardware limitation, as the
169 * counter must fit into an u32. This represents
170 * a fairly big amount of input data, so we
171 * shouldn't see this.
172 */
173 if (unlikely(count & 0xffffffff00000000ULL)) {
174 dev_warn(priv->dev,
175 "Input data is too big\n");
176 return;
177 }
178 }
179
180 if ((req->digest == CONTEXT_CONTROL_DIGEST_PRECOMPUTED) ||
181 /* Special case: zero length HMAC */
182 req->hmac_zlen ||
183 /* PE HW < 4.4 cannot do HMAC continue, fake using hash */
184 (req->processed != req->block_sz)) {
185 /* Basic hash continue operation, need digest + cnt */
186 cdesc->control_data.control0 |=
187 CONTEXT_CONTROL_SIZE((req->state_sz >> 2) + 1) |
188 CONTEXT_CONTROL_TYPE_HASH_OUT |
189 CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
190 /* For zero-len HMAC, don't finalize, already padded! */
191 if (req->hmac_zlen)
192 cdesc->control_data.control0 |=
193 CONTEXT_CONTROL_NO_FINISH_HASH;
194 cdesc->control_data.control1 |=
195 CONTEXT_CONTROL_DIGEST_CNT;
196 ctx->base.ctxr->data[req->state_sz >> 2] =
197 cpu_to_le32(count);
198 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
199
200 /* Clear zero-length HMAC flag for next operation! */
201 req->hmac_zlen = false;
202 } else { /* HMAC */
203 /* Need outer digest for HMAC finalization */
204 memcpy(ctx->base.ctxr->data + (req->state_sz >> 2),
205 &ctx->base.opad, req->state_sz);
206
207 /* Single pass HMAC - no digest count */
208 cdesc->control_data.control0 |=
209 CONTEXT_CONTROL_SIZE(req->state_sz >> 1) |
210 CONTEXT_CONTROL_TYPE_HASH_OUT |
211 CONTEXT_CONTROL_DIGEST_HMAC;
212 }
213 } else { /* Hash continuation, do not finish yet */
214 cdesc->control_data.control0 |=
215 CONTEXT_CONTROL_SIZE(req->state_sz >> 2) |
216 CONTEXT_CONTROL_DIGEST_PRECOMPUTED |
217 CONTEXT_CONTROL_TYPE_HASH_OUT |
218 CONTEXT_CONTROL_NO_FINISH_HASH;
219 }
220 }
221
222 static int safexcel_ahash_enqueue(struct ahash_request *areq);
223
safexcel_handle_req_result(struct safexcel_crypto_priv * priv,int ring,struct crypto_async_request * async,bool * should_complete,int * ret)224 static int safexcel_handle_req_result(struct safexcel_crypto_priv *priv,
225 int ring,
226 struct crypto_async_request *async,
227 bool *should_complete, int *ret)
228 {
229 struct safexcel_result_desc *rdesc;
230 struct ahash_request *areq = ahash_request_cast(async);
231 struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
232 struct safexcel_ahash_req *sreq = ahash_request_ctx(areq);
233 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(ahash);
234 u64 cache_len;
235
236 *ret = 0;
237
238 rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr);
239 if (IS_ERR(rdesc)) {
240 dev_err(priv->dev,
241 "hash: result: could not retrieve the result descriptor\n");
242 *ret = PTR_ERR(rdesc);
243 } else {
244 *ret = safexcel_rdesc_check_errors(priv, rdesc);
245 }
246
247 safexcel_complete(priv, ring);
248
249 if (sreq->nents) {
250 dma_unmap_sg(priv->dev, areq->src, sreq->nents, DMA_TO_DEVICE);
251 sreq->nents = 0;
252 }
253
254 if (sreq->result_dma) {
255 dma_unmap_single(priv->dev, sreq->result_dma, sreq->digest_sz,
256 DMA_FROM_DEVICE);
257 sreq->result_dma = 0;
258 }
259
260 if (sreq->cache_dma) {
261 dma_unmap_single(priv->dev, sreq->cache_dma, sreq->cache_sz,
262 DMA_TO_DEVICE);
263 sreq->cache_dma = 0;
264 sreq->cache_sz = 0;
265 }
266
267 if (sreq->finish) {
268 if (sreq->hmac &&
269 (sreq->digest != CONTEXT_CONTROL_DIGEST_HMAC)) {
270 /* Faking HMAC using hash - need to do outer hash */
271 memcpy(sreq->cache, sreq->state,
272 crypto_ahash_digestsize(ahash));
273
274 memcpy(sreq->state, &ctx->base.opad, sreq->digest_sz);
275
276 sreq->len = sreq->block_sz +
277 crypto_ahash_digestsize(ahash);
278 sreq->processed = sreq->block_sz;
279 sreq->hmac = 0;
280
281 if (priv->flags & EIP197_TRC_CACHE)
282 ctx->base.needs_inv = true;
283 areq->nbytes = 0;
284 safexcel_ahash_enqueue(areq);
285
286 *should_complete = false; /* Not done yet */
287 return 1;
288 }
289
290 if (unlikely(sreq->digest == CONTEXT_CONTROL_DIGEST_XCM &&
291 ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_CRC32)) {
292 /* Undo final XOR with 0xffffffff ...*/
293 *(__le32 *)areq->result = ~sreq->state[0];
294 } else {
295 memcpy(areq->result, sreq->state,
296 crypto_ahash_digestsize(ahash));
297 }
298 }
299
300 cache_len = safexcel_queued_len(sreq);
301 if (cache_len)
302 memcpy(sreq->cache, sreq->cache_next, cache_len);
303
304 *should_complete = true;
305
306 return 1;
307 }
308
safexcel_ahash_send_req(struct crypto_async_request * async,int ring,int * commands,int * results)309 static int safexcel_ahash_send_req(struct crypto_async_request *async, int ring,
310 int *commands, int *results)
311 {
312 struct ahash_request *areq = ahash_request_cast(async);
313 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
314 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
315 struct safexcel_crypto_priv *priv = ctx->base.priv;
316 struct safexcel_command_desc *cdesc, *first_cdesc = NULL;
317 struct safexcel_result_desc *rdesc;
318 struct scatterlist *sg;
319 struct safexcel_token *dmmy;
320 int i, extra = 0, n_cdesc = 0, ret = 0, cache_len, skip = 0;
321 u64 queued, len;
322
323 queued = safexcel_queued_len(req);
324 if (queued <= HASH_CACHE_SIZE)
325 cache_len = queued;
326 else
327 cache_len = queued - areq->nbytes;
328
329 if (!req->finish && !req->last_req) {
330 /* If this is not the last request and the queued data does not
331 * fit into full cache blocks, cache it for the next send call.
332 */
333 extra = queued & (HASH_CACHE_SIZE - 1);
334
335 /* If this is not the last request and the queued data
336 * is a multiple of a block, cache the last one for now.
337 */
338 if (!extra)
339 extra = HASH_CACHE_SIZE;
340
341 sg_pcopy_to_buffer(areq->src, sg_nents(areq->src),
342 req->cache_next, extra,
343 areq->nbytes - extra);
344
345 queued -= extra;
346
347 if (!queued) {
348 *commands = 0;
349 *results = 0;
350 return 0;
351 }
352
353 extra = 0;
354 }
355
356 if (unlikely(req->xcbcmac && req->processed > AES_BLOCK_SIZE)) {
357 if (unlikely(cache_len < AES_BLOCK_SIZE)) {
358 /*
359 * Cache contains less than 1 full block, complete.
360 */
361 extra = AES_BLOCK_SIZE - cache_len;
362 if (queued > cache_len) {
363 /* More data follows: borrow bytes */
364 u64 tmp = queued - cache_len;
365
366 skip = min_t(u64, tmp, extra);
367 sg_pcopy_to_buffer(areq->src,
368 sg_nents(areq->src),
369 req->cache + cache_len,
370 skip, 0);
371 }
372 extra -= skip;
373 memset(req->cache + cache_len + skip, 0, extra);
374 if (!ctx->cbcmac && extra) {
375 // 10- padding for XCBCMAC & CMAC
376 req->cache[cache_len + skip] = 0x80;
377 // HW will use K2 iso K3 - compensate!
378 for (i = 0; i < AES_BLOCK_SIZE / 4; i++) {
379 u32 *cache = (void *)req->cache;
380 u32 *ipad = ctx->base.ipad.word;
381 u32 x;
382
383 x = ipad[i] ^ ipad[i + 4];
384 cache[i] ^= swab32(x);
385 }
386 }
387 cache_len = AES_BLOCK_SIZE;
388 queued = queued + extra;
389 }
390
391 /* XCBC continue: XOR previous result into 1st word */
392 crypto_xor(req->cache, (const u8 *)req->state, AES_BLOCK_SIZE);
393 }
394
395 len = queued;
396 /* Add a command descriptor for the cached data, if any */
397 if (cache_len) {
398 req->cache_dma = dma_map_single(priv->dev, req->cache,
399 cache_len, DMA_TO_DEVICE);
400 if (dma_mapping_error(priv->dev, req->cache_dma))
401 return -EINVAL;
402
403 req->cache_sz = cache_len;
404 first_cdesc = safexcel_add_cdesc(priv, ring, 1,
405 (cache_len == len),
406 req->cache_dma, cache_len,
407 len, ctx->base.ctxr_dma,
408 &dmmy);
409 if (IS_ERR(first_cdesc)) {
410 ret = PTR_ERR(first_cdesc);
411 goto unmap_cache;
412 }
413 n_cdesc++;
414
415 queued -= cache_len;
416 if (!queued)
417 goto send_command;
418 }
419
420 /* Now handle the current ahash request buffer(s) */
421 req->nents = dma_map_sg(priv->dev, areq->src,
422 sg_nents_for_len(areq->src,
423 areq->nbytes),
424 DMA_TO_DEVICE);
425 if (!req->nents) {
426 ret = -ENOMEM;
427 goto cdesc_rollback;
428 }
429
430 for_each_sg(areq->src, sg, req->nents, i) {
431 int sglen = sg_dma_len(sg);
432
433 if (unlikely(sglen <= skip)) {
434 skip -= sglen;
435 continue;
436 }
437
438 /* Do not overflow the request */
439 if ((queued + skip) <= sglen)
440 sglen = queued;
441 else
442 sglen -= skip;
443
444 cdesc = safexcel_add_cdesc(priv, ring, !n_cdesc,
445 !(queued - sglen),
446 sg_dma_address(sg) + skip, sglen,
447 len, ctx->base.ctxr_dma, &dmmy);
448 if (IS_ERR(cdesc)) {
449 ret = PTR_ERR(cdesc);
450 goto unmap_sg;
451 }
452
453 if (!n_cdesc)
454 first_cdesc = cdesc;
455 n_cdesc++;
456
457 queued -= sglen;
458 if (!queued)
459 break;
460 skip = 0;
461 }
462
463 send_command:
464 /* Setup the context options */
465 safexcel_context_control(ctx, req, first_cdesc);
466
467 /* Add the token */
468 safexcel_hash_token(first_cdesc, len, req->digest_sz, ctx->cbcmac);
469
470 req->result_dma = dma_map_single(priv->dev, req->state, req->digest_sz,
471 DMA_FROM_DEVICE);
472 if (dma_mapping_error(priv->dev, req->result_dma)) {
473 ret = -EINVAL;
474 goto unmap_sg;
475 }
476
477 /* Add a result descriptor */
478 rdesc = safexcel_add_rdesc(priv, ring, 1, 1, req->result_dma,
479 req->digest_sz);
480 if (IS_ERR(rdesc)) {
481 ret = PTR_ERR(rdesc);
482 goto unmap_result;
483 }
484
485 safexcel_rdr_req_set(priv, ring, rdesc, &areq->base);
486
487 req->processed += len - extra;
488
489 *commands = n_cdesc;
490 *results = 1;
491 return 0;
492
493 unmap_result:
494 dma_unmap_single(priv->dev, req->result_dma, req->digest_sz,
495 DMA_FROM_DEVICE);
496 unmap_sg:
497 if (req->nents) {
498 dma_unmap_sg(priv->dev, areq->src, req->nents, DMA_TO_DEVICE);
499 req->nents = 0;
500 }
501 cdesc_rollback:
502 for (i = 0; i < n_cdesc; i++)
503 safexcel_ring_rollback_wptr(priv, &priv->ring[ring].cdr);
504 unmap_cache:
505 if (req->cache_dma) {
506 dma_unmap_single(priv->dev, req->cache_dma, req->cache_sz,
507 DMA_TO_DEVICE);
508 req->cache_dma = 0;
509 req->cache_sz = 0;
510 }
511
512 return ret;
513 }
514
safexcel_handle_inv_result(struct safexcel_crypto_priv * priv,int ring,struct crypto_async_request * async,bool * should_complete,int * ret)515 static int safexcel_handle_inv_result(struct safexcel_crypto_priv *priv,
516 int ring,
517 struct crypto_async_request *async,
518 bool *should_complete, int *ret)
519 {
520 struct safexcel_result_desc *rdesc;
521 struct ahash_request *areq = ahash_request_cast(async);
522 struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
523 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(ahash);
524 int enq_ret;
525
526 *ret = 0;
527
528 rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr);
529 if (IS_ERR(rdesc)) {
530 dev_err(priv->dev,
531 "hash: invalidate: could not retrieve the result descriptor\n");
532 *ret = PTR_ERR(rdesc);
533 } else {
534 *ret = safexcel_rdesc_check_errors(priv, rdesc);
535 }
536
537 safexcel_complete(priv, ring);
538
539 if (ctx->base.exit_inv) {
540 dma_pool_free(priv->context_pool, ctx->base.ctxr,
541 ctx->base.ctxr_dma);
542
543 *should_complete = true;
544 return 1;
545 }
546
547 ring = safexcel_select_ring(priv);
548 ctx->base.ring = ring;
549
550 spin_lock_bh(&priv->ring[ring].queue_lock);
551 enq_ret = crypto_enqueue_request(&priv->ring[ring].queue, async);
552 spin_unlock_bh(&priv->ring[ring].queue_lock);
553
554 if (enq_ret != -EINPROGRESS)
555 *ret = enq_ret;
556
557 queue_work(priv->ring[ring].workqueue,
558 &priv->ring[ring].work_data.work);
559
560 *should_complete = false;
561
562 return 1;
563 }
564
safexcel_handle_result(struct safexcel_crypto_priv * priv,int ring,struct crypto_async_request * async,bool * should_complete,int * ret)565 static int safexcel_handle_result(struct safexcel_crypto_priv *priv, int ring,
566 struct crypto_async_request *async,
567 bool *should_complete, int *ret)
568 {
569 struct ahash_request *areq = ahash_request_cast(async);
570 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
571 int err;
572
573 BUG_ON(!(priv->flags & EIP197_TRC_CACHE) && req->needs_inv);
574
575 if (req->needs_inv) {
576 req->needs_inv = false;
577 err = safexcel_handle_inv_result(priv, ring, async,
578 should_complete, ret);
579 } else {
580 err = safexcel_handle_req_result(priv, ring, async,
581 should_complete, ret);
582 }
583
584 return err;
585 }
586
safexcel_ahash_send_inv(struct crypto_async_request * async,int ring,int * commands,int * results)587 static int safexcel_ahash_send_inv(struct crypto_async_request *async,
588 int ring, int *commands, int *results)
589 {
590 struct ahash_request *areq = ahash_request_cast(async);
591 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
592 int ret;
593
594 ret = safexcel_invalidate_cache(async, ctx->base.priv,
595 ctx->base.ctxr_dma, ring);
596 if (unlikely(ret))
597 return ret;
598
599 *commands = 1;
600 *results = 1;
601
602 return 0;
603 }
604
safexcel_ahash_send(struct crypto_async_request * async,int ring,int * commands,int * results)605 static int safexcel_ahash_send(struct crypto_async_request *async,
606 int ring, int *commands, int *results)
607 {
608 struct ahash_request *areq = ahash_request_cast(async);
609 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
610 int ret;
611
612 if (req->needs_inv)
613 ret = safexcel_ahash_send_inv(async, ring, commands, results);
614 else
615 ret = safexcel_ahash_send_req(async, ring, commands, results);
616
617 return ret;
618 }
619
safexcel_ahash_exit_inv(struct crypto_tfm * tfm)620 static int safexcel_ahash_exit_inv(struct crypto_tfm *tfm)
621 {
622 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
623 struct safexcel_crypto_priv *priv = ctx->base.priv;
624 EIP197_REQUEST_ON_STACK(req, ahash, EIP197_AHASH_REQ_SIZE);
625 struct safexcel_ahash_req *rctx = ahash_request_ctx(req);
626 struct safexcel_inv_result result = {};
627 int ring = ctx->base.ring;
628
629 memset(req, 0, EIP197_AHASH_REQ_SIZE);
630
631 /* create invalidation request */
632 init_completion(&result.completion);
633 ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
634 safexcel_inv_complete, &result);
635
636 ahash_request_set_tfm(req, __crypto_ahash_cast(tfm));
637 ctx = crypto_tfm_ctx(req->base.tfm);
638 ctx->base.exit_inv = true;
639 rctx->needs_inv = true;
640
641 spin_lock_bh(&priv->ring[ring].queue_lock);
642 crypto_enqueue_request(&priv->ring[ring].queue, &req->base);
643 spin_unlock_bh(&priv->ring[ring].queue_lock);
644
645 queue_work(priv->ring[ring].workqueue,
646 &priv->ring[ring].work_data.work);
647
648 wait_for_completion(&result.completion);
649
650 if (result.error) {
651 dev_warn(priv->dev, "hash: completion error (%d)\n",
652 result.error);
653 return result.error;
654 }
655
656 return 0;
657 }
658
659 /* safexcel_ahash_cache: cache data until at least one request can be sent to
660 * the engine, aka. when there is at least 1 block size in the pipe.
661 */
safexcel_ahash_cache(struct ahash_request * areq)662 static int safexcel_ahash_cache(struct ahash_request *areq)
663 {
664 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
665 u64 cache_len;
666
667 /* cache_len: everything accepted by the driver but not sent yet,
668 * tot sz handled by update() - last req sz - tot sz handled by send()
669 */
670 cache_len = safexcel_queued_len(req);
671
672 /*
673 * In case there isn't enough bytes to proceed (less than a
674 * block size), cache the data until we have enough.
675 */
676 if (cache_len + areq->nbytes <= HASH_CACHE_SIZE) {
677 sg_pcopy_to_buffer(areq->src, sg_nents(areq->src),
678 req->cache + cache_len,
679 areq->nbytes, 0);
680 return 0;
681 }
682
683 /* We couldn't cache all the data */
684 return -E2BIG;
685 }
686
safexcel_ahash_enqueue(struct ahash_request * areq)687 static int safexcel_ahash_enqueue(struct ahash_request *areq)
688 {
689 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
690 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
691 struct safexcel_crypto_priv *priv = ctx->base.priv;
692 int ret, ring;
693
694 req->needs_inv = false;
695
696 if (ctx->base.ctxr) {
697 if (priv->flags & EIP197_TRC_CACHE && !ctx->base.needs_inv &&
698 /* invalidate for *any* non-XCBC continuation */
699 ((req->not_first && !req->xcbcmac) ||
700 /* invalidate if (i)digest changed */
701 memcmp(ctx->base.ctxr->data, req->state, req->state_sz) ||
702 /* invalidate for HMAC finish with odigest changed */
703 (req->finish && req->hmac &&
704 memcmp(ctx->base.ctxr->data + (req->state_sz>>2),
705 &ctx->base.opad, req->state_sz))))
706 /*
707 * We're still setting needs_inv here, even though it is
708 * cleared right away, because the needs_inv flag can be
709 * set in other functions and we want to keep the same
710 * logic.
711 */
712 ctx->base.needs_inv = true;
713
714 if (ctx->base.needs_inv) {
715 ctx->base.needs_inv = false;
716 req->needs_inv = true;
717 }
718 } else {
719 ctx->base.ring = safexcel_select_ring(priv);
720 ctx->base.ctxr = dma_pool_zalloc(priv->context_pool,
721 EIP197_GFP_FLAGS(areq->base),
722 &ctx->base.ctxr_dma);
723 if (!ctx->base.ctxr)
724 return -ENOMEM;
725 }
726 req->not_first = true;
727
728 ring = ctx->base.ring;
729
730 spin_lock_bh(&priv->ring[ring].queue_lock);
731 ret = crypto_enqueue_request(&priv->ring[ring].queue, &areq->base);
732 spin_unlock_bh(&priv->ring[ring].queue_lock);
733
734 queue_work(priv->ring[ring].workqueue,
735 &priv->ring[ring].work_data.work);
736
737 return ret;
738 }
739
safexcel_ahash_update(struct ahash_request * areq)740 static int safexcel_ahash_update(struct ahash_request *areq)
741 {
742 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
743 int ret;
744
745 /* If the request is 0 length, do nothing */
746 if (!areq->nbytes)
747 return 0;
748
749 /* Add request to the cache if it fits */
750 ret = safexcel_ahash_cache(areq);
751
752 /* Update total request length */
753 req->len += areq->nbytes;
754
755 /* If not all data could fit into the cache, go process the excess.
756 * Also go process immediately for an HMAC IV precompute, which
757 * will never be finished at all, but needs to be processed anyway.
758 */
759 if ((ret && !req->finish) || req->last_req)
760 return safexcel_ahash_enqueue(areq);
761
762 return 0;
763 }
764
safexcel_ahash_final(struct ahash_request * areq)765 static int safexcel_ahash_final(struct ahash_request *areq)
766 {
767 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
768 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
769
770 req->finish = true;
771
772 if (unlikely(!req->len && !areq->nbytes)) {
773 /*
774 * If we have an overall 0 length *hash* request:
775 * The HW cannot do 0 length hash, so we provide the correct
776 * result directly here.
777 */
778 if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_MD5)
779 memcpy(areq->result, md5_zero_message_hash,
780 MD5_DIGEST_SIZE);
781 else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA1)
782 memcpy(areq->result, sha1_zero_message_hash,
783 SHA1_DIGEST_SIZE);
784 else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA224)
785 memcpy(areq->result, sha224_zero_message_hash,
786 SHA224_DIGEST_SIZE);
787 else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA256)
788 memcpy(areq->result, sha256_zero_message_hash,
789 SHA256_DIGEST_SIZE);
790 else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA384)
791 memcpy(areq->result, sha384_zero_message_hash,
792 SHA384_DIGEST_SIZE);
793 else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA512)
794 memcpy(areq->result, sha512_zero_message_hash,
795 SHA512_DIGEST_SIZE);
796 else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SM3) {
797 memcpy(areq->result,
798 EIP197_SM3_ZEROM_HASH, SM3_DIGEST_SIZE);
799 }
800
801 return 0;
802 } else if (unlikely(req->digest == CONTEXT_CONTROL_DIGEST_XCM &&
803 ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_MD5 &&
804 req->len == sizeof(u32) && !areq->nbytes)) {
805 /* Zero length CRC32 */
806 memcpy(areq->result, &ctx->base.ipad, sizeof(u32));
807 return 0;
808 } else if (unlikely(ctx->cbcmac && req->len == AES_BLOCK_SIZE &&
809 !areq->nbytes)) {
810 /* Zero length CBC MAC */
811 memset(areq->result, 0, AES_BLOCK_SIZE);
812 return 0;
813 } else if (unlikely(req->xcbcmac && req->len == AES_BLOCK_SIZE &&
814 !areq->nbytes)) {
815 /* Zero length (X)CBC/CMAC */
816 int i;
817
818 for (i = 0; i < AES_BLOCK_SIZE / sizeof(u32); i++) {
819 u32 *result = (void *)areq->result;
820
821 /* K3 */
822 result[i] = swab32(ctx->base.ipad.word[i + 4]);
823 }
824 areq->result[0] ^= 0x80; // 10- padding
825 crypto_cipher_encrypt_one(ctx->kaes, areq->result, areq->result);
826 return 0;
827 } else if (unlikely(req->hmac &&
828 (req->len == req->block_sz) &&
829 !areq->nbytes)) {
830 /*
831 * If we have an overall 0 length *HMAC* request:
832 * For HMAC, we need to finalize the inner digest
833 * and then perform the outer hash.
834 */
835
836 /* generate pad block in the cache */
837 /* start with a hash block of all zeroes */
838 memset(req->cache, 0, req->block_sz);
839 /* set the first byte to 0x80 to 'append a 1 bit' */
840 req->cache[0] = 0x80;
841 /* add the length in bits in the last 2 bytes */
842 if (req->len_is_le) {
843 /* Little endian length word (e.g. MD5) */
844 req->cache[req->block_sz-8] = (req->block_sz << 3) &
845 255;
846 req->cache[req->block_sz-7] = (req->block_sz >> 5);
847 } else {
848 /* Big endian length word (e.g. any SHA) */
849 req->cache[req->block_sz-2] = (req->block_sz >> 5);
850 req->cache[req->block_sz-1] = (req->block_sz << 3) &
851 255;
852 }
853
854 req->len += req->block_sz; /* plus 1 hash block */
855
856 /* Set special zero-length HMAC flag */
857 req->hmac_zlen = true;
858
859 /* Finalize HMAC */
860 req->digest = CONTEXT_CONTROL_DIGEST_HMAC;
861 } else if (req->hmac) {
862 /* Finalize HMAC */
863 req->digest = CONTEXT_CONTROL_DIGEST_HMAC;
864 }
865
866 return safexcel_ahash_enqueue(areq);
867 }
868
safexcel_ahash_finup(struct ahash_request * areq)869 static int safexcel_ahash_finup(struct ahash_request *areq)
870 {
871 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
872
873 req->finish = true;
874
875 safexcel_ahash_update(areq);
876 return safexcel_ahash_final(areq);
877 }
878
safexcel_ahash_export(struct ahash_request * areq,void * out)879 static int safexcel_ahash_export(struct ahash_request *areq, void *out)
880 {
881 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
882 struct safexcel_ahash_export_state *export = out;
883
884 export->len = req->len;
885 export->processed = req->processed;
886
887 export->digest = req->digest;
888
889 memcpy(export->state, req->state, req->state_sz);
890 memcpy(export->cache, req->cache, HASH_CACHE_SIZE);
891
892 return 0;
893 }
894
safexcel_ahash_import(struct ahash_request * areq,const void * in)895 static int safexcel_ahash_import(struct ahash_request *areq, const void *in)
896 {
897 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
898 const struct safexcel_ahash_export_state *export = in;
899 int ret;
900
901 ret = crypto_ahash_init(areq);
902 if (ret)
903 return ret;
904
905 req->len = export->len;
906 req->processed = export->processed;
907
908 req->digest = export->digest;
909
910 memcpy(req->cache, export->cache, HASH_CACHE_SIZE);
911 memcpy(req->state, export->state, req->state_sz);
912
913 return 0;
914 }
915
safexcel_ahash_cra_init(struct crypto_tfm * tfm)916 static int safexcel_ahash_cra_init(struct crypto_tfm *tfm)
917 {
918 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
919 struct safexcel_alg_template *tmpl =
920 container_of(__crypto_ahash_alg(tfm->__crt_alg),
921 struct safexcel_alg_template, alg.ahash);
922
923 ctx->base.priv = tmpl->priv;
924 ctx->base.send = safexcel_ahash_send;
925 ctx->base.handle_result = safexcel_handle_result;
926 ctx->fb_do_setkey = false;
927
928 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
929 sizeof(struct safexcel_ahash_req));
930 return 0;
931 }
932
safexcel_sha1_init(struct ahash_request * areq)933 static int safexcel_sha1_init(struct ahash_request *areq)
934 {
935 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
936 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
937
938 memset(req, 0, sizeof(*req));
939
940 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA1;
941 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
942 req->state_sz = SHA1_DIGEST_SIZE;
943 req->digest_sz = SHA1_DIGEST_SIZE;
944 req->block_sz = SHA1_BLOCK_SIZE;
945
946 return 0;
947 }
948
safexcel_sha1_digest(struct ahash_request * areq)949 static int safexcel_sha1_digest(struct ahash_request *areq)
950 {
951 int ret = safexcel_sha1_init(areq);
952
953 if (ret)
954 return ret;
955
956 return safexcel_ahash_finup(areq);
957 }
958
safexcel_ahash_cra_exit(struct crypto_tfm * tfm)959 static void safexcel_ahash_cra_exit(struct crypto_tfm *tfm)
960 {
961 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
962 struct safexcel_crypto_priv *priv = ctx->base.priv;
963 int ret;
964
965 /* context not allocated, skip invalidation */
966 if (!ctx->base.ctxr)
967 return;
968
969 if (priv->flags & EIP197_TRC_CACHE) {
970 ret = safexcel_ahash_exit_inv(tfm);
971 if (ret)
972 dev_warn(priv->dev, "hash: invalidation error %d\n", ret);
973 } else {
974 dma_pool_free(priv->context_pool, ctx->base.ctxr,
975 ctx->base.ctxr_dma);
976 }
977 }
978
979 struct safexcel_alg_template safexcel_alg_sha1 = {
980 .type = SAFEXCEL_ALG_TYPE_AHASH,
981 .algo_mask = SAFEXCEL_ALG_SHA1,
982 .alg.ahash = {
983 .init = safexcel_sha1_init,
984 .update = safexcel_ahash_update,
985 .final = safexcel_ahash_final,
986 .finup = safexcel_ahash_finup,
987 .digest = safexcel_sha1_digest,
988 .export = safexcel_ahash_export,
989 .import = safexcel_ahash_import,
990 .halg = {
991 .digestsize = SHA1_DIGEST_SIZE,
992 .statesize = sizeof(struct safexcel_ahash_export_state),
993 .base = {
994 .cra_name = "sha1",
995 .cra_driver_name = "safexcel-sha1",
996 .cra_priority = SAFEXCEL_CRA_PRIORITY,
997 .cra_flags = CRYPTO_ALG_ASYNC |
998 CRYPTO_ALG_ALLOCATES_MEMORY |
999 CRYPTO_ALG_KERN_DRIVER_ONLY,
1000 .cra_blocksize = SHA1_BLOCK_SIZE,
1001 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1002 .cra_init = safexcel_ahash_cra_init,
1003 .cra_exit = safexcel_ahash_cra_exit,
1004 .cra_module = THIS_MODULE,
1005 },
1006 },
1007 },
1008 };
1009
safexcel_hmac_sha1_init(struct ahash_request * areq)1010 static int safexcel_hmac_sha1_init(struct ahash_request *areq)
1011 {
1012 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1013 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
1014
1015 memset(req, 0, sizeof(*req));
1016
1017 /* Start from ipad precompute */
1018 memcpy(req->state, &ctx->base.ipad, SHA1_DIGEST_SIZE);
1019 /* Already processed the key^ipad part now! */
1020 req->len = SHA1_BLOCK_SIZE;
1021 req->processed = SHA1_BLOCK_SIZE;
1022
1023 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA1;
1024 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1025 req->state_sz = SHA1_DIGEST_SIZE;
1026 req->digest_sz = SHA1_DIGEST_SIZE;
1027 req->block_sz = SHA1_BLOCK_SIZE;
1028 req->hmac = true;
1029
1030 return 0;
1031 }
1032
safexcel_hmac_sha1_digest(struct ahash_request * areq)1033 static int safexcel_hmac_sha1_digest(struct ahash_request *areq)
1034 {
1035 int ret = safexcel_hmac_sha1_init(areq);
1036
1037 if (ret)
1038 return ret;
1039
1040 return safexcel_ahash_finup(areq);
1041 }
1042
1043 struct safexcel_ahash_result {
1044 struct completion completion;
1045 int error;
1046 };
1047
safexcel_ahash_complete(struct crypto_async_request * req,int error)1048 static void safexcel_ahash_complete(struct crypto_async_request *req, int error)
1049 {
1050 struct safexcel_ahash_result *result = req->data;
1051
1052 if (error == -EINPROGRESS)
1053 return;
1054
1055 result->error = error;
1056 complete(&result->completion);
1057 }
1058
safexcel_hmac_init_pad(struct ahash_request * areq,unsigned int blocksize,const u8 * key,unsigned int keylen,u8 * ipad,u8 * opad)1059 static int safexcel_hmac_init_pad(struct ahash_request *areq,
1060 unsigned int blocksize, const u8 *key,
1061 unsigned int keylen, u8 *ipad, u8 *opad)
1062 {
1063 struct safexcel_ahash_result result;
1064 struct scatterlist sg;
1065 int ret, i;
1066 u8 *keydup;
1067
1068 if (keylen <= blocksize) {
1069 memcpy(ipad, key, keylen);
1070 } else {
1071 keydup = kmemdup(key, keylen, GFP_KERNEL);
1072 if (!keydup)
1073 return -ENOMEM;
1074
1075 ahash_request_set_callback(areq, CRYPTO_TFM_REQ_MAY_BACKLOG,
1076 safexcel_ahash_complete, &result);
1077 sg_init_one(&sg, keydup, keylen);
1078 ahash_request_set_crypt(areq, &sg, ipad, keylen);
1079 init_completion(&result.completion);
1080
1081 ret = crypto_ahash_digest(areq);
1082 if (ret == -EINPROGRESS || ret == -EBUSY) {
1083 wait_for_completion_interruptible(&result.completion);
1084 ret = result.error;
1085 }
1086
1087 /* Avoid leaking */
1088 kfree_sensitive(keydup);
1089
1090 if (ret)
1091 return ret;
1092
1093 keylen = crypto_ahash_digestsize(crypto_ahash_reqtfm(areq));
1094 }
1095
1096 memset(ipad + keylen, 0, blocksize - keylen);
1097 memcpy(opad, ipad, blocksize);
1098
1099 for (i = 0; i < blocksize; i++) {
1100 ipad[i] ^= HMAC_IPAD_VALUE;
1101 opad[i] ^= HMAC_OPAD_VALUE;
1102 }
1103
1104 return 0;
1105 }
1106
safexcel_hmac_init_iv(struct ahash_request * areq,unsigned int blocksize,u8 * pad,void * state)1107 static int safexcel_hmac_init_iv(struct ahash_request *areq,
1108 unsigned int blocksize, u8 *pad, void *state)
1109 {
1110 struct safexcel_ahash_result result;
1111 struct safexcel_ahash_req *req;
1112 struct scatterlist sg;
1113 int ret;
1114
1115 ahash_request_set_callback(areq, CRYPTO_TFM_REQ_MAY_BACKLOG,
1116 safexcel_ahash_complete, &result);
1117 sg_init_one(&sg, pad, blocksize);
1118 ahash_request_set_crypt(areq, &sg, pad, blocksize);
1119 init_completion(&result.completion);
1120
1121 ret = crypto_ahash_init(areq);
1122 if (ret)
1123 return ret;
1124
1125 req = ahash_request_ctx(areq);
1126 req->hmac = true;
1127 req->last_req = true;
1128
1129 ret = crypto_ahash_update(areq);
1130 if (ret && ret != -EINPROGRESS && ret != -EBUSY)
1131 return ret;
1132
1133 wait_for_completion_interruptible(&result.completion);
1134 if (result.error)
1135 return result.error;
1136
1137 return crypto_ahash_export(areq, state);
1138 }
1139
__safexcel_hmac_setkey(const char * alg,const u8 * key,unsigned int keylen,void * istate,void * ostate)1140 static int __safexcel_hmac_setkey(const char *alg, const u8 *key,
1141 unsigned int keylen,
1142 void *istate, void *ostate)
1143 {
1144 struct ahash_request *areq;
1145 struct crypto_ahash *tfm;
1146 unsigned int blocksize;
1147 u8 *ipad, *opad;
1148 int ret;
1149
1150 tfm = crypto_alloc_ahash(alg, 0, 0);
1151 if (IS_ERR(tfm))
1152 return PTR_ERR(tfm);
1153
1154 areq = ahash_request_alloc(tfm, GFP_KERNEL);
1155 if (!areq) {
1156 ret = -ENOMEM;
1157 goto free_ahash;
1158 }
1159
1160 crypto_ahash_clear_flags(tfm, ~0);
1161 blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
1162
1163 ipad = kcalloc(2, blocksize, GFP_KERNEL);
1164 if (!ipad) {
1165 ret = -ENOMEM;
1166 goto free_request;
1167 }
1168
1169 opad = ipad + blocksize;
1170
1171 ret = safexcel_hmac_init_pad(areq, blocksize, key, keylen, ipad, opad);
1172 if (ret)
1173 goto free_ipad;
1174
1175 ret = safexcel_hmac_init_iv(areq, blocksize, ipad, istate);
1176 if (ret)
1177 goto free_ipad;
1178
1179 ret = safexcel_hmac_init_iv(areq, blocksize, opad, ostate);
1180
1181 free_ipad:
1182 kfree(ipad);
1183 free_request:
1184 ahash_request_free(areq);
1185 free_ahash:
1186 crypto_free_ahash(tfm);
1187
1188 return ret;
1189 }
1190
safexcel_hmac_setkey(struct safexcel_context * base,const u8 * key,unsigned int keylen,const char * alg,unsigned int state_sz)1191 int safexcel_hmac_setkey(struct safexcel_context *base, const u8 *key,
1192 unsigned int keylen, const char *alg,
1193 unsigned int state_sz)
1194 {
1195 struct safexcel_crypto_priv *priv = base->priv;
1196 struct safexcel_ahash_export_state istate, ostate;
1197 int ret;
1198
1199 ret = __safexcel_hmac_setkey(alg, key, keylen, &istate, &ostate);
1200 if (ret)
1201 return ret;
1202
1203 if (priv->flags & EIP197_TRC_CACHE && base->ctxr &&
1204 (memcmp(&base->ipad, istate.state, state_sz) ||
1205 memcmp(&base->opad, ostate.state, state_sz)))
1206 base->needs_inv = true;
1207
1208 memcpy(&base->ipad, &istate.state, state_sz);
1209 memcpy(&base->opad, &ostate.state, state_sz);
1210
1211 return 0;
1212 }
1213
safexcel_hmac_alg_setkey(struct crypto_ahash * tfm,const u8 * key,unsigned int keylen,const char * alg,unsigned int state_sz)1214 static int safexcel_hmac_alg_setkey(struct crypto_ahash *tfm, const u8 *key,
1215 unsigned int keylen, const char *alg,
1216 unsigned int state_sz)
1217 {
1218 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
1219
1220 return safexcel_hmac_setkey(&ctx->base, key, keylen, alg, state_sz);
1221 }
1222
safexcel_hmac_sha1_setkey(struct crypto_ahash * tfm,const u8 * key,unsigned int keylen)1223 static int safexcel_hmac_sha1_setkey(struct crypto_ahash *tfm, const u8 *key,
1224 unsigned int keylen)
1225 {
1226 return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sha1",
1227 SHA1_DIGEST_SIZE);
1228 }
1229
1230 struct safexcel_alg_template safexcel_alg_hmac_sha1 = {
1231 .type = SAFEXCEL_ALG_TYPE_AHASH,
1232 .algo_mask = SAFEXCEL_ALG_SHA1,
1233 .alg.ahash = {
1234 .init = safexcel_hmac_sha1_init,
1235 .update = safexcel_ahash_update,
1236 .final = safexcel_ahash_final,
1237 .finup = safexcel_ahash_finup,
1238 .digest = safexcel_hmac_sha1_digest,
1239 .setkey = safexcel_hmac_sha1_setkey,
1240 .export = safexcel_ahash_export,
1241 .import = safexcel_ahash_import,
1242 .halg = {
1243 .digestsize = SHA1_DIGEST_SIZE,
1244 .statesize = sizeof(struct safexcel_ahash_export_state),
1245 .base = {
1246 .cra_name = "hmac(sha1)",
1247 .cra_driver_name = "safexcel-hmac-sha1",
1248 .cra_priority = SAFEXCEL_CRA_PRIORITY,
1249 .cra_flags = CRYPTO_ALG_ASYNC |
1250 CRYPTO_ALG_ALLOCATES_MEMORY |
1251 CRYPTO_ALG_KERN_DRIVER_ONLY,
1252 .cra_blocksize = SHA1_BLOCK_SIZE,
1253 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1254 .cra_init = safexcel_ahash_cra_init,
1255 .cra_exit = safexcel_ahash_cra_exit,
1256 .cra_module = THIS_MODULE,
1257 },
1258 },
1259 },
1260 };
1261
safexcel_sha256_init(struct ahash_request * areq)1262 static int safexcel_sha256_init(struct ahash_request *areq)
1263 {
1264 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1265 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
1266
1267 memset(req, 0, sizeof(*req));
1268
1269 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA256;
1270 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1271 req->state_sz = SHA256_DIGEST_SIZE;
1272 req->digest_sz = SHA256_DIGEST_SIZE;
1273 req->block_sz = SHA256_BLOCK_SIZE;
1274
1275 return 0;
1276 }
1277
safexcel_sha256_digest(struct ahash_request * areq)1278 static int safexcel_sha256_digest(struct ahash_request *areq)
1279 {
1280 int ret = safexcel_sha256_init(areq);
1281
1282 if (ret)
1283 return ret;
1284
1285 return safexcel_ahash_finup(areq);
1286 }
1287
1288 struct safexcel_alg_template safexcel_alg_sha256 = {
1289 .type = SAFEXCEL_ALG_TYPE_AHASH,
1290 .algo_mask = SAFEXCEL_ALG_SHA2_256,
1291 .alg.ahash = {
1292 .init = safexcel_sha256_init,
1293 .update = safexcel_ahash_update,
1294 .final = safexcel_ahash_final,
1295 .finup = safexcel_ahash_finup,
1296 .digest = safexcel_sha256_digest,
1297 .export = safexcel_ahash_export,
1298 .import = safexcel_ahash_import,
1299 .halg = {
1300 .digestsize = SHA256_DIGEST_SIZE,
1301 .statesize = sizeof(struct safexcel_ahash_export_state),
1302 .base = {
1303 .cra_name = "sha256",
1304 .cra_driver_name = "safexcel-sha256",
1305 .cra_priority = SAFEXCEL_CRA_PRIORITY,
1306 .cra_flags = CRYPTO_ALG_ASYNC |
1307 CRYPTO_ALG_ALLOCATES_MEMORY |
1308 CRYPTO_ALG_KERN_DRIVER_ONLY,
1309 .cra_blocksize = SHA256_BLOCK_SIZE,
1310 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1311 .cra_init = safexcel_ahash_cra_init,
1312 .cra_exit = safexcel_ahash_cra_exit,
1313 .cra_module = THIS_MODULE,
1314 },
1315 },
1316 },
1317 };
1318
safexcel_sha224_init(struct ahash_request * areq)1319 static int safexcel_sha224_init(struct ahash_request *areq)
1320 {
1321 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1322 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
1323
1324 memset(req, 0, sizeof(*req));
1325
1326 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA224;
1327 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1328 req->state_sz = SHA256_DIGEST_SIZE;
1329 req->digest_sz = SHA256_DIGEST_SIZE;
1330 req->block_sz = SHA256_BLOCK_SIZE;
1331
1332 return 0;
1333 }
1334
safexcel_sha224_digest(struct ahash_request * areq)1335 static int safexcel_sha224_digest(struct ahash_request *areq)
1336 {
1337 int ret = safexcel_sha224_init(areq);
1338
1339 if (ret)
1340 return ret;
1341
1342 return safexcel_ahash_finup(areq);
1343 }
1344
1345 struct safexcel_alg_template safexcel_alg_sha224 = {
1346 .type = SAFEXCEL_ALG_TYPE_AHASH,
1347 .algo_mask = SAFEXCEL_ALG_SHA2_256,
1348 .alg.ahash = {
1349 .init = safexcel_sha224_init,
1350 .update = safexcel_ahash_update,
1351 .final = safexcel_ahash_final,
1352 .finup = safexcel_ahash_finup,
1353 .digest = safexcel_sha224_digest,
1354 .export = safexcel_ahash_export,
1355 .import = safexcel_ahash_import,
1356 .halg = {
1357 .digestsize = SHA224_DIGEST_SIZE,
1358 .statesize = sizeof(struct safexcel_ahash_export_state),
1359 .base = {
1360 .cra_name = "sha224",
1361 .cra_driver_name = "safexcel-sha224",
1362 .cra_priority = SAFEXCEL_CRA_PRIORITY,
1363 .cra_flags = CRYPTO_ALG_ASYNC |
1364 CRYPTO_ALG_ALLOCATES_MEMORY |
1365 CRYPTO_ALG_KERN_DRIVER_ONLY,
1366 .cra_blocksize = SHA224_BLOCK_SIZE,
1367 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1368 .cra_init = safexcel_ahash_cra_init,
1369 .cra_exit = safexcel_ahash_cra_exit,
1370 .cra_module = THIS_MODULE,
1371 },
1372 },
1373 },
1374 };
1375
safexcel_hmac_sha224_setkey(struct crypto_ahash * tfm,const u8 * key,unsigned int keylen)1376 static int safexcel_hmac_sha224_setkey(struct crypto_ahash *tfm, const u8 *key,
1377 unsigned int keylen)
1378 {
1379 return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sha224",
1380 SHA256_DIGEST_SIZE);
1381 }
1382
safexcel_hmac_sha224_init(struct ahash_request * areq)1383 static int safexcel_hmac_sha224_init(struct ahash_request *areq)
1384 {
1385 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1386 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
1387
1388 memset(req, 0, sizeof(*req));
1389
1390 /* Start from ipad precompute */
1391 memcpy(req->state, &ctx->base.ipad, SHA256_DIGEST_SIZE);
1392 /* Already processed the key^ipad part now! */
1393 req->len = SHA256_BLOCK_SIZE;
1394 req->processed = SHA256_BLOCK_SIZE;
1395
1396 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA224;
1397 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1398 req->state_sz = SHA256_DIGEST_SIZE;
1399 req->digest_sz = SHA256_DIGEST_SIZE;
1400 req->block_sz = SHA256_BLOCK_SIZE;
1401 req->hmac = true;
1402
1403 return 0;
1404 }
1405
safexcel_hmac_sha224_digest(struct ahash_request * areq)1406 static int safexcel_hmac_sha224_digest(struct ahash_request *areq)
1407 {
1408 int ret = safexcel_hmac_sha224_init(areq);
1409
1410 if (ret)
1411 return ret;
1412
1413 return safexcel_ahash_finup(areq);
1414 }
1415
1416 struct safexcel_alg_template safexcel_alg_hmac_sha224 = {
1417 .type = SAFEXCEL_ALG_TYPE_AHASH,
1418 .algo_mask = SAFEXCEL_ALG_SHA2_256,
1419 .alg.ahash = {
1420 .init = safexcel_hmac_sha224_init,
1421 .update = safexcel_ahash_update,
1422 .final = safexcel_ahash_final,
1423 .finup = safexcel_ahash_finup,
1424 .digest = safexcel_hmac_sha224_digest,
1425 .setkey = safexcel_hmac_sha224_setkey,
1426 .export = safexcel_ahash_export,
1427 .import = safexcel_ahash_import,
1428 .halg = {
1429 .digestsize = SHA224_DIGEST_SIZE,
1430 .statesize = sizeof(struct safexcel_ahash_export_state),
1431 .base = {
1432 .cra_name = "hmac(sha224)",
1433 .cra_driver_name = "safexcel-hmac-sha224",
1434 .cra_priority = SAFEXCEL_CRA_PRIORITY,
1435 .cra_flags = CRYPTO_ALG_ASYNC |
1436 CRYPTO_ALG_ALLOCATES_MEMORY |
1437 CRYPTO_ALG_KERN_DRIVER_ONLY,
1438 .cra_blocksize = SHA224_BLOCK_SIZE,
1439 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1440 .cra_init = safexcel_ahash_cra_init,
1441 .cra_exit = safexcel_ahash_cra_exit,
1442 .cra_module = THIS_MODULE,
1443 },
1444 },
1445 },
1446 };
1447
safexcel_hmac_sha256_setkey(struct crypto_ahash * tfm,const u8 * key,unsigned int keylen)1448 static int safexcel_hmac_sha256_setkey(struct crypto_ahash *tfm, const u8 *key,
1449 unsigned int keylen)
1450 {
1451 return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sha256",
1452 SHA256_DIGEST_SIZE);
1453 }
1454
safexcel_hmac_sha256_init(struct ahash_request * areq)1455 static int safexcel_hmac_sha256_init(struct ahash_request *areq)
1456 {
1457 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1458 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
1459
1460 memset(req, 0, sizeof(*req));
1461
1462 /* Start from ipad precompute */
1463 memcpy(req->state, &ctx->base.ipad, SHA256_DIGEST_SIZE);
1464 /* Already processed the key^ipad part now! */
1465 req->len = SHA256_BLOCK_SIZE;
1466 req->processed = SHA256_BLOCK_SIZE;
1467
1468 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA256;
1469 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1470 req->state_sz = SHA256_DIGEST_SIZE;
1471 req->digest_sz = SHA256_DIGEST_SIZE;
1472 req->block_sz = SHA256_BLOCK_SIZE;
1473 req->hmac = true;
1474
1475 return 0;
1476 }
1477
safexcel_hmac_sha256_digest(struct ahash_request * areq)1478 static int safexcel_hmac_sha256_digest(struct ahash_request *areq)
1479 {
1480 int ret = safexcel_hmac_sha256_init(areq);
1481
1482 if (ret)
1483 return ret;
1484
1485 return safexcel_ahash_finup(areq);
1486 }
1487
1488 struct safexcel_alg_template safexcel_alg_hmac_sha256 = {
1489 .type = SAFEXCEL_ALG_TYPE_AHASH,
1490 .algo_mask = SAFEXCEL_ALG_SHA2_256,
1491 .alg.ahash = {
1492 .init = safexcel_hmac_sha256_init,
1493 .update = safexcel_ahash_update,
1494 .final = safexcel_ahash_final,
1495 .finup = safexcel_ahash_finup,
1496 .digest = safexcel_hmac_sha256_digest,
1497 .setkey = safexcel_hmac_sha256_setkey,
1498 .export = safexcel_ahash_export,
1499 .import = safexcel_ahash_import,
1500 .halg = {
1501 .digestsize = SHA256_DIGEST_SIZE,
1502 .statesize = sizeof(struct safexcel_ahash_export_state),
1503 .base = {
1504 .cra_name = "hmac(sha256)",
1505 .cra_driver_name = "safexcel-hmac-sha256",
1506 .cra_priority = SAFEXCEL_CRA_PRIORITY,
1507 .cra_flags = CRYPTO_ALG_ASYNC |
1508 CRYPTO_ALG_ALLOCATES_MEMORY |
1509 CRYPTO_ALG_KERN_DRIVER_ONLY,
1510 .cra_blocksize = SHA256_BLOCK_SIZE,
1511 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1512 .cra_init = safexcel_ahash_cra_init,
1513 .cra_exit = safexcel_ahash_cra_exit,
1514 .cra_module = THIS_MODULE,
1515 },
1516 },
1517 },
1518 };
1519
safexcel_sha512_init(struct ahash_request * areq)1520 static int safexcel_sha512_init(struct ahash_request *areq)
1521 {
1522 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1523 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
1524
1525 memset(req, 0, sizeof(*req));
1526
1527 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA512;
1528 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1529 req->state_sz = SHA512_DIGEST_SIZE;
1530 req->digest_sz = SHA512_DIGEST_SIZE;
1531 req->block_sz = SHA512_BLOCK_SIZE;
1532
1533 return 0;
1534 }
1535
safexcel_sha512_digest(struct ahash_request * areq)1536 static int safexcel_sha512_digest(struct ahash_request *areq)
1537 {
1538 int ret = safexcel_sha512_init(areq);
1539
1540 if (ret)
1541 return ret;
1542
1543 return safexcel_ahash_finup(areq);
1544 }
1545
1546 struct safexcel_alg_template safexcel_alg_sha512 = {
1547 .type = SAFEXCEL_ALG_TYPE_AHASH,
1548 .algo_mask = SAFEXCEL_ALG_SHA2_512,
1549 .alg.ahash = {
1550 .init = safexcel_sha512_init,
1551 .update = safexcel_ahash_update,
1552 .final = safexcel_ahash_final,
1553 .finup = safexcel_ahash_finup,
1554 .digest = safexcel_sha512_digest,
1555 .export = safexcel_ahash_export,
1556 .import = safexcel_ahash_import,
1557 .halg = {
1558 .digestsize = SHA512_DIGEST_SIZE,
1559 .statesize = sizeof(struct safexcel_ahash_export_state),
1560 .base = {
1561 .cra_name = "sha512",
1562 .cra_driver_name = "safexcel-sha512",
1563 .cra_priority = SAFEXCEL_CRA_PRIORITY,
1564 .cra_flags = CRYPTO_ALG_ASYNC |
1565 CRYPTO_ALG_ALLOCATES_MEMORY |
1566 CRYPTO_ALG_KERN_DRIVER_ONLY,
1567 .cra_blocksize = SHA512_BLOCK_SIZE,
1568 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1569 .cra_init = safexcel_ahash_cra_init,
1570 .cra_exit = safexcel_ahash_cra_exit,
1571 .cra_module = THIS_MODULE,
1572 },
1573 },
1574 },
1575 };
1576
safexcel_sha384_init(struct ahash_request * areq)1577 static int safexcel_sha384_init(struct ahash_request *areq)
1578 {
1579 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1580 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
1581
1582 memset(req, 0, sizeof(*req));
1583
1584 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA384;
1585 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1586 req->state_sz = SHA512_DIGEST_SIZE;
1587 req->digest_sz = SHA512_DIGEST_SIZE;
1588 req->block_sz = SHA512_BLOCK_SIZE;
1589
1590 return 0;
1591 }
1592
safexcel_sha384_digest(struct ahash_request * areq)1593 static int safexcel_sha384_digest(struct ahash_request *areq)
1594 {
1595 int ret = safexcel_sha384_init(areq);
1596
1597 if (ret)
1598 return ret;
1599
1600 return safexcel_ahash_finup(areq);
1601 }
1602
1603 struct safexcel_alg_template safexcel_alg_sha384 = {
1604 .type = SAFEXCEL_ALG_TYPE_AHASH,
1605 .algo_mask = SAFEXCEL_ALG_SHA2_512,
1606 .alg.ahash = {
1607 .init = safexcel_sha384_init,
1608 .update = safexcel_ahash_update,
1609 .final = safexcel_ahash_final,
1610 .finup = safexcel_ahash_finup,
1611 .digest = safexcel_sha384_digest,
1612 .export = safexcel_ahash_export,
1613 .import = safexcel_ahash_import,
1614 .halg = {
1615 .digestsize = SHA384_DIGEST_SIZE,
1616 .statesize = sizeof(struct safexcel_ahash_export_state),
1617 .base = {
1618 .cra_name = "sha384",
1619 .cra_driver_name = "safexcel-sha384",
1620 .cra_priority = SAFEXCEL_CRA_PRIORITY,
1621 .cra_flags = CRYPTO_ALG_ASYNC |
1622 CRYPTO_ALG_ALLOCATES_MEMORY |
1623 CRYPTO_ALG_KERN_DRIVER_ONLY,
1624 .cra_blocksize = SHA384_BLOCK_SIZE,
1625 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1626 .cra_init = safexcel_ahash_cra_init,
1627 .cra_exit = safexcel_ahash_cra_exit,
1628 .cra_module = THIS_MODULE,
1629 },
1630 },
1631 },
1632 };
1633
safexcel_hmac_sha512_setkey(struct crypto_ahash * tfm,const u8 * key,unsigned int keylen)1634 static int safexcel_hmac_sha512_setkey(struct crypto_ahash *tfm, const u8 *key,
1635 unsigned int keylen)
1636 {
1637 return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sha512",
1638 SHA512_DIGEST_SIZE);
1639 }
1640
safexcel_hmac_sha512_init(struct ahash_request * areq)1641 static int safexcel_hmac_sha512_init(struct ahash_request *areq)
1642 {
1643 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1644 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
1645
1646 memset(req, 0, sizeof(*req));
1647
1648 /* Start from ipad precompute */
1649 memcpy(req->state, &ctx->base.ipad, SHA512_DIGEST_SIZE);
1650 /* Already processed the key^ipad part now! */
1651 req->len = SHA512_BLOCK_SIZE;
1652 req->processed = SHA512_BLOCK_SIZE;
1653
1654 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA512;
1655 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1656 req->state_sz = SHA512_DIGEST_SIZE;
1657 req->digest_sz = SHA512_DIGEST_SIZE;
1658 req->block_sz = SHA512_BLOCK_SIZE;
1659 req->hmac = true;
1660
1661 return 0;
1662 }
1663
safexcel_hmac_sha512_digest(struct ahash_request * areq)1664 static int safexcel_hmac_sha512_digest(struct ahash_request *areq)
1665 {
1666 int ret = safexcel_hmac_sha512_init(areq);
1667
1668 if (ret)
1669 return ret;
1670
1671 return safexcel_ahash_finup(areq);
1672 }
1673
1674 struct safexcel_alg_template safexcel_alg_hmac_sha512 = {
1675 .type = SAFEXCEL_ALG_TYPE_AHASH,
1676 .algo_mask = SAFEXCEL_ALG_SHA2_512,
1677 .alg.ahash = {
1678 .init = safexcel_hmac_sha512_init,
1679 .update = safexcel_ahash_update,
1680 .final = safexcel_ahash_final,
1681 .finup = safexcel_ahash_finup,
1682 .digest = safexcel_hmac_sha512_digest,
1683 .setkey = safexcel_hmac_sha512_setkey,
1684 .export = safexcel_ahash_export,
1685 .import = safexcel_ahash_import,
1686 .halg = {
1687 .digestsize = SHA512_DIGEST_SIZE,
1688 .statesize = sizeof(struct safexcel_ahash_export_state),
1689 .base = {
1690 .cra_name = "hmac(sha512)",
1691 .cra_driver_name = "safexcel-hmac-sha512",
1692 .cra_priority = SAFEXCEL_CRA_PRIORITY,
1693 .cra_flags = CRYPTO_ALG_ASYNC |
1694 CRYPTO_ALG_ALLOCATES_MEMORY |
1695 CRYPTO_ALG_KERN_DRIVER_ONLY,
1696 .cra_blocksize = SHA512_BLOCK_SIZE,
1697 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1698 .cra_init = safexcel_ahash_cra_init,
1699 .cra_exit = safexcel_ahash_cra_exit,
1700 .cra_module = THIS_MODULE,
1701 },
1702 },
1703 },
1704 };
1705
safexcel_hmac_sha384_setkey(struct crypto_ahash * tfm,const u8 * key,unsigned int keylen)1706 static int safexcel_hmac_sha384_setkey(struct crypto_ahash *tfm, const u8 *key,
1707 unsigned int keylen)
1708 {
1709 return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sha384",
1710 SHA512_DIGEST_SIZE);
1711 }
1712
safexcel_hmac_sha384_init(struct ahash_request * areq)1713 static int safexcel_hmac_sha384_init(struct ahash_request *areq)
1714 {
1715 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1716 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
1717
1718 memset(req, 0, sizeof(*req));
1719
1720 /* Start from ipad precompute */
1721 memcpy(req->state, &ctx->base.ipad, SHA512_DIGEST_SIZE);
1722 /* Already processed the key^ipad part now! */
1723 req->len = SHA512_BLOCK_SIZE;
1724 req->processed = SHA512_BLOCK_SIZE;
1725
1726 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA384;
1727 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1728 req->state_sz = SHA512_DIGEST_SIZE;
1729 req->digest_sz = SHA512_DIGEST_SIZE;
1730 req->block_sz = SHA512_BLOCK_SIZE;
1731 req->hmac = true;
1732
1733 return 0;
1734 }
1735
safexcel_hmac_sha384_digest(struct ahash_request * areq)1736 static int safexcel_hmac_sha384_digest(struct ahash_request *areq)
1737 {
1738 int ret = safexcel_hmac_sha384_init(areq);
1739
1740 if (ret)
1741 return ret;
1742
1743 return safexcel_ahash_finup(areq);
1744 }
1745
1746 struct safexcel_alg_template safexcel_alg_hmac_sha384 = {
1747 .type = SAFEXCEL_ALG_TYPE_AHASH,
1748 .algo_mask = SAFEXCEL_ALG_SHA2_512,
1749 .alg.ahash = {
1750 .init = safexcel_hmac_sha384_init,
1751 .update = safexcel_ahash_update,
1752 .final = safexcel_ahash_final,
1753 .finup = safexcel_ahash_finup,
1754 .digest = safexcel_hmac_sha384_digest,
1755 .setkey = safexcel_hmac_sha384_setkey,
1756 .export = safexcel_ahash_export,
1757 .import = safexcel_ahash_import,
1758 .halg = {
1759 .digestsize = SHA384_DIGEST_SIZE,
1760 .statesize = sizeof(struct safexcel_ahash_export_state),
1761 .base = {
1762 .cra_name = "hmac(sha384)",
1763 .cra_driver_name = "safexcel-hmac-sha384",
1764 .cra_priority = SAFEXCEL_CRA_PRIORITY,
1765 .cra_flags = CRYPTO_ALG_ASYNC |
1766 CRYPTO_ALG_ALLOCATES_MEMORY |
1767 CRYPTO_ALG_KERN_DRIVER_ONLY,
1768 .cra_blocksize = SHA384_BLOCK_SIZE,
1769 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1770 .cra_init = safexcel_ahash_cra_init,
1771 .cra_exit = safexcel_ahash_cra_exit,
1772 .cra_module = THIS_MODULE,
1773 },
1774 },
1775 },
1776 };
1777
safexcel_md5_init(struct ahash_request * areq)1778 static int safexcel_md5_init(struct ahash_request *areq)
1779 {
1780 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1781 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
1782
1783 memset(req, 0, sizeof(*req));
1784
1785 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_MD5;
1786 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1787 req->state_sz = MD5_DIGEST_SIZE;
1788 req->digest_sz = MD5_DIGEST_SIZE;
1789 req->block_sz = MD5_HMAC_BLOCK_SIZE;
1790
1791 return 0;
1792 }
1793
safexcel_md5_digest(struct ahash_request * areq)1794 static int safexcel_md5_digest(struct ahash_request *areq)
1795 {
1796 int ret = safexcel_md5_init(areq);
1797
1798 if (ret)
1799 return ret;
1800
1801 return safexcel_ahash_finup(areq);
1802 }
1803
1804 struct safexcel_alg_template safexcel_alg_md5 = {
1805 .type = SAFEXCEL_ALG_TYPE_AHASH,
1806 .algo_mask = SAFEXCEL_ALG_MD5,
1807 .alg.ahash = {
1808 .init = safexcel_md5_init,
1809 .update = safexcel_ahash_update,
1810 .final = safexcel_ahash_final,
1811 .finup = safexcel_ahash_finup,
1812 .digest = safexcel_md5_digest,
1813 .export = safexcel_ahash_export,
1814 .import = safexcel_ahash_import,
1815 .halg = {
1816 .digestsize = MD5_DIGEST_SIZE,
1817 .statesize = sizeof(struct safexcel_ahash_export_state),
1818 .base = {
1819 .cra_name = "md5",
1820 .cra_driver_name = "safexcel-md5",
1821 .cra_priority = SAFEXCEL_CRA_PRIORITY,
1822 .cra_flags = CRYPTO_ALG_ASYNC |
1823 CRYPTO_ALG_ALLOCATES_MEMORY |
1824 CRYPTO_ALG_KERN_DRIVER_ONLY,
1825 .cra_blocksize = MD5_HMAC_BLOCK_SIZE,
1826 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1827 .cra_init = safexcel_ahash_cra_init,
1828 .cra_exit = safexcel_ahash_cra_exit,
1829 .cra_module = THIS_MODULE,
1830 },
1831 },
1832 },
1833 };
1834
safexcel_hmac_md5_init(struct ahash_request * areq)1835 static int safexcel_hmac_md5_init(struct ahash_request *areq)
1836 {
1837 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1838 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
1839
1840 memset(req, 0, sizeof(*req));
1841
1842 /* Start from ipad precompute */
1843 memcpy(req->state, &ctx->base.ipad, MD5_DIGEST_SIZE);
1844 /* Already processed the key^ipad part now! */
1845 req->len = MD5_HMAC_BLOCK_SIZE;
1846 req->processed = MD5_HMAC_BLOCK_SIZE;
1847
1848 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_MD5;
1849 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1850 req->state_sz = MD5_DIGEST_SIZE;
1851 req->digest_sz = MD5_DIGEST_SIZE;
1852 req->block_sz = MD5_HMAC_BLOCK_SIZE;
1853 req->len_is_le = true; /* MD5 is little endian! ... */
1854 req->hmac = true;
1855
1856 return 0;
1857 }
1858
safexcel_hmac_md5_setkey(struct crypto_ahash * tfm,const u8 * key,unsigned int keylen)1859 static int safexcel_hmac_md5_setkey(struct crypto_ahash *tfm, const u8 *key,
1860 unsigned int keylen)
1861 {
1862 return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-md5",
1863 MD5_DIGEST_SIZE);
1864 }
1865
safexcel_hmac_md5_digest(struct ahash_request * areq)1866 static int safexcel_hmac_md5_digest(struct ahash_request *areq)
1867 {
1868 int ret = safexcel_hmac_md5_init(areq);
1869
1870 if (ret)
1871 return ret;
1872
1873 return safexcel_ahash_finup(areq);
1874 }
1875
1876 struct safexcel_alg_template safexcel_alg_hmac_md5 = {
1877 .type = SAFEXCEL_ALG_TYPE_AHASH,
1878 .algo_mask = SAFEXCEL_ALG_MD5,
1879 .alg.ahash = {
1880 .init = safexcel_hmac_md5_init,
1881 .update = safexcel_ahash_update,
1882 .final = safexcel_ahash_final,
1883 .finup = safexcel_ahash_finup,
1884 .digest = safexcel_hmac_md5_digest,
1885 .setkey = safexcel_hmac_md5_setkey,
1886 .export = safexcel_ahash_export,
1887 .import = safexcel_ahash_import,
1888 .halg = {
1889 .digestsize = MD5_DIGEST_SIZE,
1890 .statesize = sizeof(struct safexcel_ahash_export_state),
1891 .base = {
1892 .cra_name = "hmac(md5)",
1893 .cra_driver_name = "safexcel-hmac-md5",
1894 .cra_priority = SAFEXCEL_CRA_PRIORITY,
1895 .cra_flags = CRYPTO_ALG_ASYNC |
1896 CRYPTO_ALG_ALLOCATES_MEMORY |
1897 CRYPTO_ALG_KERN_DRIVER_ONLY,
1898 .cra_blocksize = MD5_HMAC_BLOCK_SIZE,
1899 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1900 .cra_init = safexcel_ahash_cra_init,
1901 .cra_exit = safexcel_ahash_cra_exit,
1902 .cra_module = THIS_MODULE,
1903 },
1904 },
1905 },
1906 };
1907
safexcel_crc32_cra_init(struct crypto_tfm * tfm)1908 static int safexcel_crc32_cra_init(struct crypto_tfm *tfm)
1909 {
1910 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
1911 int ret = safexcel_ahash_cra_init(tfm);
1912
1913 /* Default 'key' is all zeroes */
1914 memset(&ctx->base.ipad, 0, sizeof(u32));
1915 return ret;
1916 }
1917
safexcel_crc32_init(struct ahash_request * areq)1918 static int safexcel_crc32_init(struct ahash_request *areq)
1919 {
1920 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1921 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
1922
1923 memset(req, 0, sizeof(*req));
1924
1925 /* Start from loaded key */
1926 req->state[0] = cpu_to_le32(~ctx->base.ipad.word[0]);
1927 /* Set processed to non-zero to enable invalidation detection */
1928 req->len = sizeof(u32);
1929 req->processed = sizeof(u32);
1930
1931 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_CRC32;
1932 req->digest = CONTEXT_CONTROL_DIGEST_XCM;
1933 req->state_sz = sizeof(u32);
1934 req->digest_sz = sizeof(u32);
1935 req->block_sz = sizeof(u32);
1936
1937 return 0;
1938 }
1939
safexcel_crc32_setkey(struct crypto_ahash * tfm,const u8 * key,unsigned int keylen)1940 static int safexcel_crc32_setkey(struct crypto_ahash *tfm, const u8 *key,
1941 unsigned int keylen)
1942 {
1943 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
1944
1945 if (keylen != sizeof(u32))
1946 return -EINVAL;
1947
1948 memcpy(&ctx->base.ipad, key, sizeof(u32));
1949 return 0;
1950 }
1951
safexcel_crc32_digest(struct ahash_request * areq)1952 static int safexcel_crc32_digest(struct ahash_request *areq)
1953 {
1954 return safexcel_crc32_init(areq) ?: safexcel_ahash_finup(areq);
1955 }
1956
1957 struct safexcel_alg_template safexcel_alg_crc32 = {
1958 .type = SAFEXCEL_ALG_TYPE_AHASH,
1959 .algo_mask = 0,
1960 .alg.ahash = {
1961 .init = safexcel_crc32_init,
1962 .update = safexcel_ahash_update,
1963 .final = safexcel_ahash_final,
1964 .finup = safexcel_ahash_finup,
1965 .digest = safexcel_crc32_digest,
1966 .setkey = safexcel_crc32_setkey,
1967 .export = safexcel_ahash_export,
1968 .import = safexcel_ahash_import,
1969 .halg = {
1970 .digestsize = sizeof(u32),
1971 .statesize = sizeof(struct safexcel_ahash_export_state),
1972 .base = {
1973 .cra_name = "crc32",
1974 .cra_driver_name = "safexcel-crc32",
1975 .cra_priority = SAFEXCEL_CRA_PRIORITY,
1976 .cra_flags = CRYPTO_ALG_OPTIONAL_KEY |
1977 CRYPTO_ALG_ASYNC |
1978 CRYPTO_ALG_ALLOCATES_MEMORY |
1979 CRYPTO_ALG_KERN_DRIVER_ONLY,
1980 .cra_blocksize = 1,
1981 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1982 .cra_init = safexcel_crc32_cra_init,
1983 .cra_exit = safexcel_ahash_cra_exit,
1984 .cra_module = THIS_MODULE,
1985 },
1986 },
1987 },
1988 };
1989
safexcel_cbcmac_init(struct ahash_request * areq)1990 static int safexcel_cbcmac_init(struct ahash_request *areq)
1991 {
1992 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1993 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
1994
1995 memset(req, 0, sizeof(*req));
1996
1997 /* Start from loaded keys */
1998 memcpy(req->state, &ctx->base.ipad, ctx->key_sz);
1999 /* Set processed to non-zero to enable invalidation detection */
2000 req->len = AES_BLOCK_SIZE;
2001 req->processed = AES_BLOCK_SIZE;
2002
2003 req->digest = CONTEXT_CONTROL_DIGEST_XCM;
2004 req->state_sz = ctx->key_sz;
2005 req->digest_sz = AES_BLOCK_SIZE;
2006 req->block_sz = AES_BLOCK_SIZE;
2007 req->xcbcmac = true;
2008
2009 return 0;
2010 }
2011
safexcel_cbcmac_setkey(struct crypto_ahash * tfm,const u8 * key,unsigned int len)2012 static int safexcel_cbcmac_setkey(struct crypto_ahash *tfm, const u8 *key,
2013 unsigned int len)
2014 {
2015 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
2016 struct crypto_aes_ctx aes;
2017 int ret, i;
2018
2019 ret = aes_expandkey(&aes, key, len);
2020 if (ret)
2021 return ret;
2022
2023 memset(&ctx->base.ipad, 0, 2 * AES_BLOCK_SIZE);
2024 for (i = 0; i < len / sizeof(u32); i++)
2025 ctx->base.ipad.be[i + 8] = cpu_to_be32(aes.key_enc[i]);
2026
2027 if (len == AES_KEYSIZE_192) {
2028 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC192;
2029 ctx->key_sz = AES_MAX_KEY_SIZE + 2 * AES_BLOCK_SIZE;
2030 } else if (len == AES_KEYSIZE_256) {
2031 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC256;
2032 ctx->key_sz = AES_MAX_KEY_SIZE + 2 * AES_BLOCK_SIZE;
2033 } else {
2034 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC128;
2035 ctx->key_sz = AES_MIN_KEY_SIZE + 2 * AES_BLOCK_SIZE;
2036 }
2037 ctx->cbcmac = true;
2038
2039 memzero_explicit(&aes, sizeof(aes));
2040 return 0;
2041 }
2042
safexcel_cbcmac_digest(struct ahash_request * areq)2043 static int safexcel_cbcmac_digest(struct ahash_request *areq)
2044 {
2045 return safexcel_cbcmac_init(areq) ?: safexcel_ahash_finup(areq);
2046 }
2047
2048 struct safexcel_alg_template safexcel_alg_cbcmac = {
2049 .type = SAFEXCEL_ALG_TYPE_AHASH,
2050 .algo_mask = 0,
2051 .alg.ahash = {
2052 .init = safexcel_cbcmac_init,
2053 .update = safexcel_ahash_update,
2054 .final = safexcel_ahash_final,
2055 .finup = safexcel_ahash_finup,
2056 .digest = safexcel_cbcmac_digest,
2057 .setkey = safexcel_cbcmac_setkey,
2058 .export = safexcel_ahash_export,
2059 .import = safexcel_ahash_import,
2060 .halg = {
2061 .digestsize = AES_BLOCK_SIZE,
2062 .statesize = sizeof(struct safexcel_ahash_export_state),
2063 .base = {
2064 .cra_name = "cbcmac(aes)",
2065 .cra_driver_name = "safexcel-cbcmac-aes",
2066 .cra_priority = SAFEXCEL_CRA_PRIORITY,
2067 .cra_flags = CRYPTO_ALG_ASYNC |
2068 CRYPTO_ALG_ALLOCATES_MEMORY |
2069 CRYPTO_ALG_KERN_DRIVER_ONLY,
2070 .cra_blocksize = 1,
2071 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2072 .cra_init = safexcel_ahash_cra_init,
2073 .cra_exit = safexcel_ahash_cra_exit,
2074 .cra_module = THIS_MODULE,
2075 },
2076 },
2077 },
2078 };
2079
safexcel_xcbcmac_setkey(struct crypto_ahash * tfm,const u8 * key,unsigned int len)2080 static int safexcel_xcbcmac_setkey(struct crypto_ahash *tfm, const u8 *key,
2081 unsigned int len)
2082 {
2083 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
2084 struct crypto_aes_ctx aes;
2085 u32 key_tmp[3 * AES_BLOCK_SIZE / sizeof(u32)];
2086 int ret, i;
2087
2088 ret = aes_expandkey(&aes, key, len);
2089 if (ret)
2090 return ret;
2091
2092 /* precompute the XCBC key material */
2093 crypto_cipher_clear_flags(ctx->kaes, CRYPTO_TFM_REQ_MASK);
2094 crypto_cipher_set_flags(ctx->kaes, crypto_ahash_get_flags(tfm) &
2095 CRYPTO_TFM_REQ_MASK);
2096 ret = crypto_cipher_setkey(ctx->kaes, key, len);
2097 if (ret)
2098 return ret;
2099
2100 crypto_cipher_encrypt_one(ctx->kaes, (u8 *)key_tmp + 2 * AES_BLOCK_SIZE,
2101 "\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1");
2102 crypto_cipher_encrypt_one(ctx->kaes, (u8 *)key_tmp,
2103 "\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2");
2104 crypto_cipher_encrypt_one(ctx->kaes, (u8 *)key_tmp + AES_BLOCK_SIZE,
2105 "\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3");
2106 for (i = 0; i < 3 * AES_BLOCK_SIZE / sizeof(u32); i++)
2107 ctx->base.ipad.word[i] = swab32(key_tmp[i]);
2108
2109 crypto_cipher_clear_flags(ctx->kaes, CRYPTO_TFM_REQ_MASK);
2110 crypto_cipher_set_flags(ctx->kaes, crypto_ahash_get_flags(tfm) &
2111 CRYPTO_TFM_REQ_MASK);
2112 ret = crypto_cipher_setkey(ctx->kaes,
2113 (u8 *)key_tmp + 2 * AES_BLOCK_SIZE,
2114 AES_MIN_KEY_SIZE);
2115 if (ret)
2116 return ret;
2117
2118 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC128;
2119 ctx->key_sz = AES_MIN_KEY_SIZE + 2 * AES_BLOCK_SIZE;
2120 ctx->cbcmac = false;
2121
2122 memzero_explicit(&aes, sizeof(aes));
2123 return 0;
2124 }
2125
safexcel_xcbcmac_cra_init(struct crypto_tfm * tfm)2126 static int safexcel_xcbcmac_cra_init(struct crypto_tfm *tfm)
2127 {
2128 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
2129
2130 safexcel_ahash_cra_init(tfm);
2131 ctx->kaes = crypto_alloc_cipher("aes", 0, 0);
2132 return PTR_ERR_OR_ZERO(ctx->kaes);
2133 }
2134
safexcel_xcbcmac_cra_exit(struct crypto_tfm * tfm)2135 static void safexcel_xcbcmac_cra_exit(struct crypto_tfm *tfm)
2136 {
2137 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
2138
2139 crypto_free_cipher(ctx->kaes);
2140 safexcel_ahash_cra_exit(tfm);
2141 }
2142
2143 struct safexcel_alg_template safexcel_alg_xcbcmac = {
2144 .type = SAFEXCEL_ALG_TYPE_AHASH,
2145 .algo_mask = 0,
2146 .alg.ahash = {
2147 .init = safexcel_cbcmac_init,
2148 .update = safexcel_ahash_update,
2149 .final = safexcel_ahash_final,
2150 .finup = safexcel_ahash_finup,
2151 .digest = safexcel_cbcmac_digest,
2152 .setkey = safexcel_xcbcmac_setkey,
2153 .export = safexcel_ahash_export,
2154 .import = safexcel_ahash_import,
2155 .halg = {
2156 .digestsize = AES_BLOCK_SIZE,
2157 .statesize = sizeof(struct safexcel_ahash_export_state),
2158 .base = {
2159 .cra_name = "xcbc(aes)",
2160 .cra_driver_name = "safexcel-xcbc-aes",
2161 .cra_priority = SAFEXCEL_CRA_PRIORITY,
2162 .cra_flags = CRYPTO_ALG_ASYNC |
2163 CRYPTO_ALG_ALLOCATES_MEMORY |
2164 CRYPTO_ALG_KERN_DRIVER_ONLY,
2165 .cra_blocksize = AES_BLOCK_SIZE,
2166 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2167 .cra_init = safexcel_xcbcmac_cra_init,
2168 .cra_exit = safexcel_xcbcmac_cra_exit,
2169 .cra_module = THIS_MODULE,
2170 },
2171 },
2172 },
2173 };
2174
safexcel_cmac_setkey(struct crypto_ahash * tfm,const u8 * key,unsigned int len)2175 static int safexcel_cmac_setkey(struct crypto_ahash *tfm, const u8 *key,
2176 unsigned int len)
2177 {
2178 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
2179 struct crypto_aes_ctx aes;
2180 __be64 consts[4];
2181 u64 _const[2];
2182 u8 msb_mask, gfmask;
2183 int ret, i;
2184
2185 ret = aes_expandkey(&aes, key, len);
2186 if (ret)
2187 return ret;
2188
2189 for (i = 0; i < len / sizeof(u32); i++)
2190 ctx->base.ipad.word[i + 8] = swab32(aes.key_enc[i]);
2191
2192 /* precompute the CMAC key material */
2193 crypto_cipher_clear_flags(ctx->kaes, CRYPTO_TFM_REQ_MASK);
2194 crypto_cipher_set_flags(ctx->kaes, crypto_ahash_get_flags(tfm) &
2195 CRYPTO_TFM_REQ_MASK);
2196 ret = crypto_cipher_setkey(ctx->kaes, key, len);
2197 if (ret)
2198 return ret;
2199
2200 /* code below borrowed from crypto/cmac.c */
2201 /* encrypt the zero block */
2202 memset(consts, 0, AES_BLOCK_SIZE);
2203 crypto_cipher_encrypt_one(ctx->kaes, (u8 *)consts, (u8 *)consts);
2204
2205 gfmask = 0x87;
2206 _const[0] = be64_to_cpu(consts[1]);
2207 _const[1] = be64_to_cpu(consts[0]);
2208
2209 /* gf(2^128) multiply zero-ciphertext with u and u^2 */
2210 for (i = 0; i < 4; i += 2) {
2211 msb_mask = ((s64)_const[1] >> 63) & gfmask;
2212 _const[1] = (_const[1] << 1) | (_const[0] >> 63);
2213 _const[0] = (_const[0] << 1) ^ msb_mask;
2214
2215 consts[i + 0] = cpu_to_be64(_const[1]);
2216 consts[i + 1] = cpu_to_be64(_const[0]);
2217 }
2218 /* end of code borrowed from crypto/cmac.c */
2219
2220 for (i = 0; i < 2 * AES_BLOCK_SIZE / sizeof(u32); i++)
2221 ctx->base.ipad.be[i] = cpu_to_be32(((u32 *)consts)[i]);
2222
2223 if (len == AES_KEYSIZE_192) {
2224 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC192;
2225 ctx->key_sz = AES_MAX_KEY_SIZE + 2 * AES_BLOCK_SIZE;
2226 } else if (len == AES_KEYSIZE_256) {
2227 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC256;
2228 ctx->key_sz = AES_MAX_KEY_SIZE + 2 * AES_BLOCK_SIZE;
2229 } else {
2230 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC128;
2231 ctx->key_sz = AES_MIN_KEY_SIZE + 2 * AES_BLOCK_SIZE;
2232 }
2233 ctx->cbcmac = false;
2234
2235 memzero_explicit(&aes, sizeof(aes));
2236 return 0;
2237 }
2238
2239 struct safexcel_alg_template safexcel_alg_cmac = {
2240 .type = SAFEXCEL_ALG_TYPE_AHASH,
2241 .algo_mask = 0,
2242 .alg.ahash = {
2243 .init = safexcel_cbcmac_init,
2244 .update = safexcel_ahash_update,
2245 .final = safexcel_ahash_final,
2246 .finup = safexcel_ahash_finup,
2247 .digest = safexcel_cbcmac_digest,
2248 .setkey = safexcel_cmac_setkey,
2249 .export = safexcel_ahash_export,
2250 .import = safexcel_ahash_import,
2251 .halg = {
2252 .digestsize = AES_BLOCK_SIZE,
2253 .statesize = sizeof(struct safexcel_ahash_export_state),
2254 .base = {
2255 .cra_name = "cmac(aes)",
2256 .cra_driver_name = "safexcel-cmac-aes",
2257 .cra_priority = SAFEXCEL_CRA_PRIORITY,
2258 .cra_flags = CRYPTO_ALG_ASYNC |
2259 CRYPTO_ALG_ALLOCATES_MEMORY |
2260 CRYPTO_ALG_KERN_DRIVER_ONLY,
2261 .cra_blocksize = AES_BLOCK_SIZE,
2262 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2263 .cra_init = safexcel_xcbcmac_cra_init,
2264 .cra_exit = safexcel_xcbcmac_cra_exit,
2265 .cra_module = THIS_MODULE,
2266 },
2267 },
2268 },
2269 };
2270
safexcel_sm3_init(struct ahash_request * areq)2271 static int safexcel_sm3_init(struct ahash_request *areq)
2272 {
2273 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
2274 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
2275
2276 memset(req, 0, sizeof(*req));
2277
2278 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SM3;
2279 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
2280 req->state_sz = SM3_DIGEST_SIZE;
2281 req->digest_sz = SM3_DIGEST_SIZE;
2282 req->block_sz = SM3_BLOCK_SIZE;
2283
2284 return 0;
2285 }
2286
safexcel_sm3_digest(struct ahash_request * areq)2287 static int safexcel_sm3_digest(struct ahash_request *areq)
2288 {
2289 int ret = safexcel_sm3_init(areq);
2290
2291 if (ret)
2292 return ret;
2293
2294 return safexcel_ahash_finup(areq);
2295 }
2296
2297 struct safexcel_alg_template safexcel_alg_sm3 = {
2298 .type = SAFEXCEL_ALG_TYPE_AHASH,
2299 .algo_mask = SAFEXCEL_ALG_SM3,
2300 .alg.ahash = {
2301 .init = safexcel_sm3_init,
2302 .update = safexcel_ahash_update,
2303 .final = safexcel_ahash_final,
2304 .finup = safexcel_ahash_finup,
2305 .digest = safexcel_sm3_digest,
2306 .export = safexcel_ahash_export,
2307 .import = safexcel_ahash_import,
2308 .halg = {
2309 .digestsize = SM3_DIGEST_SIZE,
2310 .statesize = sizeof(struct safexcel_ahash_export_state),
2311 .base = {
2312 .cra_name = "sm3",
2313 .cra_driver_name = "safexcel-sm3",
2314 .cra_priority = SAFEXCEL_CRA_PRIORITY,
2315 .cra_flags = CRYPTO_ALG_ASYNC |
2316 CRYPTO_ALG_ALLOCATES_MEMORY |
2317 CRYPTO_ALG_KERN_DRIVER_ONLY,
2318 .cra_blocksize = SM3_BLOCK_SIZE,
2319 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2320 .cra_init = safexcel_ahash_cra_init,
2321 .cra_exit = safexcel_ahash_cra_exit,
2322 .cra_module = THIS_MODULE,
2323 },
2324 },
2325 },
2326 };
2327
safexcel_hmac_sm3_setkey(struct crypto_ahash * tfm,const u8 * key,unsigned int keylen)2328 static int safexcel_hmac_sm3_setkey(struct crypto_ahash *tfm, const u8 *key,
2329 unsigned int keylen)
2330 {
2331 return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sm3",
2332 SM3_DIGEST_SIZE);
2333 }
2334
safexcel_hmac_sm3_init(struct ahash_request * areq)2335 static int safexcel_hmac_sm3_init(struct ahash_request *areq)
2336 {
2337 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
2338 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
2339
2340 memset(req, 0, sizeof(*req));
2341
2342 /* Start from ipad precompute */
2343 memcpy(req->state, &ctx->base.ipad, SM3_DIGEST_SIZE);
2344 /* Already processed the key^ipad part now! */
2345 req->len = SM3_BLOCK_SIZE;
2346 req->processed = SM3_BLOCK_SIZE;
2347
2348 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SM3;
2349 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
2350 req->state_sz = SM3_DIGEST_SIZE;
2351 req->digest_sz = SM3_DIGEST_SIZE;
2352 req->block_sz = SM3_BLOCK_SIZE;
2353 req->hmac = true;
2354
2355 return 0;
2356 }
2357
safexcel_hmac_sm3_digest(struct ahash_request * areq)2358 static int safexcel_hmac_sm3_digest(struct ahash_request *areq)
2359 {
2360 int ret = safexcel_hmac_sm3_init(areq);
2361
2362 if (ret)
2363 return ret;
2364
2365 return safexcel_ahash_finup(areq);
2366 }
2367
2368 struct safexcel_alg_template safexcel_alg_hmac_sm3 = {
2369 .type = SAFEXCEL_ALG_TYPE_AHASH,
2370 .algo_mask = SAFEXCEL_ALG_SM3,
2371 .alg.ahash = {
2372 .init = safexcel_hmac_sm3_init,
2373 .update = safexcel_ahash_update,
2374 .final = safexcel_ahash_final,
2375 .finup = safexcel_ahash_finup,
2376 .digest = safexcel_hmac_sm3_digest,
2377 .setkey = safexcel_hmac_sm3_setkey,
2378 .export = safexcel_ahash_export,
2379 .import = safexcel_ahash_import,
2380 .halg = {
2381 .digestsize = SM3_DIGEST_SIZE,
2382 .statesize = sizeof(struct safexcel_ahash_export_state),
2383 .base = {
2384 .cra_name = "hmac(sm3)",
2385 .cra_driver_name = "safexcel-hmac-sm3",
2386 .cra_priority = SAFEXCEL_CRA_PRIORITY,
2387 .cra_flags = CRYPTO_ALG_ASYNC |
2388 CRYPTO_ALG_ALLOCATES_MEMORY |
2389 CRYPTO_ALG_KERN_DRIVER_ONLY,
2390 .cra_blocksize = SM3_BLOCK_SIZE,
2391 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2392 .cra_init = safexcel_ahash_cra_init,
2393 .cra_exit = safexcel_ahash_cra_exit,
2394 .cra_module = THIS_MODULE,
2395 },
2396 },
2397 },
2398 };
2399
safexcel_sha3_224_init(struct ahash_request * areq)2400 static int safexcel_sha3_224_init(struct ahash_request *areq)
2401 {
2402 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2403 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2404 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
2405
2406 memset(req, 0, sizeof(*req));
2407
2408 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_224;
2409 req->digest = CONTEXT_CONTROL_DIGEST_INITIAL;
2410 req->state_sz = SHA3_224_DIGEST_SIZE;
2411 req->digest_sz = SHA3_224_DIGEST_SIZE;
2412 req->block_sz = SHA3_224_BLOCK_SIZE;
2413 ctx->do_fallback = false;
2414 ctx->fb_init_done = false;
2415 return 0;
2416 }
2417
safexcel_sha3_fbcheck(struct ahash_request * req)2418 static int safexcel_sha3_fbcheck(struct ahash_request *req)
2419 {
2420 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2421 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2422 struct ahash_request *subreq = ahash_request_ctx(req);
2423 int ret = 0;
2424
2425 if (ctx->do_fallback) {
2426 ahash_request_set_tfm(subreq, ctx->fback);
2427 ahash_request_set_callback(subreq, req->base.flags,
2428 req->base.complete, req->base.data);
2429 ahash_request_set_crypt(subreq, req->src, req->result,
2430 req->nbytes);
2431 if (!ctx->fb_init_done) {
2432 if (ctx->fb_do_setkey) {
2433 /* Set fallback cipher HMAC key */
2434 u8 key[SHA3_224_BLOCK_SIZE];
2435
2436 memcpy(key, &ctx->base.ipad,
2437 crypto_ahash_blocksize(ctx->fback) / 2);
2438 memcpy(key +
2439 crypto_ahash_blocksize(ctx->fback) / 2,
2440 &ctx->base.opad,
2441 crypto_ahash_blocksize(ctx->fback) / 2);
2442 ret = crypto_ahash_setkey(ctx->fback, key,
2443 crypto_ahash_blocksize(ctx->fback));
2444 memzero_explicit(key,
2445 crypto_ahash_blocksize(ctx->fback));
2446 ctx->fb_do_setkey = false;
2447 }
2448 ret = ret ?: crypto_ahash_init(subreq);
2449 ctx->fb_init_done = true;
2450 }
2451 }
2452 return ret;
2453 }
2454
safexcel_sha3_update(struct ahash_request * req)2455 static int safexcel_sha3_update(struct ahash_request *req)
2456 {
2457 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2458 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2459 struct ahash_request *subreq = ahash_request_ctx(req);
2460
2461 ctx->do_fallback = true;
2462 return safexcel_sha3_fbcheck(req) ?: crypto_ahash_update(subreq);
2463 }
2464
safexcel_sha3_final(struct ahash_request * req)2465 static int safexcel_sha3_final(struct ahash_request *req)
2466 {
2467 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2468 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2469 struct ahash_request *subreq = ahash_request_ctx(req);
2470
2471 ctx->do_fallback = true;
2472 return safexcel_sha3_fbcheck(req) ?: crypto_ahash_final(subreq);
2473 }
2474
safexcel_sha3_finup(struct ahash_request * req)2475 static int safexcel_sha3_finup(struct ahash_request *req)
2476 {
2477 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2478 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2479 struct ahash_request *subreq = ahash_request_ctx(req);
2480
2481 ctx->do_fallback |= !req->nbytes;
2482 if (ctx->do_fallback)
2483 /* Update or ex/import happened or len 0, cannot use the HW */
2484 return safexcel_sha3_fbcheck(req) ?:
2485 crypto_ahash_finup(subreq);
2486 else
2487 return safexcel_ahash_finup(req);
2488 }
2489
safexcel_sha3_digest_fallback(struct ahash_request * req)2490 static int safexcel_sha3_digest_fallback(struct ahash_request *req)
2491 {
2492 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2493 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2494 struct ahash_request *subreq = ahash_request_ctx(req);
2495
2496 ctx->do_fallback = true;
2497 ctx->fb_init_done = false;
2498 return safexcel_sha3_fbcheck(req) ?: crypto_ahash_finup(subreq);
2499 }
2500
safexcel_sha3_224_digest(struct ahash_request * req)2501 static int safexcel_sha3_224_digest(struct ahash_request *req)
2502 {
2503 if (req->nbytes)
2504 return safexcel_sha3_224_init(req) ?: safexcel_ahash_finup(req);
2505
2506 /* HW cannot do zero length hash, use fallback instead */
2507 return safexcel_sha3_digest_fallback(req);
2508 }
2509
safexcel_sha3_export(struct ahash_request * req,void * out)2510 static int safexcel_sha3_export(struct ahash_request *req, void *out)
2511 {
2512 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2513 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2514 struct ahash_request *subreq = ahash_request_ctx(req);
2515
2516 ctx->do_fallback = true;
2517 return safexcel_sha3_fbcheck(req) ?: crypto_ahash_export(subreq, out);
2518 }
2519
safexcel_sha3_import(struct ahash_request * req,const void * in)2520 static int safexcel_sha3_import(struct ahash_request *req, const void *in)
2521 {
2522 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2523 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2524 struct ahash_request *subreq = ahash_request_ctx(req);
2525
2526 ctx->do_fallback = true;
2527 return safexcel_sha3_fbcheck(req) ?: crypto_ahash_import(subreq, in);
2528 // return safexcel_ahash_import(req, in);
2529 }
2530
safexcel_sha3_cra_init(struct crypto_tfm * tfm)2531 static int safexcel_sha3_cra_init(struct crypto_tfm *tfm)
2532 {
2533 struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
2534 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
2535
2536 safexcel_ahash_cra_init(tfm);
2537
2538 /* Allocate fallback implementation */
2539 ctx->fback = crypto_alloc_ahash(crypto_tfm_alg_name(tfm), 0,
2540 CRYPTO_ALG_ASYNC |
2541 CRYPTO_ALG_NEED_FALLBACK);
2542 if (IS_ERR(ctx->fback))
2543 return PTR_ERR(ctx->fback);
2544
2545 /* Update statesize from fallback algorithm! */
2546 crypto_hash_alg_common(ahash)->statesize =
2547 crypto_ahash_statesize(ctx->fback);
2548 crypto_ahash_set_reqsize(ahash, max(sizeof(struct safexcel_ahash_req),
2549 sizeof(struct ahash_request) +
2550 crypto_ahash_reqsize(ctx->fback)));
2551 return 0;
2552 }
2553
safexcel_sha3_cra_exit(struct crypto_tfm * tfm)2554 static void safexcel_sha3_cra_exit(struct crypto_tfm *tfm)
2555 {
2556 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
2557
2558 crypto_free_ahash(ctx->fback);
2559 safexcel_ahash_cra_exit(tfm);
2560 }
2561
2562 struct safexcel_alg_template safexcel_alg_sha3_224 = {
2563 .type = SAFEXCEL_ALG_TYPE_AHASH,
2564 .algo_mask = SAFEXCEL_ALG_SHA3,
2565 .alg.ahash = {
2566 .init = safexcel_sha3_224_init,
2567 .update = safexcel_sha3_update,
2568 .final = safexcel_sha3_final,
2569 .finup = safexcel_sha3_finup,
2570 .digest = safexcel_sha3_224_digest,
2571 .export = safexcel_sha3_export,
2572 .import = safexcel_sha3_import,
2573 .halg = {
2574 .digestsize = SHA3_224_DIGEST_SIZE,
2575 .statesize = sizeof(struct safexcel_ahash_export_state),
2576 .base = {
2577 .cra_name = "sha3-224",
2578 .cra_driver_name = "safexcel-sha3-224",
2579 .cra_priority = SAFEXCEL_CRA_PRIORITY,
2580 .cra_flags = CRYPTO_ALG_ASYNC |
2581 CRYPTO_ALG_KERN_DRIVER_ONLY |
2582 CRYPTO_ALG_NEED_FALLBACK,
2583 .cra_blocksize = SHA3_224_BLOCK_SIZE,
2584 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2585 .cra_init = safexcel_sha3_cra_init,
2586 .cra_exit = safexcel_sha3_cra_exit,
2587 .cra_module = THIS_MODULE,
2588 },
2589 },
2590 },
2591 };
2592
safexcel_sha3_256_init(struct ahash_request * areq)2593 static int safexcel_sha3_256_init(struct ahash_request *areq)
2594 {
2595 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2596 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2597 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
2598
2599 memset(req, 0, sizeof(*req));
2600
2601 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_256;
2602 req->digest = CONTEXT_CONTROL_DIGEST_INITIAL;
2603 req->state_sz = SHA3_256_DIGEST_SIZE;
2604 req->digest_sz = SHA3_256_DIGEST_SIZE;
2605 req->block_sz = SHA3_256_BLOCK_SIZE;
2606 ctx->do_fallback = false;
2607 ctx->fb_init_done = false;
2608 return 0;
2609 }
2610
safexcel_sha3_256_digest(struct ahash_request * req)2611 static int safexcel_sha3_256_digest(struct ahash_request *req)
2612 {
2613 if (req->nbytes)
2614 return safexcel_sha3_256_init(req) ?: safexcel_ahash_finup(req);
2615
2616 /* HW cannot do zero length hash, use fallback instead */
2617 return safexcel_sha3_digest_fallback(req);
2618 }
2619
2620 struct safexcel_alg_template safexcel_alg_sha3_256 = {
2621 .type = SAFEXCEL_ALG_TYPE_AHASH,
2622 .algo_mask = SAFEXCEL_ALG_SHA3,
2623 .alg.ahash = {
2624 .init = safexcel_sha3_256_init,
2625 .update = safexcel_sha3_update,
2626 .final = safexcel_sha3_final,
2627 .finup = safexcel_sha3_finup,
2628 .digest = safexcel_sha3_256_digest,
2629 .export = safexcel_sha3_export,
2630 .import = safexcel_sha3_import,
2631 .halg = {
2632 .digestsize = SHA3_256_DIGEST_SIZE,
2633 .statesize = sizeof(struct safexcel_ahash_export_state),
2634 .base = {
2635 .cra_name = "sha3-256",
2636 .cra_driver_name = "safexcel-sha3-256",
2637 .cra_priority = SAFEXCEL_CRA_PRIORITY,
2638 .cra_flags = CRYPTO_ALG_ASYNC |
2639 CRYPTO_ALG_KERN_DRIVER_ONLY |
2640 CRYPTO_ALG_NEED_FALLBACK,
2641 .cra_blocksize = SHA3_256_BLOCK_SIZE,
2642 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2643 .cra_init = safexcel_sha3_cra_init,
2644 .cra_exit = safexcel_sha3_cra_exit,
2645 .cra_module = THIS_MODULE,
2646 },
2647 },
2648 },
2649 };
2650
safexcel_sha3_384_init(struct ahash_request * areq)2651 static int safexcel_sha3_384_init(struct ahash_request *areq)
2652 {
2653 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2654 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2655 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
2656
2657 memset(req, 0, sizeof(*req));
2658
2659 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_384;
2660 req->digest = CONTEXT_CONTROL_DIGEST_INITIAL;
2661 req->state_sz = SHA3_384_DIGEST_SIZE;
2662 req->digest_sz = SHA3_384_DIGEST_SIZE;
2663 req->block_sz = SHA3_384_BLOCK_SIZE;
2664 ctx->do_fallback = false;
2665 ctx->fb_init_done = false;
2666 return 0;
2667 }
2668
safexcel_sha3_384_digest(struct ahash_request * req)2669 static int safexcel_sha3_384_digest(struct ahash_request *req)
2670 {
2671 if (req->nbytes)
2672 return safexcel_sha3_384_init(req) ?: safexcel_ahash_finup(req);
2673
2674 /* HW cannot do zero length hash, use fallback instead */
2675 return safexcel_sha3_digest_fallback(req);
2676 }
2677
2678 struct safexcel_alg_template safexcel_alg_sha3_384 = {
2679 .type = SAFEXCEL_ALG_TYPE_AHASH,
2680 .algo_mask = SAFEXCEL_ALG_SHA3,
2681 .alg.ahash = {
2682 .init = safexcel_sha3_384_init,
2683 .update = safexcel_sha3_update,
2684 .final = safexcel_sha3_final,
2685 .finup = safexcel_sha3_finup,
2686 .digest = safexcel_sha3_384_digest,
2687 .export = safexcel_sha3_export,
2688 .import = safexcel_sha3_import,
2689 .halg = {
2690 .digestsize = SHA3_384_DIGEST_SIZE,
2691 .statesize = sizeof(struct safexcel_ahash_export_state),
2692 .base = {
2693 .cra_name = "sha3-384",
2694 .cra_driver_name = "safexcel-sha3-384",
2695 .cra_priority = SAFEXCEL_CRA_PRIORITY,
2696 .cra_flags = CRYPTO_ALG_ASYNC |
2697 CRYPTO_ALG_KERN_DRIVER_ONLY |
2698 CRYPTO_ALG_NEED_FALLBACK,
2699 .cra_blocksize = SHA3_384_BLOCK_SIZE,
2700 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2701 .cra_init = safexcel_sha3_cra_init,
2702 .cra_exit = safexcel_sha3_cra_exit,
2703 .cra_module = THIS_MODULE,
2704 },
2705 },
2706 },
2707 };
2708
safexcel_sha3_512_init(struct ahash_request * areq)2709 static int safexcel_sha3_512_init(struct ahash_request *areq)
2710 {
2711 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2712 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2713 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
2714
2715 memset(req, 0, sizeof(*req));
2716
2717 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_512;
2718 req->digest = CONTEXT_CONTROL_DIGEST_INITIAL;
2719 req->state_sz = SHA3_512_DIGEST_SIZE;
2720 req->digest_sz = SHA3_512_DIGEST_SIZE;
2721 req->block_sz = SHA3_512_BLOCK_SIZE;
2722 ctx->do_fallback = false;
2723 ctx->fb_init_done = false;
2724 return 0;
2725 }
2726
safexcel_sha3_512_digest(struct ahash_request * req)2727 static int safexcel_sha3_512_digest(struct ahash_request *req)
2728 {
2729 if (req->nbytes)
2730 return safexcel_sha3_512_init(req) ?: safexcel_ahash_finup(req);
2731
2732 /* HW cannot do zero length hash, use fallback instead */
2733 return safexcel_sha3_digest_fallback(req);
2734 }
2735
2736 struct safexcel_alg_template safexcel_alg_sha3_512 = {
2737 .type = SAFEXCEL_ALG_TYPE_AHASH,
2738 .algo_mask = SAFEXCEL_ALG_SHA3,
2739 .alg.ahash = {
2740 .init = safexcel_sha3_512_init,
2741 .update = safexcel_sha3_update,
2742 .final = safexcel_sha3_final,
2743 .finup = safexcel_sha3_finup,
2744 .digest = safexcel_sha3_512_digest,
2745 .export = safexcel_sha3_export,
2746 .import = safexcel_sha3_import,
2747 .halg = {
2748 .digestsize = SHA3_512_DIGEST_SIZE,
2749 .statesize = sizeof(struct safexcel_ahash_export_state),
2750 .base = {
2751 .cra_name = "sha3-512",
2752 .cra_driver_name = "safexcel-sha3-512",
2753 .cra_priority = SAFEXCEL_CRA_PRIORITY,
2754 .cra_flags = CRYPTO_ALG_ASYNC |
2755 CRYPTO_ALG_KERN_DRIVER_ONLY |
2756 CRYPTO_ALG_NEED_FALLBACK,
2757 .cra_blocksize = SHA3_512_BLOCK_SIZE,
2758 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2759 .cra_init = safexcel_sha3_cra_init,
2760 .cra_exit = safexcel_sha3_cra_exit,
2761 .cra_module = THIS_MODULE,
2762 },
2763 },
2764 },
2765 };
2766
safexcel_hmac_sha3_cra_init(struct crypto_tfm * tfm,const char * alg)2767 static int safexcel_hmac_sha3_cra_init(struct crypto_tfm *tfm, const char *alg)
2768 {
2769 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
2770 int ret;
2771
2772 ret = safexcel_sha3_cra_init(tfm);
2773 if (ret)
2774 return ret;
2775
2776 /* Allocate precalc basic digest implementation */
2777 ctx->shpre = crypto_alloc_shash(alg, 0, CRYPTO_ALG_NEED_FALLBACK);
2778 if (IS_ERR(ctx->shpre))
2779 return PTR_ERR(ctx->shpre);
2780
2781 ctx->shdesc = kmalloc(sizeof(*ctx->shdesc) +
2782 crypto_shash_descsize(ctx->shpre), GFP_KERNEL);
2783 if (!ctx->shdesc) {
2784 crypto_free_shash(ctx->shpre);
2785 return -ENOMEM;
2786 }
2787 ctx->shdesc->tfm = ctx->shpre;
2788 return 0;
2789 }
2790
safexcel_hmac_sha3_cra_exit(struct crypto_tfm * tfm)2791 static void safexcel_hmac_sha3_cra_exit(struct crypto_tfm *tfm)
2792 {
2793 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
2794
2795 crypto_free_ahash(ctx->fback);
2796 crypto_free_shash(ctx->shpre);
2797 kfree(ctx->shdesc);
2798 safexcel_ahash_cra_exit(tfm);
2799 }
2800
safexcel_hmac_sha3_setkey(struct crypto_ahash * tfm,const u8 * key,unsigned int keylen)2801 static int safexcel_hmac_sha3_setkey(struct crypto_ahash *tfm, const u8 *key,
2802 unsigned int keylen)
2803 {
2804 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2805 int ret = 0;
2806
2807 if (keylen > crypto_ahash_blocksize(tfm)) {
2808 /*
2809 * If the key is larger than the blocksize, then hash it
2810 * first using our fallback cipher
2811 */
2812 ret = crypto_shash_digest(ctx->shdesc, key, keylen,
2813 ctx->base.ipad.byte);
2814 keylen = crypto_shash_digestsize(ctx->shpre);
2815
2816 /*
2817 * If the digest is larger than half the blocksize, we need to
2818 * move the rest to opad due to the way our HMAC infra works.
2819 */
2820 if (keylen > crypto_ahash_blocksize(tfm) / 2)
2821 /* Buffers overlap, need to use memmove iso memcpy! */
2822 memmove(&ctx->base.opad,
2823 ctx->base.ipad.byte +
2824 crypto_ahash_blocksize(tfm) / 2,
2825 keylen - crypto_ahash_blocksize(tfm) / 2);
2826 } else {
2827 /*
2828 * Copy the key to our ipad & opad buffers
2829 * Note that ipad and opad each contain one half of the key,
2830 * to match the existing HMAC driver infrastructure.
2831 */
2832 if (keylen <= crypto_ahash_blocksize(tfm) / 2) {
2833 memcpy(&ctx->base.ipad, key, keylen);
2834 } else {
2835 memcpy(&ctx->base.ipad, key,
2836 crypto_ahash_blocksize(tfm) / 2);
2837 memcpy(&ctx->base.opad,
2838 key + crypto_ahash_blocksize(tfm) / 2,
2839 keylen - crypto_ahash_blocksize(tfm) / 2);
2840 }
2841 }
2842
2843 /* Pad key with zeroes */
2844 if (keylen <= crypto_ahash_blocksize(tfm) / 2) {
2845 memset(ctx->base.ipad.byte + keylen, 0,
2846 crypto_ahash_blocksize(tfm) / 2 - keylen);
2847 memset(&ctx->base.opad, 0, crypto_ahash_blocksize(tfm) / 2);
2848 } else {
2849 memset(ctx->base.opad.byte + keylen -
2850 crypto_ahash_blocksize(tfm) / 2, 0,
2851 crypto_ahash_blocksize(tfm) - keylen);
2852 }
2853
2854 /* If doing fallback, still need to set the new key! */
2855 ctx->fb_do_setkey = true;
2856 return ret;
2857 }
2858
safexcel_hmac_sha3_224_init(struct ahash_request * areq)2859 static int safexcel_hmac_sha3_224_init(struct ahash_request *areq)
2860 {
2861 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2862 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2863 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
2864
2865 memset(req, 0, sizeof(*req));
2866
2867 /* Copy (half of) the key */
2868 memcpy(req->state, &ctx->base.ipad, SHA3_224_BLOCK_SIZE / 2);
2869 /* Start of HMAC should have len == processed == blocksize */
2870 req->len = SHA3_224_BLOCK_SIZE;
2871 req->processed = SHA3_224_BLOCK_SIZE;
2872 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_224;
2873 req->digest = CONTEXT_CONTROL_DIGEST_HMAC;
2874 req->state_sz = SHA3_224_BLOCK_SIZE / 2;
2875 req->digest_sz = SHA3_224_DIGEST_SIZE;
2876 req->block_sz = SHA3_224_BLOCK_SIZE;
2877 req->hmac = true;
2878 ctx->do_fallback = false;
2879 ctx->fb_init_done = false;
2880 return 0;
2881 }
2882
safexcel_hmac_sha3_224_digest(struct ahash_request * req)2883 static int safexcel_hmac_sha3_224_digest(struct ahash_request *req)
2884 {
2885 if (req->nbytes)
2886 return safexcel_hmac_sha3_224_init(req) ?:
2887 safexcel_ahash_finup(req);
2888
2889 /* HW cannot do zero length HMAC, use fallback instead */
2890 return safexcel_sha3_digest_fallback(req);
2891 }
2892
safexcel_hmac_sha3_224_cra_init(struct crypto_tfm * tfm)2893 static int safexcel_hmac_sha3_224_cra_init(struct crypto_tfm *tfm)
2894 {
2895 return safexcel_hmac_sha3_cra_init(tfm, "sha3-224");
2896 }
2897
2898 struct safexcel_alg_template safexcel_alg_hmac_sha3_224 = {
2899 .type = SAFEXCEL_ALG_TYPE_AHASH,
2900 .algo_mask = SAFEXCEL_ALG_SHA3,
2901 .alg.ahash = {
2902 .init = safexcel_hmac_sha3_224_init,
2903 .update = safexcel_sha3_update,
2904 .final = safexcel_sha3_final,
2905 .finup = safexcel_sha3_finup,
2906 .digest = safexcel_hmac_sha3_224_digest,
2907 .setkey = safexcel_hmac_sha3_setkey,
2908 .export = safexcel_sha3_export,
2909 .import = safexcel_sha3_import,
2910 .halg = {
2911 .digestsize = SHA3_224_DIGEST_SIZE,
2912 .statesize = sizeof(struct safexcel_ahash_export_state),
2913 .base = {
2914 .cra_name = "hmac(sha3-224)",
2915 .cra_driver_name = "safexcel-hmac-sha3-224",
2916 .cra_priority = SAFEXCEL_CRA_PRIORITY,
2917 .cra_flags = CRYPTO_ALG_ASYNC |
2918 CRYPTO_ALG_KERN_DRIVER_ONLY |
2919 CRYPTO_ALG_NEED_FALLBACK,
2920 .cra_blocksize = SHA3_224_BLOCK_SIZE,
2921 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2922 .cra_init = safexcel_hmac_sha3_224_cra_init,
2923 .cra_exit = safexcel_hmac_sha3_cra_exit,
2924 .cra_module = THIS_MODULE,
2925 },
2926 },
2927 },
2928 };
2929
safexcel_hmac_sha3_256_init(struct ahash_request * areq)2930 static int safexcel_hmac_sha3_256_init(struct ahash_request *areq)
2931 {
2932 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2933 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2934 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
2935
2936 memset(req, 0, sizeof(*req));
2937
2938 /* Copy (half of) the key */
2939 memcpy(req->state, &ctx->base.ipad, SHA3_256_BLOCK_SIZE / 2);
2940 /* Start of HMAC should have len == processed == blocksize */
2941 req->len = SHA3_256_BLOCK_SIZE;
2942 req->processed = SHA3_256_BLOCK_SIZE;
2943 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_256;
2944 req->digest = CONTEXT_CONTROL_DIGEST_HMAC;
2945 req->state_sz = SHA3_256_BLOCK_SIZE / 2;
2946 req->digest_sz = SHA3_256_DIGEST_SIZE;
2947 req->block_sz = SHA3_256_BLOCK_SIZE;
2948 req->hmac = true;
2949 ctx->do_fallback = false;
2950 ctx->fb_init_done = false;
2951 return 0;
2952 }
2953
safexcel_hmac_sha3_256_digest(struct ahash_request * req)2954 static int safexcel_hmac_sha3_256_digest(struct ahash_request *req)
2955 {
2956 if (req->nbytes)
2957 return safexcel_hmac_sha3_256_init(req) ?:
2958 safexcel_ahash_finup(req);
2959
2960 /* HW cannot do zero length HMAC, use fallback instead */
2961 return safexcel_sha3_digest_fallback(req);
2962 }
2963
safexcel_hmac_sha3_256_cra_init(struct crypto_tfm * tfm)2964 static int safexcel_hmac_sha3_256_cra_init(struct crypto_tfm *tfm)
2965 {
2966 return safexcel_hmac_sha3_cra_init(tfm, "sha3-256");
2967 }
2968
2969 struct safexcel_alg_template safexcel_alg_hmac_sha3_256 = {
2970 .type = SAFEXCEL_ALG_TYPE_AHASH,
2971 .algo_mask = SAFEXCEL_ALG_SHA3,
2972 .alg.ahash = {
2973 .init = safexcel_hmac_sha3_256_init,
2974 .update = safexcel_sha3_update,
2975 .final = safexcel_sha3_final,
2976 .finup = safexcel_sha3_finup,
2977 .digest = safexcel_hmac_sha3_256_digest,
2978 .setkey = safexcel_hmac_sha3_setkey,
2979 .export = safexcel_sha3_export,
2980 .import = safexcel_sha3_import,
2981 .halg = {
2982 .digestsize = SHA3_256_DIGEST_SIZE,
2983 .statesize = sizeof(struct safexcel_ahash_export_state),
2984 .base = {
2985 .cra_name = "hmac(sha3-256)",
2986 .cra_driver_name = "safexcel-hmac-sha3-256",
2987 .cra_priority = SAFEXCEL_CRA_PRIORITY,
2988 .cra_flags = CRYPTO_ALG_ASYNC |
2989 CRYPTO_ALG_KERN_DRIVER_ONLY |
2990 CRYPTO_ALG_NEED_FALLBACK,
2991 .cra_blocksize = SHA3_256_BLOCK_SIZE,
2992 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2993 .cra_init = safexcel_hmac_sha3_256_cra_init,
2994 .cra_exit = safexcel_hmac_sha3_cra_exit,
2995 .cra_module = THIS_MODULE,
2996 },
2997 },
2998 },
2999 };
3000
safexcel_hmac_sha3_384_init(struct ahash_request * areq)3001 static int safexcel_hmac_sha3_384_init(struct ahash_request *areq)
3002 {
3003 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
3004 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
3005 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
3006
3007 memset(req, 0, sizeof(*req));
3008
3009 /* Copy (half of) the key */
3010 memcpy(req->state, &ctx->base.ipad, SHA3_384_BLOCK_SIZE / 2);
3011 /* Start of HMAC should have len == processed == blocksize */
3012 req->len = SHA3_384_BLOCK_SIZE;
3013 req->processed = SHA3_384_BLOCK_SIZE;
3014 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_384;
3015 req->digest = CONTEXT_CONTROL_DIGEST_HMAC;
3016 req->state_sz = SHA3_384_BLOCK_SIZE / 2;
3017 req->digest_sz = SHA3_384_DIGEST_SIZE;
3018 req->block_sz = SHA3_384_BLOCK_SIZE;
3019 req->hmac = true;
3020 ctx->do_fallback = false;
3021 ctx->fb_init_done = false;
3022 return 0;
3023 }
3024
safexcel_hmac_sha3_384_digest(struct ahash_request * req)3025 static int safexcel_hmac_sha3_384_digest(struct ahash_request *req)
3026 {
3027 if (req->nbytes)
3028 return safexcel_hmac_sha3_384_init(req) ?:
3029 safexcel_ahash_finup(req);
3030
3031 /* HW cannot do zero length HMAC, use fallback instead */
3032 return safexcel_sha3_digest_fallback(req);
3033 }
3034
safexcel_hmac_sha3_384_cra_init(struct crypto_tfm * tfm)3035 static int safexcel_hmac_sha3_384_cra_init(struct crypto_tfm *tfm)
3036 {
3037 return safexcel_hmac_sha3_cra_init(tfm, "sha3-384");
3038 }
3039
3040 struct safexcel_alg_template safexcel_alg_hmac_sha3_384 = {
3041 .type = SAFEXCEL_ALG_TYPE_AHASH,
3042 .algo_mask = SAFEXCEL_ALG_SHA3,
3043 .alg.ahash = {
3044 .init = safexcel_hmac_sha3_384_init,
3045 .update = safexcel_sha3_update,
3046 .final = safexcel_sha3_final,
3047 .finup = safexcel_sha3_finup,
3048 .digest = safexcel_hmac_sha3_384_digest,
3049 .setkey = safexcel_hmac_sha3_setkey,
3050 .export = safexcel_sha3_export,
3051 .import = safexcel_sha3_import,
3052 .halg = {
3053 .digestsize = SHA3_384_DIGEST_SIZE,
3054 .statesize = sizeof(struct safexcel_ahash_export_state),
3055 .base = {
3056 .cra_name = "hmac(sha3-384)",
3057 .cra_driver_name = "safexcel-hmac-sha3-384",
3058 .cra_priority = SAFEXCEL_CRA_PRIORITY,
3059 .cra_flags = CRYPTO_ALG_ASYNC |
3060 CRYPTO_ALG_KERN_DRIVER_ONLY |
3061 CRYPTO_ALG_NEED_FALLBACK,
3062 .cra_blocksize = SHA3_384_BLOCK_SIZE,
3063 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
3064 .cra_init = safexcel_hmac_sha3_384_cra_init,
3065 .cra_exit = safexcel_hmac_sha3_cra_exit,
3066 .cra_module = THIS_MODULE,
3067 },
3068 },
3069 },
3070 };
3071
safexcel_hmac_sha3_512_init(struct ahash_request * areq)3072 static int safexcel_hmac_sha3_512_init(struct ahash_request *areq)
3073 {
3074 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
3075 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
3076 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
3077
3078 memset(req, 0, sizeof(*req));
3079
3080 /* Copy (half of) the key */
3081 memcpy(req->state, &ctx->base.ipad, SHA3_512_BLOCK_SIZE / 2);
3082 /* Start of HMAC should have len == processed == blocksize */
3083 req->len = SHA3_512_BLOCK_SIZE;
3084 req->processed = SHA3_512_BLOCK_SIZE;
3085 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_512;
3086 req->digest = CONTEXT_CONTROL_DIGEST_HMAC;
3087 req->state_sz = SHA3_512_BLOCK_SIZE / 2;
3088 req->digest_sz = SHA3_512_DIGEST_SIZE;
3089 req->block_sz = SHA3_512_BLOCK_SIZE;
3090 req->hmac = true;
3091 ctx->do_fallback = false;
3092 ctx->fb_init_done = false;
3093 return 0;
3094 }
3095
safexcel_hmac_sha3_512_digest(struct ahash_request * req)3096 static int safexcel_hmac_sha3_512_digest(struct ahash_request *req)
3097 {
3098 if (req->nbytes)
3099 return safexcel_hmac_sha3_512_init(req) ?:
3100 safexcel_ahash_finup(req);
3101
3102 /* HW cannot do zero length HMAC, use fallback instead */
3103 return safexcel_sha3_digest_fallback(req);
3104 }
3105
safexcel_hmac_sha3_512_cra_init(struct crypto_tfm * tfm)3106 static int safexcel_hmac_sha3_512_cra_init(struct crypto_tfm *tfm)
3107 {
3108 return safexcel_hmac_sha3_cra_init(tfm, "sha3-512");
3109 }
3110 struct safexcel_alg_template safexcel_alg_hmac_sha3_512 = {
3111 .type = SAFEXCEL_ALG_TYPE_AHASH,
3112 .algo_mask = SAFEXCEL_ALG_SHA3,
3113 .alg.ahash = {
3114 .init = safexcel_hmac_sha3_512_init,
3115 .update = safexcel_sha3_update,
3116 .final = safexcel_sha3_final,
3117 .finup = safexcel_sha3_finup,
3118 .digest = safexcel_hmac_sha3_512_digest,
3119 .setkey = safexcel_hmac_sha3_setkey,
3120 .export = safexcel_sha3_export,
3121 .import = safexcel_sha3_import,
3122 .halg = {
3123 .digestsize = SHA3_512_DIGEST_SIZE,
3124 .statesize = sizeof(struct safexcel_ahash_export_state),
3125 .base = {
3126 .cra_name = "hmac(sha3-512)",
3127 .cra_driver_name = "safexcel-hmac-sha3-512",
3128 .cra_priority = SAFEXCEL_CRA_PRIORITY,
3129 .cra_flags = CRYPTO_ALG_ASYNC |
3130 CRYPTO_ALG_KERN_DRIVER_ONLY |
3131 CRYPTO_ALG_NEED_FALLBACK,
3132 .cra_blocksize = SHA3_512_BLOCK_SIZE,
3133 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
3134 .cra_init = safexcel_hmac_sha3_512_cra_init,
3135 .cra_exit = safexcel_hmac_sha3_cra_exit,
3136 .cra_module = THIS_MODULE,
3137 },
3138 },
3139 },
3140 };
3141