1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * aes-ce-glue.c - wrapper code for ARMv8 AES
4 *
5 * Copyright (C) 2015 Linaro Ltd <ard.biesheuvel@linaro.org>
6 */
7
8 #include <asm/hwcap.h>
9 #include <asm/neon.h>
10 #include <asm/simd.h>
11 #include <asm/unaligned.h>
12 #include <crypto/aes.h>
13 #include <crypto/ctr.h>
14 #include <crypto/internal/simd.h>
15 #include <crypto/internal/skcipher.h>
16 #include <crypto/scatterwalk.h>
17 #include <linux/cpufeature.h>
18 #include <linux/module.h>
19 #include <crypto/xts.h>
20
21 MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS using ARMv8 Crypto Extensions");
22 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
23 MODULE_LICENSE("GPL v2");
24
25 /* defined in aes-ce-core.S */
26 asmlinkage u32 ce_aes_sub(u32 input);
27 asmlinkage void ce_aes_invert(void *dst, void *src);
28
29 asmlinkage void ce_aes_ecb_encrypt(u8 out[], u8 const in[], u32 const rk[],
30 int rounds, int blocks);
31 asmlinkage void ce_aes_ecb_decrypt(u8 out[], u8 const in[], u32 const rk[],
32 int rounds, int blocks);
33
34 asmlinkage void ce_aes_cbc_encrypt(u8 out[], u8 const in[], u32 const rk[],
35 int rounds, int blocks, u8 iv[]);
36 asmlinkage void ce_aes_cbc_decrypt(u8 out[], u8 const in[], u32 const rk[],
37 int rounds, int blocks, u8 iv[]);
38 asmlinkage void ce_aes_cbc_cts_encrypt(u8 out[], u8 const in[], u32 const rk[],
39 int rounds, int bytes, u8 const iv[]);
40 asmlinkage void ce_aes_cbc_cts_decrypt(u8 out[], u8 const in[], u32 const rk[],
41 int rounds, int bytes, u8 const iv[]);
42
43 asmlinkage void ce_aes_ctr_encrypt(u8 out[], u8 const in[], u32 const rk[],
44 int rounds, int blocks, u8 ctr[]);
45
46 asmlinkage void ce_aes_xts_encrypt(u8 out[], u8 const in[], u32 const rk1[],
47 int rounds, int bytes, u8 iv[],
48 u32 const rk2[], int first);
49 asmlinkage void ce_aes_xts_decrypt(u8 out[], u8 const in[], u32 const rk1[],
50 int rounds, int bytes, u8 iv[],
51 u32 const rk2[], int first);
52
53 struct aes_block {
54 u8 b[AES_BLOCK_SIZE];
55 };
56
num_rounds(struct crypto_aes_ctx * ctx)57 static int num_rounds(struct crypto_aes_ctx *ctx)
58 {
59 /*
60 * # of rounds specified by AES:
61 * 128 bit key 10 rounds
62 * 192 bit key 12 rounds
63 * 256 bit key 14 rounds
64 * => n byte key => 6 + (n/4) rounds
65 */
66 return 6 + ctx->key_length / 4;
67 }
68
ce_aes_expandkey(struct crypto_aes_ctx * ctx,const u8 * in_key,unsigned int key_len)69 static int ce_aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key,
70 unsigned int key_len)
71 {
72 /*
73 * The AES key schedule round constants
74 */
75 static u8 const rcon[] = {
76 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36,
77 };
78
79 u32 kwords = key_len / sizeof(u32);
80 struct aes_block *key_enc, *key_dec;
81 int i, j;
82
83 if (key_len != AES_KEYSIZE_128 &&
84 key_len != AES_KEYSIZE_192 &&
85 key_len != AES_KEYSIZE_256)
86 return -EINVAL;
87
88 ctx->key_length = key_len;
89 for (i = 0; i < kwords; i++)
90 ctx->key_enc[i] = get_unaligned_le32(in_key + i * sizeof(u32));
91
92 kernel_neon_begin();
93 for (i = 0; i < sizeof(rcon); i++) {
94 u32 *rki = ctx->key_enc + (i * kwords);
95 u32 *rko = rki + kwords;
96
97 rko[0] = ror32(ce_aes_sub(rki[kwords - 1]), 8);
98 rko[0] = rko[0] ^ rki[0] ^ rcon[i];
99 rko[1] = rko[0] ^ rki[1];
100 rko[2] = rko[1] ^ rki[2];
101 rko[3] = rko[2] ^ rki[3];
102
103 if (key_len == AES_KEYSIZE_192) {
104 if (i >= 7)
105 break;
106 rko[4] = rko[3] ^ rki[4];
107 rko[5] = rko[4] ^ rki[5];
108 } else if (key_len == AES_KEYSIZE_256) {
109 if (i >= 6)
110 break;
111 rko[4] = ce_aes_sub(rko[3]) ^ rki[4];
112 rko[5] = rko[4] ^ rki[5];
113 rko[6] = rko[5] ^ rki[6];
114 rko[7] = rko[6] ^ rki[7];
115 }
116 }
117
118 /*
119 * Generate the decryption keys for the Equivalent Inverse Cipher.
120 * This involves reversing the order of the round keys, and applying
121 * the Inverse Mix Columns transformation on all but the first and
122 * the last one.
123 */
124 key_enc = (struct aes_block *)ctx->key_enc;
125 key_dec = (struct aes_block *)ctx->key_dec;
126 j = num_rounds(ctx);
127
128 key_dec[0] = key_enc[j];
129 for (i = 1, j--; j > 0; i++, j--)
130 ce_aes_invert(key_dec + i, key_enc + j);
131 key_dec[i] = key_enc[0];
132
133 kernel_neon_end();
134 return 0;
135 }
136
ce_aes_setkey(struct crypto_skcipher * tfm,const u8 * in_key,unsigned int key_len)137 static int ce_aes_setkey(struct crypto_skcipher *tfm, const u8 *in_key,
138 unsigned int key_len)
139 {
140 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
141 int ret;
142
143 ret = ce_aes_expandkey(ctx, in_key, key_len);
144 if (!ret)
145 return 0;
146
147 crypto_skcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
148 return -EINVAL;
149 }
150
151 struct crypto_aes_xts_ctx {
152 struct crypto_aes_ctx key1;
153 struct crypto_aes_ctx __aligned(8) key2;
154 };
155
xts_set_key(struct crypto_skcipher * tfm,const u8 * in_key,unsigned int key_len)156 static int xts_set_key(struct crypto_skcipher *tfm, const u8 *in_key,
157 unsigned int key_len)
158 {
159 struct crypto_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
160 int ret;
161
162 ret = xts_verify_key(tfm, in_key, key_len);
163 if (ret)
164 return ret;
165
166 ret = ce_aes_expandkey(&ctx->key1, in_key, key_len / 2);
167 if (!ret)
168 ret = ce_aes_expandkey(&ctx->key2, &in_key[key_len / 2],
169 key_len / 2);
170 if (!ret)
171 return 0;
172
173 crypto_skcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
174 return -EINVAL;
175 }
176
ecb_encrypt(struct skcipher_request * req)177 static int ecb_encrypt(struct skcipher_request *req)
178 {
179 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
180 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
181 struct skcipher_walk walk;
182 unsigned int blocks;
183 int err;
184
185 err = skcipher_walk_virt(&walk, req, false);
186
187 while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
188 kernel_neon_begin();
189 ce_aes_ecb_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
190 ctx->key_enc, num_rounds(ctx), blocks);
191 kernel_neon_end();
192 err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
193 }
194 return err;
195 }
196
ecb_decrypt(struct skcipher_request * req)197 static int ecb_decrypt(struct skcipher_request *req)
198 {
199 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
200 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
201 struct skcipher_walk walk;
202 unsigned int blocks;
203 int err;
204
205 err = skcipher_walk_virt(&walk, req, false);
206
207 while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
208 kernel_neon_begin();
209 ce_aes_ecb_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
210 ctx->key_dec, num_rounds(ctx), blocks);
211 kernel_neon_end();
212 err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
213 }
214 return err;
215 }
216
cbc_encrypt_walk(struct skcipher_request * req,struct skcipher_walk * walk)217 static int cbc_encrypt_walk(struct skcipher_request *req,
218 struct skcipher_walk *walk)
219 {
220 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
221 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
222 unsigned int blocks;
223 int err = 0;
224
225 while ((blocks = (walk->nbytes / AES_BLOCK_SIZE))) {
226 kernel_neon_begin();
227 ce_aes_cbc_encrypt(walk->dst.virt.addr, walk->src.virt.addr,
228 ctx->key_enc, num_rounds(ctx), blocks,
229 walk->iv);
230 kernel_neon_end();
231 err = skcipher_walk_done(walk, walk->nbytes % AES_BLOCK_SIZE);
232 }
233 return err;
234 }
235
cbc_encrypt(struct skcipher_request * req)236 static int cbc_encrypt(struct skcipher_request *req)
237 {
238 struct skcipher_walk walk;
239 int err;
240
241 err = skcipher_walk_virt(&walk, req, false);
242 if (err)
243 return err;
244 return cbc_encrypt_walk(req, &walk);
245 }
246
cbc_decrypt_walk(struct skcipher_request * req,struct skcipher_walk * walk)247 static int cbc_decrypt_walk(struct skcipher_request *req,
248 struct skcipher_walk *walk)
249 {
250 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
251 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
252 unsigned int blocks;
253 int err = 0;
254
255 while ((blocks = (walk->nbytes / AES_BLOCK_SIZE))) {
256 kernel_neon_begin();
257 ce_aes_cbc_decrypt(walk->dst.virt.addr, walk->src.virt.addr,
258 ctx->key_dec, num_rounds(ctx), blocks,
259 walk->iv);
260 kernel_neon_end();
261 err = skcipher_walk_done(walk, walk->nbytes % AES_BLOCK_SIZE);
262 }
263 return err;
264 }
265
cbc_decrypt(struct skcipher_request * req)266 static int cbc_decrypt(struct skcipher_request *req)
267 {
268 struct skcipher_walk walk;
269 int err;
270
271 err = skcipher_walk_virt(&walk, req, false);
272 if (err)
273 return err;
274 return cbc_decrypt_walk(req, &walk);
275 }
276
cts_cbc_encrypt(struct skcipher_request * req)277 static int cts_cbc_encrypt(struct skcipher_request *req)
278 {
279 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
280 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
281 int cbc_blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2;
282 struct scatterlist *src = req->src, *dst = req->dst;
283 struct scatterlist sg_src[2], sg_dst[2];
284 struct skcipher_request subreq;
285 struct skcipher_walk walk;
286 int err;
287
288 skcipher_request_set_tfm(&subreq, tfm);
289 skcipher_request_set_callback(&subreq, skcipher_request_flags(req),
290 NULL, NULL);
291
292 if (req->cryptlen <= AES_BLOCK_SIZE) {
293 if (req->cryptlen < AES_BLOCK_SIZE)
294 return -EINVAL;
295 cbc_blocks = 1;
296 }
297
298 if (cbc_blocks > 0) {
299 skcipher_request_set_crypt(&subreq, req->src, req->dst,
300 cbc_blocks * AES_BLOCK_SIZE,
301 req->iv);
302
303 err = skcipher_walk_virt(&walk, &subreq, false) ?:
304 cbc_encrypt_walk(&subreq, &walk);
305 if (err)
306 return err;
307
308 if (req->cryptlen == AES_BLOCK_SIZE)
309 return 0;
310
311 dst = src = scatterwalk_ffwd(sg_src, req->src, subreq.cryptlen);
312 if (req->dst != req->src)
313 dst = scatterwalk_ffwd(sg_dst, req->dst,
314 subreq.cryptlen);
315 }
316
317 /* handle ciphertext stealing */
318 skcipher_request_set_crypt(&subreq, src, dst,
319 req->cryptlen - cbc_blocks * AES_BLOCK_SIZE,
320 req->iv);
321
322 err = skcipher_walk_virt(&walk, &subreq, false);
323 if (err)
324 return err;
325
326 kernel_neon_begin();
327 ce_aes_cbc_cts_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
328 ctx->key_enc, num_rounds(ctx), walk.nbytes,
329 walk.iv);
330 kernel_neon_end();
331
332 return skcipher_walk_done(&walk, 0);
333 }
334
cts_cbc_decrypt(struct skcipher_request * req)335 static int cts_cbc_decrypt(struct skcipher_request *req)
336 {
337 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
338 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
339 int cbc_blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2;
340 struct scatterlist *src = req->src, *dst = req->dst;
341 struct scatterlist sg_src[2], sg_dst[2];
342 struct skcipher_request subreq;
343 struct skcipher_walk walk;
344 int err;
345
346 skcipher_request_set_tfm(&subreq, tfm);
347 skcipher_request_set_callback(&subreq, skcipher_request_flags(req),
348 NULL, NULL);
349
350 if (req->cryptlen <= AES_BLOCK_SIZE) {
351 if (req->cryptlen < AES_BLOCK_SIZE)
352 return -EINVAL;
353 cbc_blocks = 1;
354 }
355
356 if (cbc_blocks > 0) {
357 skcipher_request_set_crypt(&subreq, req->src, req->dst,
358 cbc_blocks * AES_BLOCK_SIZE,
359 req->iv);
360
361 err = skcipher_walk_virt(&walk, &subreq, false) ?:
362 cbc_decrypt_walk(&subreq, &walk);
363 if (err)
364 return err;
365
366 if (req->cryptlen == AES_BLOCK_SIZE)
367 return 0;
368
369 dst = src = scatterwalk_ffwd(sg_src, req->src, subreq.cryptlen);
370 if (req->dst != req->src)
371 dst = scatterwalk_ffwd(sg_dst, req->dst,
372 subreq.cryptlen);
373 }
374
375 /* handle ciphertext stealing */
376 skcipher_request_set_crypt(&subreq, src, dst,
377 req->cryptlen - cbc_blocks * AES_BLOCK_SIZE,
378 req->iv);
379
380 err = skcipher_walk_virt(&walk, &subreq, false);
381 if (err)
382 return err;
383
384 kernel_neon_begin();
385 ce_aes_cbc_cts_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
386 ctx->key_dec, num_rounds(ctx), walk.nbytes,
387 walk.iv);
388 kernel_neon_end();
389
390 return skcipher_walk_done(&walk, 0);
391 }
392
ctr_encrypt(struct skcipher_request * req)393 static int ctr_encrypt(struct skcipher_request *req)
394 {
395 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
396 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
397 struct skcipher_walk walk;
398 int err, blocks;
399
400 err = skcipher_walk_virt(&walk, req, false);
401
402 while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
403 kernel_neon_begin();
404 ce_aes_ctr_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
405 ctx->key_enc, num_rounds(ctx), blocks,
406 walk.iv);
407 kernel_neon_end();
408 err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
409 }
410 if (walk.nbytes) {
411 u8 __aligned(8) tail[AES_BLOCK_SIZE];
412 unsigned int nbytes = walk.nbytes;
413 u8 *tdst = walk.dst.virt.addr;
414 u8 *tsrc = walk.src.virt.addr;
415
416 /*
417 * Tell aes_ctr_encrypt() to process a tail block.
418 */
419 blocks = -1;
420
421 kernel_neon_begin();
422 ce_aes_ctr_encrypt(tail, NULL, ctx->key_enc, num_rounds(ctx),
423 blocks, walk.iv);
424 kernel_neon_end();
425 crypto_xor_cpy(tdst, tsrc, tail, nbytes);
426 err = skcipher_walk_done(&walk, 0);
427 }
428 return err;
429 }
430
ctr_encrypt_one(struct crypto_skcipher * tfm,const u8 * src,u8 * dst)431 static void ctr_encrypt_one(struct crypto_skcipher *tfm, const u8 *src, u8 *dst)
432 {
433 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
434 unsigned long flags;
435
436 /*
437 * Temporarily disable interrupts to avoid races where
438 * cachelines are evicted when the CPU is interrupted
439 * to do something else.
440 */
441 local_irq_save(flags);
442 aes_encrypt(ctx, dst, src);
443 local_irq_restore(flags);
444 }
445
ctr_encrypt_sync(struct skcipher_request * req)446 static int ctr_encrypt_sync(struct skcipher_request *req)
447 {
448 if (!crypto_simd_usable())
449 return crypto_ctr_encrypt_walk(req, ctr_encrypt_one);
450
451 return ctr_encrypt(req);
452 }
453
xts_encrypt(struct skcipher_request * req)454 static int xts_encrypt(struct skcipher_request *req)
455 {
456 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
457 struct crypto_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
458 int err, first, rounds = num_rounds(&ctx->key1);
459 int tail = req->cryptlen % AES_BLOCK_SIZE;
460 struct scatterlist sg_src[2], sg_dst[2];
461 struct skcipher_request subreq;
462 struct scatterlist *src, *dst;
463 struct skcipher_walk walk;
464
465 if (req->cryptlen < AES_BLOCK_SIZE)
466 return -EINVAL;
467
468 err = skcipher_walk_virt(&walk, req, false);
469
470 if (unlikely(tail > 0 && walk.nbytes < walk.total)) {
471 int xts_blocks = DIV_ROUND_UP(req->cryptlen,
472 AES_BLOCK_SIZE) - 2;
473
474 skcipher_walk_abort(&walk);
475
476 skcipher_request_set_tfm(&subreq, tfm);
477 skcipher_request_set_callback(&subreq,
478 skcipher_request_flags(req),
479 NULL, NULL);
480 skcipher_request_set_crypt(&subreq, req->src, req->dst,
481 xts_blocks * AES_BLOCK_SIZE,
482 req->iv);
483 req = &subreq;
484 err = skcipher_walk_virt(&walk, req, false);
485 } else {
486 tail = 0;
487 }
488
489 for (first = 1; walk.nbytes >= AES_BLOCK_SIZE; first = 0) {
490 int nbytes = walk.nbytes;
491
492 if (walk.nbytes < walk.total)
493 nbytes &= ~(AES_BLOCK_SIZE - 1);
494
495 kernel_neon_begin();
496 ce_aes_xts_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
497 ctx->key1.key_enc, rounds, nbytes, walk.iv,
498 ctx->key2.key_enc, first);
499 kernel_neon_end();
500 err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
501 }
502
503 if (err || likely(!tail))
504 return err;
505
506 dst = src = scatterwalk_ffwd(sg_src, req->src, req->cryptlen);
507 if (req->dst != req->src)
508 dst = scatterwalk_ffwd(sg_dst, req->dst, req->cryptlen);
509
510 skcipher_request_set_crypt(req, src, dst, AES_BLOCK_SIZE + tail,
511 req->iv);
512
513 err = skcipher_walk_virt(&walk, req, false);
514 if (err)
515 return err;
516
517 kernel_neon_begin();
518 ce_aes_xts_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
519 ctx->key1.key_enc, rounds, walk.nbytes, walk.iv,
520 ctx->key2.key_enc, first);
521 kernel_neon_end();
522
523 return skcipher_walk_done(&walk, 0);
524 }
525
xts_decrypt(struct skcipher_request * req)526 static int xts_decrypt(struct skcipher_request *req)
527 {
528 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
529 struct crypto_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
530 int err, first, rounds = num_rounds(&ctx->key1);
531 int tail = req->cryptlen % AES_BLOCK_SIZE;
532 struct scatterlist sg_src[2], sg_dst[2];
533 struct skcipher_request subreq;
534 struct scatterlist *src, *dst;
535 struct skcipher_walk walk;
536
537 if (req->cryptlen < AES_BLOCK_SIZE)
538 return -EINVAL;
539
540 err = skcipher_walk_virt(&walk, req, false);
541
542 if (unlikely(tail > 0 && walk.nbytes < walk.total)) {
543 int xts_blocks = DIV_ROUND_UP(req->cryptlen,
544 AES_BLOCK_SIZE) - 2;
545
546 skcipher_walk_abort(&walk);
547
548 skcipher_request_set_tfm(&subreq, tfm);
549 skcipher_request_set_callback(&subreq,
550 skcipher_request_flags(req),
551 NULL, NULL);
552 skcipher_request_set_crypt(&subreq, req->src, req->dst,
553 xts_blocks * AES_BLOCK_SIZE,
554 req->iv);
555 req = &subreq;
556 err = skcipher_walk_virt(&walk, req, false);
557 } else {
558 tail = 0;
559 }
560
561 for (first = 1; walk.nbytes >= AES_BLOCK_SIZE; first = 0) {
562 int nbytes = walk.nbytes;
563
564 if (walk.nbytes < walk.total)
565 nbytes &= ~(AES_BLOCK_SIZE - 1);
566
567 kernel_neon_begin();
568 ce_aes_xts_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
569 ctx->key1.key_dec, rounds, nbytes, walk.iv,
570 ctx->key2.key_enc, first);
571 kernel_neon_end();
572 err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
573 }
574
575 if (err || likely(!tail))
576 return err;
577
578 dst = src = scatterwalk_ffwd(sg_src, req->src, req->cryptlen);
579 if (req->dst != req->src)
580 dst = scatterwalk_ffwd(sg_dst, req->dst, req->cryptlen);
581
582 skcipher_request_set_crypt(req, src, dst, AES_BLOCK_SIZE + tail,
583 req->iv);
584
585 err = skcipher_walk_virt(&walk, req, false);
586 if (err)
587 return err;
588
589 kernel_neon_begin();
590 ce_aes_xts_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
591 ctx->key1.key_dec, rounds, walk.nbytes, walk.iv,
592 ctx->key2.key_enc, first);
593 kernel_neon_end();
594
595 return skcipher_walk_done(&walk, 0);
596 }
597
598 static struct skcipher_alg aes_algs[] = { {
599 .base.cra_name = "__ecb(aes)",
600 .base.cra_driver_name = "__ecb-aes-ce",
601 .base.cra_priority = 300,
602 .base.cra_flags = CRYPTO_ALG_INTERNAL,
603 .base.cra_blocksize = AES_BLOCK_SIZE,
604 .base.cra_ctxsize = sizeof(struct crypto_aes_ctx),
605 .base.cra_module = THIS_MODULE,
606
607 .min_keysize = AES_MIN_KEY_SIZE,
608 .max_keysize = AES_MAX_KEY_SIZE,
609 .setkey = ce_aes_setkey,
610 .encrypt = ecb_encrypt,
611 .decrypt = ecb_decrypt,
612 }, {
613 .base.cra_name = "__cbc(aes)",
614 .base.cra_driver_name = "__cbc-aes-ce",
615 .base.cra_priority = 300,
616 .base.cra_flags = CRYPTO_ALG_INTERNAL,
617 .base.cra_blocksize = AES_BLOCK_SIZE,
618 .base.cra_ctxsize = sizeof(struct crypto_aes_ctx),
619 .base.cra_module = THIS_MODULE,
620
621 .min_keysize = AES_MIN_KEY_SIZE,
622 .max_keysize = AES_MAX_KEY_SIZE,
623 .ivsize = AES_BLOCK_SIZE,
624 .setkey = ce_aes_setkey,
625 .encrypt = cbc_encrypt,
626 .decrypt = cbc_decrypt,
627 }, {
628 .base.cra_name = "__cts(cbc(aes))",
629 .base.cra_driver_name = "__cts-cbc-aes-ce",
630 .base.cra_priority = 300,
631 .base.cra_flags = CRYPTO_ALG_INTERNAL,
632 .base.cra_blocksize = AES_BLOCK_SIZE,
633 .base.cra_ctxsize = sizeof(struct crypto_aes_ctx),
634 .base.cra_module = THIS_MODULE,
635
636 .min_keysize = AES_MIN_KEY_SIZE,
637 .max_keysize = AES_MAX_KEY_SIZE,
638 .ivsize = AES_BLOCK_SIZE,
639 .walksize = 2 * AES_BLOCK_SIZE,
640 .setkey = ce_aes_setkey,
641 .encrypt = cts_cbc_encrypt,
642 .decrypt = cts_cbc_decrypt,
643 }, {
644 .base.cra_name = "__ctr(aes)",
645 .base.cra_driver_name = "__ctr-aes-ce",
646 .base.cra_priority = 300,
647 .base.cra_flags = CRYPTO_ALG_INTERNAL,
648 .base.cra_blocksize = 1,
649 .base.cra_ctxsize = sizeof(struct crypto_aes_ctx),
650 .base.cra_module = THIS_MODULE,
651
652 .min_keysize = AES_MIN_KEY_SIZE,
653 .max_keysize = AES_MAX_KEY_SIZE,
654 .ivsize = AES_BLOCK_SIZE,
655 .chunksize = AES_BLOCK_SIZE,
656 .setkey = ce_aes_setkey,
657 .encrypt = ctr_encrypt,
658 .decrypt = ctr_encrypt,
659 }, {
660 .base.cra_name = "ctr(aes)",
661 .base.cra_driver_name = "ctr-aes-ce-sync",
662 .base.cra_priority = 300 - 1,
663 .base.cra_blocksize = 1,
664 .base.cra_ctxsize = sizeof(struct crypto_aes_ctx),
665 .base.cra_module = THIS_MODULE,
666
667 .min_keysize = AES_MIN_KEY_SIZE,
668 .max_keysize = AES_MAX_KEY_SIZE,
669 .ivsize = AES_BLOCK_SIZE,
670 .chunksize = AES_BLOCK_SIZE,
671 .setkey = ce_aes_setkey,
672 .encrypt = ctr_encrypt_sync,
673 .decrypt = ctr_encrypt_sync,
674 }, {
675 .base.cra_name = "__xts(aes)",
676 .base.cra_driver_name = "__xts-aes-ce",
677 .base.cra_priority = 300,
678 .base.cra_flags = CRYPTO_ALG_INTERNAL,
679 .base.cra_blocksize = AES_BLOCK_SIZE,
680 .base.cra_ctxsize = sizeof(struct crypto_aes_xts_ctx),
681 .base.cra_module = THIS_MODULE,
682
683 .min_keysize = 2 * AES_MIN_KEY_SIZE,
684 .max_keysize = 2 * AES_MAX_KEY_SIZE,
685 .ivsize = AES_BLOCK_SIZE,
686 .walksize = 2 * AES_BLOCK_SIZE,
687 .setkey = xts_set_key,
688 .encrypt = xts_encrypt,
689 .decrypt = xts_decrypt,
690 } };
691
692 static struct simd_skcipher_alg *aes_simd_algs[ARRAY_SIZE(aes_algs)];
693
aes_exit(void)694 static void aes_exit(void)
695 {
696 int i;
697
698 for (i = 0; i < ARRAY_SIZE(aes_simd_algs) && aes_simd_algs[i]; i++)
699 simd_skcipher_free(aes_simd_algs[i]);
700
701 crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs));
702 }
703
aes_init(void)704 static int __init aes_init(void)
705 {
706 struct simd_skcipher_alg *simd;
707 const char *basename;
708 const char *algname;
709 const char *drvname;
710 int err;
711 int i;
712
713 err = crypto_register_skciphers(aes_algs, ARRAY_SIZE(aes_algs));
714 if (err)
715 return err;
716
717 for (i = 0; i < ARRAY_SIZE(aes_algs); i++) {
718 if (!(aes_algs[i].base.cra_flags & CRYPTO_ALG_INTERNAL))
719 continue;
720
721 algname = aes_algs[i].base.cra_name + 2;
722 drvname = aes_algs[i].base.cra_driver_name + 2;
723 basename = aes_algs[i].base.cra_driver_name;
724 simd = simd_skcipher_create_compat(algname, drvname, basename);
725 err = PTR_ERR(simd);
726 if (IS_ERR(simd))
727 goto unregister_simds;
728
729 aes_simd_algs[i] = simd;
730 }
731
732 return 0;
733
734 unregister_simds:
735 aes_exit();
736 return err;
737 }
738
739 module_cpu_feature_match(AES, aes_init);
740 module_exit(aes_exit);
741