1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Bit sliced AES using NEON instructions
4 *
5 * Copyright (C) 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
6 */
7
8 #include <asm/neon.h>
9 #include <asm/simd.h>
10 #include <crypto/aes.h>
11 #include <crypto/ctr.h>
12 #include <crypto/internal/simd.h>
13 #include <crypto/internal/skcipher.h>
14 #include <crypto/scatterwalk.h>
15 #include <crypto/xts.h>
16 #include <linux/module.h>
17
18 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
19 MODULE_LICENSE("GPL v2");
20
21 MODULE_ALIAS_CRYPTO("ecb(aes)");
22 MODULE_ALIAS_CRYPTO("cbc(aes)-all");
23 MODULE_ALIAS_CRYPTO("ctr(aes)");
24 MODULE_ALIAS_CRYPTO("xts(aes)");
25
26 asmlinkage void aesbs_convert_key(u8 out[], u32 const rk[], int rounds);
27
28 asmlinkage void aesbs_ecb_encrypt(u8 out[], u8 const in[], u8 const rk[],
29 int rounds, int blocks);
30 asmlinkage void aesbs_ecb_decrypt(u8 out[], u8 const in[], u8 const rk[],
31 int rounds, int blocks);
32
33 asmlinkage void aesbs_cbc_decrypt(u8 out[], u8 const in[], u8 const rk[],
34 int rounds, int blocks, u8 iv[]);
35
36 asmlinkage void aesbs_ctr_encrypt(u8 out[], u8 const in[], u8 const rk[],
37 int rounds, int blocks, u8 ctr[], u8 final[]);
38
39 asmlinkage void aesbs_xts_encrypt(u8 out[], u8 const in[], u8 const rk[],
40 int rounds, int blocks, u8 iv[], int);
41 asmlinkage void aesbs_xts_decrypt(u8 out[], u8 const in[], u8 const rk[],
42 int rounds, int blocks, u8 iv[], int);
43
44 struct aesbs_ctx {
45 int rounds;
46 u8 rk[13 * (8 * AES_BLOCK_SIZE) + 32] __aligned(AES_BLOCK_SIZE);
47 };
48
49 struct aesbs_cbc_ctx {
50 struct aesbs_ctx key;
51 struct crypto_skcipher *enc_tfm;
52 };
53
54 struct aesbs_xts_ctx {
55 struct aesbs_ctx key;
56 struct crypto_cipher *cts_tfm;
57 struct crypto_cipher *tweak_tfm;
58 };
59
60 struct aesbs_ctr_ctx {
61 struct aesbs_ctx key; /* must be first member */
62 struct crypto_aes_ctx fallback;
63 };
64
aesbs_setkey(struct crypto_skcipher * tfm,const u8 * in_key,unsigned int key_len)65 static int aesbs_setkey(struct crypto_skcipher *tfm, const u8 *in_key,
66 unsigned int key_len)
67 {
68 struct aesbs_ctx *ctx = crypto_skcipher_ctx(tfm);
69 struct crypto_aes_ctx rk;
70 int err;
71
72 err = aes_expandkey(&rk, in_key, key_len);
73 if (err)
74 return err;
75
76 ctx->rounds = 6 + key_len / 4;
77
78 kernel_neon_begin();
79 aesbs_convert_key(ctx->rk, rk.key_enc, ctx->rounds);
80 kernel_neon_end();
81
82 return 0;
83 }
84
__ecb_crypt(struct skcipher_request * req,void (* fn)(u8 out[],u8 const in[],u8 const rk[],int rounds,int blocks))85 static int __ecb_crypt(struct skcipher_request *req,
86 void (*fn)(u8 out[], u8 const in[], u8 const rk[],
87 int rounds, int blocks))
88 {
89 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
90 struct aesbs_ctx *ctx = crypto_skcipher_ctx(tfm);
91 struct skcipher_walk walk;
92 int err;
93
94 err = skcipher_walk_virt(&walk, req, false);
95
96 while (walk.nbytes >= AES_BLOCK_SIZE) {
97 unsigned int blocks = walk.nbytes / AES_BLOCK_SIZE;
98
99 if (walk.nbytes < walk.total)
100 blocks = round_down(blocks,
101 walk.stride / AES_BLOCK_SIZE);
102
103 kernel_neon_begin();
104 fn(walk.dst.virt.addr, walk.src.virt.addr, ctx->rk,
105 ctx->rounds, blocks);
106 kernel_neon_end();
107 err = skcipher_walk_done(&walk,
108 walk.nbytes - blocks * AES_BLOCK_SIZE);
109 }
110
111 return err;
112 }
113
ecb_encrypt(struct skcipher_request * req)114 static int ecb_encrypt(struct skcipher_request *req)
115 {
116 return __ecb_crypt(req, aesbs_ecb_encrypt);
117 }
118
ecb_decrypt(struct skcipher_request * req)119 static int ecb_decrypt(struct skcipher_request *req)
120 {
121 return __ecb_crypt(req, aesbs_ecb_decrypt);
122 }
123
aesbs_cbc_setkey(struct crypto_skcipher * tfm,const u8 * in_key,unsigned int key_len)124 static int aesbs_cbc_setkey(struct crypto_skcipher *tfm, const u8 *in_key,
125 unsigned int key_len)
126 {
127 struct aesbs_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
128 struct crypto_aes_ctx rk;
129 int err;
130
131 err = aes_expandkey(&rk, in_key, key_len);
132 if (err)
133 return err;
134
135 ctx->key.rounds = 6 + key_len / 4;
136
137 kernel_neon_begin();
138 aesbs_convert_key(ctx->key.rk, rk.key_enc, ctx->key.rounds);
139 kernel_neon_end();
140 memzero_explicit(&rk, sizeof(rk));
141
142 return crypto_skcipher_setkey(ctx->enc_tfm, in_key, key_len);
143 }
144
cbc_encrypt(struct skcipher_request * req)145 static int cbc_encrypt(struct skcipher_request *req)
146 {
147 struct skcipher_request *subreq = skcipher_request_ctx(req);
148 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
149 struct aesbs_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
150
151 skcipher_request_set_tfm(subreq, ctx->enc_tfm);
152 skcipher_request_set_callback(subreq,
153 skcipher_request_flags(req),
154 NULL, NULL);
155 skcipher_request_set_crypt(subreq, req->src, req->dst,
156 req->cryptlen, req->iv);
157
158 return crypto_skcipher_encrypt(subreq);
159 }
160
cbc_decrypt(struct skcipher_request * req)161 static int cbc_decrypt(struct skcipher_request *req)
162 {
163 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
164 struct aesbs_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
165 struct skcipher_walk walk;
166 int err;
167
168 err = skcipher_walk_virt(&walk, req, false);
169
170 while (walk.nbytes >= AES_BLOCK_SIZE) {
171 unsigned int blocks = walk.nbytes / AES_BLOCK_SIZE;
172
173 if (walk.nbytes < walk.total)
174 blocks = round_down(blocks,
175 walk.stride / AES_BLOCK_SIZE);
176
177 kernel_neon_begin();
178 aesbs_cbc_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
179 ctx->key.rk, ctx->key.rounds, blocks,
180 walk.iv);
181 kernel_neon_end();
182 err = skcipher_walk_done(&walk,
183 walk.nbytes - blocks * AES_BLOCK_SIZE);
184 }
185
186 return err;
187 }
188
cbc_init(struct crypto_skcipher * tfm)189 static int cbc_init(struct crypto_skcipher *tfm)
190 {
191 struct aesbs_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
192 unsigned int reqsize;
193
194 ctx->enc_tfm = crypto_alloc_skcipher("cbc(aes)", 0, CRYPTO_ALG_ASYNC |
195 CRYPTO_ALG_NEED_FALLBACK);
196 if (IS_ERR(ctx->enc_tfm))
197 return PTR_ERR(ctx->enc_tfm);
198
199 reqsize = sizeof(struct skcipher_request);
200 reqsize += crypto_skcipher_reqsize(ctx->enc_tfm);
201 crypto_skcipher_set_reqsize(tfm, reqsize);
202
203 return 0;
204 }
205
cbc_exit(struct crypto_skcipher * tfm)206 static void cbc_exit(struct crypto_skcipher *tfm)
207 {
208 struct aesbs_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
209
210 crypto_free_skcipher(ctx->enc_tfm);
211 }
212
aesbs_ctr_setkey_sync(struct crypto_skcipher * tfm,const u8 * in_key,unsigned int key_len)213 static int aesbs_ctr_setkey_sync(struct crypto_skcipher *tfm, const u8 *in_key,
214 unsigned int key_len)
215 {
216 struct aesbs_ctr_ctx *ctx = crypto_skcipher_ctx(tfm);
217 int err;
218
219 err = aes_expandkey(&ctx->fallback, in_key, key_len);
220 if (err)
221 return err;
222
223 ctx->key.rounds = 6 + key_len / 4;
224
225 kernel_neon_begin();
226 aesbs_convert_key(ctx->key.rk, ctx->fallback.key_enc, ctx->key.rounds);
227 kernel_neon_end();
228
229 return 0;
230 }
231
ctr_encrypt(struct skcipher_request * req)232 static int ctr_encrypt(struct skcipher_request *req)
233 {
234 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
235 struct aesbs_ctx *ctx = crypto_skcipher_ctx(tfm);
236 struct skcipher_walk walk;
237 u8 buf[AES_BLOCK_SIZE];
238 int err;
239
240 err = skcipher_walk_virt(&walk, req, false);
241
242 while (walk.nbytes > 0) {
243 unsigned int blocks = walk.nbytes / AES_BLOCK_SIZE;
244 u8 *final = (walk.total % AES_BLOCK_SIZE) ? buf : NULL;
245
246 if (walk.nbytes < walk.total) {
247 blocks = round_down(blocks,
248 walk.stride / AES_BLOCK_SIZE);
249 final = NULL;
250 }
251
252 kernel_neon_begin();
253 aesbs_ctr_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
254 ctx->rk, ctx->rounds, blocks, walk.iv, final);
255 kernel_neon_end();
256
257 if (final) {
258 u8 *dst = walk.dst.virt.addr + blocks * AES_BLOCK_SIZE;
259 u8 *src = walk.src.virt.addr + blocks * AES_BLOCK_SIZE;
260
261 crypto_xor_cpy(dst, src, final,
262 walk.total % AES_BLOCK_SIZE);
263
264 err = skcipher_walk_done(&walk, 0);
265 break;
266 }
267 err = skcipher_walk_done(&walk,
268 walk.nbytes - blocks * AES_BLOCK_SIZE);
269 }
270
271 return err;
272 }
273
ctr_encrypt_one(struct crypto_skcipher * tfm,const u8 * src,u8 * dst)274 static void ctr_encrypt_one(struct crypto_skcipher *tfm, const u8 *src, u8 *dst)
275 {
276 struct aesbs_ctr_ctx *ctx = crypto_skcipher_ctx(tfm);
277 unsigned long flags;
278
279 /*
280 * Temporarily disable interrupts to avoid races where
281 * cachelines are evicted when the CPU is interrupted
282 * to do something else.
283 */
284 local_irq_save(flags);
285 aes_encrypt(&ctx->fallback, dst, src);
286 local_irq_restore(flags);
287 }
288
ctr_encrypt_sync(struct skcipher_request * req)289 static int ctr_encrypt_sync(struct skcipher_request *req)
290 {
291 if (!crypto_simd_usable())
292 return crypto_ctr_encrypt_walk(req, ctr_encrypt_one);
293
294 return ctr_encrypt(req);
295 }
296
aesbs_xts_setkey(struct crypto_skcipher * tfm,const u8 * in_key,unsigned int key_len)297 static int aesbs_xts_setkey(struct crypto_skcipher *tfm, const u8 *in_key,
298 unsigned int key_len)
299 {
300 struct aesbs_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
301 int err;
302
303 err = xts_verify_key(tfm, in_key, key_len);
304 if (err)
305 return err;
306
307 key_len /= 2;
308 err = crypto_cipher_setkey(ctx->cts_tfm, in_key, key_len);
309 if (err)
310 return err;
311 err = crypto_cipher_setkey(ctx->tweak_tfm, in_key + key_len, key_len);
312 if (err)
313 return err;
314
315 return aesbs_setkey(tfm, in_key, key_len);
316 }
317
xts_init(struct crypto_skcipher * tfm)318 static int xts_init(struct crypto_skcipher *tfm)
319 {
320 struct aesbs_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
321
322 ctx->cts_tfm = crypto_alloc_cipher("aes", 0, 0);
323 if (IS_ERR(ctx->cts_tfm))
324 return PTR_ERR(ctx->cts_tfm);
325
326 ctx->tweak_tfm = crypto_alloc_cipher("aes", 0, 0);
327 if (IS_ERR(ctx->tweak_tfm))
328 crypto_free_cipher(ctx->cts_tfm);
329
330 return PTR_ERR_OR_ZERO(ctx->tweak_tfm);
331 }
332
xts_exit(struct crypto_skcipher * tfm)333 static void xts_exit(struct crypto_skcipher *tfm)
334 {
335 struct aesbs_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
336
337 crypto_free_cipher(ctx->tweak_tfm);
338 crypto_free_cipher(ctx->cts_tfm);
339 }
340
__xts_crypt(struct skcipher_request * req,bool encrypt,void (* fn)(u8 out[],u8 const in[],u8 const rk[],int rounds,int blocks,u8 iv[],int))341 static int __xts_crypt(struct skcipher_request *req, bool encrypt,
342 void (*fn)(u8 out[], u8 const in[], u8 const rk[],
343 int rounds, int blocks, u8 iv[], int))
344 {
345 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
346 struct aesbs_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
347 int tail = req->cryptlen % AES_BLOCK_SIZE;
348 struct skcipher_request subreq;
349 u8 buf[2 * AES_BLOCK_SIZE];
350 struct skcipher_walk walk;
351 int err;
352
353 if (req->cryptlen < AES_BLOCK_SIZE)
354 return -EINVAL;
355
356 if (unlikely(tail)) {
357 skcipher_request_set_tfm(&subreq, tfm);
358 skcipher_request_set_callback(&subreq,
359 skcipher_request_flags(req),
360 NULL, NULL);
361 skcipher_request_set_crypt(&subreq, req->src, req->dst,
362 req->cryptlen - tail, req->iv);
363 req = &subreq;
364 }
365
366 err = skcipher_walk_virt(&walk, req, true);
367 if (err)
368 return err;
369
370 crypto_cipher_encrypt_one(ctx->tweak_tfm, walk.iv, walk.iv);
371
372 while (walk.nbytes >= AES_BLOCK_SIZE) {
373 unsigned int blocks = walk.nbytes / AES_BLOCK_SIZE;
374 int reorder_last_tweak = !encrypt && tail > 0;
375
376 if (walk.nbytes < walk.total) {
377 blocks = round_down(blocks,
378 walk.stride / AES_BLOCK_SIZE);
379 reorder_last_tweak = 0;
380 }
381
382 kernel_neon_begin();
383 fn(walk.dst.virt.addr, walk.src.virt.addr, ctx->key.rk,
384 ctx->key.rounds, blocks, walk.iv, reorder_last_tweak);
385 kernel_neon_end();
386 err = skcipher_walk_done(&walk,
387 walk.nbytes - blocks * AES_BLOCK_SIZE);
388 }
389
390 if (err || likely(!tail))
391 return err;
392
393 /* handle ciphertext stealing */
394 scatterwalk_map_and_copy(buf, req->dst, req->cryptlen - AES_BLOCK_SIZE,
395 AES_BLOCK_SIZE, 0);
396 memcpy(buf + AES_BLOCK_SIZE, buf, tail);
397 scatterwalk_map_and_copy(buf, req->src, req->cryptlen, tail, 0);
398
399 crypto_xor(buf, req->iv, AES_BLOCK_SIZE);
400
401 if (encrypt)
402 crypto_cipher_encrypt_one(ctx->cts_tfm, buf, buf);
403 else
404 crypto_cipher_decrypt_one(ctx->cts_tfm, buf, buf);
405
406 crypto_xor(buf, req->iv, AES_BLOCK_SIZE);
407
408 scatterwalk_map_and_copy(buf, req->dst, req->cryptlen - AES_BLOCK_SIZE,
409 AES_BLOCK_SIZE + tail, 1);
410 return 0;
411 }
412
xts_encrypt(struct skcipher_request * req)413 static int xts_encrypt(struct skcipher_request *req)
414 {
415 return __xts_crypt(req, true, aesbs_xts_encrypt);
416 }
417
xts_decrypt(struct skcipher_request * req)418 static int xts_decrypt(struct skcipher_request *req)
419 {
420 return __xts_crypt(req, false, aesbs_xts_decrypt);
421 }
422
423 static struct skcipher_alg aes_algs[] = { {
424 .base.cra_name = "__ecb(aes)",
425 .base.cra_driver_name = "__ecb-aes-neonbs",
426 .base.cra_priority = 250,
427 .base.cra_blocksize = AES_BLOCK_SIZE,
428 .base.cra_ctxsize = sizeof(struct aesbs_ctx),
429 .base.cra_module = THIS_MODULE,
430 .base.cra_flags = CRYPTO_ALG_INTERNAL,
431
432 .min_keysize = AES_MIN_KEY_SIZE,
433 .max_keysize = AES_MAX_KEY_SIZE,
434 .walksize = 8 * AES_BLOCK_SIZE,
435 .setkey = aesbs_setkey,
436 .encrypt = ecb_encrypt,
437 .decrypt = ecb_decrypt,
438 }, {
439 .base.cra_name = "__cbc(aes)",
440 .base.cra_driver_name = "__cbc-aes-neonbs",
441 .base.cra_priority = 250,
442 .base.cra_blocksize = AES_BLOCK_SIZE,
443 .base.cra_ctxsize = sizeof(struct aesbs_cbc_ctx),
444 .base.cra_module = THIS_MODULE,
445 .base.cra_flags = CRYPTO_ALG_INTERNAL |
446 CRYPTO_ALG_NEED_FALLBACK,
447
448 .min_keysize = AES_MIN_KEY_SIZE,
449 .max_keysize = AES_MAX_KEY_SIZE,
450 .walksize = 8 * AES_BLOCK_SIZE,
451 .ivsize = AES_BLOCK_SIZE,
452 .setkey = aesbs_cbc_setkey,
453 .encrypt = cbc_encrypt,
454 .decrypt = cbc_decrypt,
455 .init = cbc_init,
456 .exit = cbc_exit,
457 }, {
458 .base.cra_name = "__ctr(aes)",
459 .base.cra_driver_name = "__ctr-aes-neonbs",
460 .base.cra_priority = 250,
461 .base.cra_blocksize = 1,
462 .base.cra_ctxsize = sizeof(struct aesbs_ctx),
463 .base.cra_module = THIS_MODULE,
464 .base.cra_flags = CRYPTO_ALG_INTERNAL,
465
466 .min_keysize = AES_MIN_KEY_SIZE,
467 .max_keysize = AES_MAX_KEY_SIZE,
468 .chunksize = AES_BLOCK_SIZE,
469 .walksize = 8 * AES_BLOCK_SIZE,
470 .ivsize = AES_BLOCK_SIZE,
471 .setkey = aesbs_setkey,
472 .encrypt = ctr_encrypt,
473 .decrypt = ctr_encrypt,
474 }, {
475 .base.cra_name = "ctr(aes)",
476 .base.cra_driver_name = "ctr-aes-neonbs-sync",
477 .base.cra_priority = 250 - 1,
478 .base.cra_blocksize = 1,
479 .base.cra_ctxsize = sizeof(struct aesbs_ctr_ctx),
480 .base.cra_module = THIS_MODULE,
481
482 .min_keysize = AES_MIN_KEY_SIZE,
483 .max_keysize = AES_MAX_KEY_SIZE,
484 .chunksize = AES_BLOCK_SIZE,
485 .walksize = 8 * AES_BLOCK_SIZE,
486 .ivsize = AES_BLOCK_SIZE,
487 .setkey = aesbs_ctr_setkey_sync,
488 .encrypt = ctr_encrypt_sync,
489 .decrypt = ctr_encrypt_sync,
490 }, {
491 .base.cra_name = "__xts(aes)",
492 .base.cra_driver_name = "__xts-aes-neonbs",
493 .base.cra_priority = 250,
494 .base.cra_blocksize = AES_BLOCK_SIZE,
495 .base.cra_ctxsize = sizeof(struct aesbs_xts_ctx),
496 .base.cra_module = THIS_MODULE,
497 .base.cra_flags = CRYPTO_ALG_INTERNAL,
498
499 .min_keysize = 2 * AES_MIN_KEY_SIZE,
500 .max_keysize = 2 * AES_MAX_KEY_SIZE,
501 .walksize = 8 * AES_BLOCK_SIZE,
502 .ivsize = AES_BLOCK_SIZE,
503 .setkey = aesbs_xts_setkey,
504 .encrypt = xts_encrypt,
505 .decrypt = xts_decrypt,
506 .init = xts_init,
507 .exit = xts_exit,
508 } };
509
510 static struct simd_skcipher_alg *aes_simd_algs[ARRAY_SIZE(aes_algs)];
511
aes_exit(void)512 static void aes_exit(void)
513 {
514 int i;
515
516 for (i = 0; i < ARRAY_SIZE(aes_simd_algs); i++)
517 if (aes_simd_algs[i])
518 simd_skcipher_free(aes_simd_algs[i]);
519
520 crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs));
521 }
522
aes_init(void)523 static int __init aes_init(void)
524 {
525 struct simd_skcipher_alg *simd;
526 const char *basename;
527 const char *algname;
528 const char *drvname;
529 int err;
530 int i;
531
532 if (!(elf_hwcap & HWCAP_NEON))
533 return -ENODEV;
534
535 err = crypto_register_skciphers(aes_algs, ARRAY_SIZE(aes_algs));
536 if (err)
537 return err;
538
539 for (i = 0; i < ARRAY_SIZE(aes_algs); i++) {
540 if (!(aes_algs[i].base.cra_flags & CRYPTO_ALG_INTERNAL))
541 continue;
542
543 algname = aes_algs[i].base.cra_name + 2;
544 drvname = aes_algs[i].base.cra_driver_name + 2;
545 basename = aes_algs[i].base.cra_driver_name;
546 simd = simd_skcipher_create_compat(algname, drvname, basename);
547 err = PTR_ERR(simd);
548 if (IS_ERR(simd))
549 goto unregister_simds;
550
551 aes_simd_algs[i] = simd;
552 }
553 return 0;
554
555 unregister_simds:
556 aes_exit();
557 return err;
558 }
559
560 late_initcall(aes_init);
561 module_exit(aes_exit);
562