1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * NEON-accelerated implementation of Speck128-XTS and Speck64-XTS
4 * (64-bit version; based on the 32-bit version)
5 *
6 * Copyright (c) 2018 Google, Inc
7 */
8
9 #include <asm/hwcap.h>
10 #include <asm/neon.h>
11 #include <asm/simd.h>
12 #include <crypto/algapi.h>
13 #include <crypto/gf128mul.h>
14 #include <crypto/speck.h>
15 #include <crypto/xts.h>
16 #include <linux/kernel.h>
17 #include <linux/module.h>
18
19 /* The assembly functions only handle multiples of 128 bytes */
20 #define SPECK_NEON_CHUNK_SIZE 128
21
22 /* Speck128 */
23
24 struct speck128_xts_tfm_ctx {
25 struct speck128_tfm_ctx main_key;
26 struct speck128_tfm_ctx tweak_key;
27 };
28
29 asmlinkage void speck128_xts_encrypt_neon(const u64 *round_keys, int nrounds,
30 void *dst, const void *src,
31 unsigned int nbytes, void *tweak);
32
33 asmlinkage void speck128_xts_decrypt_neon(const u64 *round_keys, int nrounds,
34 void *dst, const void *src,
35 unsigned int nbytes, void *tweak);
36
37 typedef void (*speck128_crypt_one_t)(const struct speck128_tfm_ctx *,
38 u8 *, const u8 *);
39 typedef void (*speck128_xts_crypt_many_t)(const u64 *, int, void *,
40 const void *, unsigned int, void *);
41
42 static __always_inline int
__speck128_xts_crypt(struct blkcipher_desc * desc,struct scatterlist * dst,struct scatterlist * src,unsigned int nbytes,speck128_crypt_one_t crypt_one,speck128_xts_crypt_many_t crypt_many)43 __speck128_xts_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,
44 struct scatterlist *src, unsigned int nbytes,
45 speck128_crypt_one_t crypt_one,
46 speck128_xts_crypt_many_t crypt_many)
47 {
48 struct crypto_blkcipher *tfm = desc->tfm;
49 const struct speck128_xts_tfm_ctx *ctx = crypto_blkcipher_ctx(tfm);
50 struct blkcipher_walk walk;
51 le128 tweak;
52 int err;
53
54 blkcipher_walk_init(&walk, dst, src, nbytes);
55 err = blkcipher_walk_virt_block(desc, &walk, SPECK_NEON_CHUNK_SIZE);
56
57 crypto_speck128_encrypt(&ctx->tweak_key, (u8 *)&tweak, walk.iv);
58
59 while (walk.nbytes > 0) {
60 unsigned int nbytes = walk.nbytes;
61 u8 *dst = walk.dst.virt.addr;
62 const u8 *src = walk.src.virt.addr;
63
64 if (nbytes >= SPECK_NEON_CHUNK_SIZE && may_use_simd()) {
65 unsigned int count;
66
67 count = round_down(nbytes, SPECK_NEON_CHUNK_SIZE);
68 kernel_neon_begin();
69 (*crypt_many)(ctx->main_key.round_keys,
70 ctx->main_key.nrounds,
71 dst, src, count, &tweak);
72 kernel_neon_end();
73 dst += count;
74 src += count;
75 nbytes -= count;
76 }
77
78 /* Handle any remainder with generic code */
79 while (nbytes >= sizeof(tweak)) {
80 le128_xor((le128 *)dst, (const le128 *)src, &tweak);
81 (*crypt_one)(&ctx->main_key, dst, dst);
82 le128_xor((le128 *)dst, (const le128 *)dst, &tweak);
83 gf128mul_x_ble((be128 *)&tweak, (const be128 *)&tweak);
84
85 dst += sizeof(tweak);
86 src += sizeof(tweak);
87 nbytes -= sizeof(tweak);
88 }
89 err = blkcipher_walk_done(desc, &walk, nbytes);
90 }
91
92 return err;
93 }
94
speck128_xts_encrypt(struct blkcipher_desc * desc,struct scatterlist * dst,struct scatterlist * src,unsigned int nbytes)95 static int speck128_xts_encrypt(struct blkcipher_desc *desc,
96 struct scatterlist *dst,
97 struct scatterlist *src,
98 unsigned int nbytes)
99 {
100 return __speck128_xts_crypt(desc, dst, src, nbytes,
101 crypto_speck128_encrypt,
102 speck128_xts_encrypt_neon);
103 }
104
speck128_xts_decrypt(struct blkcipher_desc * desc,struct scatterlist * dst,struct scatterlist * src,unsigned int nbytes)105 static int speck128_xts_decrypt(struct blkcipher_desc *desc,
106 struct scatterlist *dst,
107 struct scatterlist *src,
108 unsigned int nbytes)
109 {
110 return __speck128_xts_crypt(desc, dst, src, nbytes,
111 crypto_speck128_decrypt,
112 speck128_xts_decrypt_neon);
113 }
114
speck128_xts_setkey(struct crypto_tfm * tfm,const u8 * key,unsigned int keylen)115 static int speck128_xts_setkey(struct crypto_tfm *tfm, const u8 *key,
116 unsigned int keylen)
117 {
118 struct speck128_xts_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
119 int err;
120
121 if (keylen % 2)
122 return -EINVAL;
123
124 keylen /= 2;
125
126 err = crypto_speck128_setkey(&ctx->main_key, key, keylen);
127 if (err)
128 return err;
129
130 return crypto_speck128_setkey(&ctx->tweak_key, key + keylen, keylen);
131 }
132
133 /* Speck64 */
134
135 struct speck64_xts_tfm_ctx {
136 struct speck64_tfm_ctx main_key;
137 struct speck64_tfm_ctx tweak_key;
138 };
139
140 asmlinkage void speck64_xts_encrypt_neon(const u32 *round_keys, int nrounds,
141 void *dst, const void *src,
142 unsigned int nbytes, void *tweak);
143
144 asmlinkage void speck64_xts_decrypt_neon(const u32 *round_keys, int nrounds,
145 void *dst, const void *src,
146 unsigned int nbytes, void *tweak);
147
148 typedef void (*speck64_crypt_one_t)(const struct speck64_tfm_ctx *,
149 u8 *, const u8 *);
150 typedef void (*speck64_xts_crypt_many_t)(const u32 *, int, void *,
151 const void *, unsigned int, void *);
152
153 static __always_inline int
__speck64_xts_crypt(struct blkcipher_desc * desc,struct scatterlist * dst,struct scatterlist * src,unsigned int nbytes,speck64_crypt_one_t crypt_one,speck64_xts_crypt_many_t crypt_many)154 __speck64_xts_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,
155 struct scatterlist *src, unsigned int nbytes,
156 speck64_crypt_one_t crypt_one,
157 speck64_xts_crypt_many_t crypt_many)
158 {
159 struct crypto_blkcipher *tfm = desc->tfm;
160 const struct speck64_xts_tfm_ctx *ctx = crypto_blkcipher_ctx(tfm);
161 struct blkcipher_walk walk;
162 __le64 tweak;
163 int err;
164
165 blkcipher_walk_init(&walk, dst, src, nbytes);
166 err = blkcipher_walk_virt_block(desc, &walk, SPECK_NEON_CHUNK_SIZE);
167
168 crypto_speck64_encrypt(&ctx->tweak_key, (u8 *)&tweak, walk.iv);
169
170 while (walk.nbytes > 0) {
171 unsigned int nbytes = walk.nbytes;
172 u8 *dst = walk.dst.virt.addr;
173 const u8 *src = walk.src.virt.addr;
174
175 if (nbytes >= SPECK_NEON_CHUNK_SIZE && may_use_simd()) {
176 unsigned int count;
177
178 count = round_down(nbytes, SPECK_NEON_CHUNK_SIZE);
179 kernel_neon_begin();
180 (*crypt_many)(ctx->main_key.round_keys,
181 ctx->main_key.nrounds,
182 dst, src, count, &tweak);
183 kernel_neon_end();
184 dst += count;
185 src += count;
186 nbytes -= count;
187 }
188
189 /* Handle any remainder with generic code */
190 while (nbytes >= sizeof(tweak)) {
191 *(__le64 *)dst = *(__le64 *)src ^ tweak;
192 (*crypt_one)(&ctx->main_key, dst, dst);
193 *(__le64 *)dst ^= tweak;
194 tweak = cpu_to_le64((le64_to_cpu(tweak) << 1) ^
195 ((tweak & cpu_to_le64(1ULL << 63)) ?
196 0x1B : 0));
197 dst += sizeof(tweak);
198 src += sizeof(tweak);
199 nbytes -= sizeof(tweak);
200 }
201 err = blkcipher_walk_done(desc, &walk, nbytes);
202 }
203
204 return err;
205 }
206
speck64_xts_encrypt(struct blkcipher_desc * desc,struct scatterlist * dst,struct scatterlist * src,unsigned int nbytes)207 static int speck64_xts_encrypt(struct blkcipher_desc *desc,
208 struct scatterlist *dst, struct scatterlist *src,
209 unsigned int nbytes)
210 {
211 return __speck64_xts_crypt(desc, dst, src, nbytes,
212 crypto_speck64_encrypt,
213 speck64_xts_encrypt_neon);
214 }
215
speck64_xts_decrypt(struct blkcipher_desc * desc,struct scatterlist * dst,struct scatterlist * src,unsigned int nbytes)216 static int speck64_xts_decrypt(struct blkcipher_desc *desc,
217 struct scatterlist *dst, struct scatterlist *src,
218 unsigned int nbytes)
219 {
220 return __speck64_xts_crypt(desc, dst, src, nbytes,
221 crypto_speck64_decrypt,
222 speck64_xts_decrypt_neon);
223 }
224
speck64_xts_setkey(struct crypto_tfm * tfm,const u8 * key,unsigned int keylen)225 static int speck64_xts_setkey(struct crypto_tfm *tfm, const u8 *key,
226 unsigned int keylen)
227 {
228 struct speck64_xts_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
229 int err;
230
231 if (keylen % 2)
232 return -EINVAL;
233
234 keylen /= 2;
235
236 err = crypto_speck64_setkey(&ctx->main_key, key, keylen);
237 if (err)
238 return err;
239
240 return crypto_speck64_setkey(&ctx->tweak_key, key + keylen, keylen);
241 }
242
243 static struct crypto_alg speck_algs[] = {
244 {
245 .cra_name = "xts(speck128)",
246 .cra_driver_name = "xts-speck128-neon",
247 .cra_priority = 300,
248 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
249 .cra_blocksize = SPECK128_BLOCK_SIZE,
250 .cra_type = &crypto_blkcipher_type,
251 .cra_ctxsize = sizeof(struct speck128_xts_tfm_ctx),
252 .cra_alignmask = 7,
253 .cra_module = THIS_MODULE,
254 .cra_u = {
255 .blkcipher = {
256 .min_keysize = 2 * SPECK128_128_KEY_SIZE,
257 .max_keysize = 2 * SPECK128_256_KEY_SIZE,
258 .ivsize = SPECK128_BLOCK_SIZE,
259 .setkey = speck128_xts_setkey,
260 .encrypt = speck128_xts_encrypt,
261 .decrypt = speck128_xts_decrypt,
262 }
263 }
264 }, {
265 .cra_name = "xts(speck64)",
266 .cra_driver_name = "xts-speck64-neon",
267 .cra_priority = 300,
268 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
269 .cra_blocksize = SPECK64_BLOCK_SIZE,
270 .cra_type = &crypto_blkcipher_type,
271 .cra_ctxsize = sizeof(struct speck64_xts_tfm_ctx),
272 .cra_alignmask = 7,
273 .cra_module = THIS_MODULE,
274 .cra_u = {
275 .blkcipher = {
276 .min_keysize = 2 * SPECK64_96_KEY_SIZE,
277 .max_keysize = 2 * SPECK64_128_KEY_SIZE,
278 .ivsize = SPECK64_BLOCK_SIZE,
279 .setkey = speck64_xts_setkey,
280 .encrypt = speck64_xts_encrypt,
281 .decrypt = speck64_xts_decrypt,
282 }
283 }
284 }
285 };
286
speck_neon_module_init(void)287 static int __init speck_neon_module_init(void)
288 {
289 if (!(elf_hwcap & HWCAP_ASIMD))
290 return -ENODEV;
291 return crypto_register_algs(speck_algs, ARRAY_SIZE(speck_algs));
292 }
293
speck_neon_module_exit(void)294 static void __exit speck_neon_module_exit(void)
295 {
296 crypto_unregister_algs(speck_algs, ARRAY_SIZE(speck_algs));
297 }
298
299 module_init(speck_neon_module_init);
300 module_exit(speck_neon_module_exit);
301
302 MODULE_DESCRIPTION("Speck block cipher (NEON-accelerated)");
303 MODULE_LICENSE("GPL");
304 MODULE_AUTHOR("Eric Biggers <ebiggers@google.com>");
305 MODULE_ALIAS_CRYPTO("xts(speck128)");
306 MODULE_ALIAS_CRYPTO("xts-speck128-neon");
307 MODULE_ALIAS_CRYPTO("xts(speck64)");
308 MODULE_ALIAS_CRYPTO("xts-speck64-neon");
309