1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * Glue Code for AVX assembler version of Twofish Cipher
4 *
5 * Copyright (C) 2012 Johannes Goetzfried
6 * <Johannes.Goetzfried@informatik.stud.uni-erlangen.de>
7 *
8 * Copyright © 2013 Jussi Kivilinna <jussi.kivilinna@iki.fi>
9 */
10
11 #include <linux/module.h>
12 #include <linux/types.h>
13 #include <linux/crypto.h>
14 #include <linux/err.h>
15 #include <crypto/algapi.h>
16 #include <crypto/internal/simd.h>
17 #include <crypto/twofish.h>
18 #include <crypto/xts.h>
19 #include <asm/crypto/glue_helper.h>
20 #include <asm/crypto/twofish.h>
21
22 #define TWOFISH_PARALLEL_BLOCKS 8
23
24 /* 8-way parallel cipher functions */
25 asmlinkage void twofish_ecb_enc_8way(const void *ctx, u8 *dst, const u8 *src);
26 asmlinkage void twofish_ecb_dec_8way(const void *ctx, u8 *dst, const u8 *src);
27
28 asmlinkage void twofish_cbc_dec_8way(const void *ctx, u8 *dst, const u8 *src);
29 asmlinkage void twofish_ctr_8way(const void *ctx, u8 *dst, const u8 *src,
30 le128 *iv);
31
32 asmlinkage void twofish_xts_enc_8way(const void *ctx, u8 *dst, const u8 *src,
33 le128 *iv);
34 asmlinkage void twofish_xts_dec_8way(const void *ctx, u8 *dst, const u8 *src,
35 le128 *iv);
36
twofish_setkey_skcipher(struct crypto_skcipher * tfm,const u8 * key,unsigned int keylen)37 static int twofish_setkey_skcipher(struct crypto_skcipher *tfm,
38 const u8 *key, unsigned int keylen)
39 {
40 return twofish_setkey(&tfm->base, key, keylen);
41 }
42
twofish_enc_blk_3way(const void * ctx,u8 * dst,const u8 * src)43 static inline void twofish_enc_blk_3way(const void *ctx, u8 *dst, const u8 *src)
44 {
45 __twofish_enc_blk_3way(ctx, dst, src, false);
46 }
47
twofish_xts_enc(const void * ctx,u8 * dst,const u8 * src,le128 * iv)48 static void twofish_xts_enc(const void *ctx, u8 *dst, const u8 *src, le128 *iv)
49 {
50 glue_xts_crypt_128bit_one(ctx, dst, src, iv, twofish_enc_blk);
51 }
52
twofish_xts_dec(const void * ctx,u8 * dst,const u8 * src,le128 * iv)53 static void twofish_xts_dec(const void *ctx, u8 *dst, const u8 *src, le128 *iv)
54 {
55 glue_xts_crypt_128bit_one(ctx, dst, src, iv, twofish_dec_blk);
56 }
57
58 struct twofish_xts_ctx {
59 struct twofish_ctx tweak_ctx;
60 struct twofish_ctx crypt_ctx;
61 };
62
xts_twofish_setkey(struct crypto_skcipher * tfm,const u8 * key,unsigned int keylen)63 static int xts_twofish_setkey(struct crypto_skcipher *tfm, const u8 *key,
64 unsigned int keylen)
65 {
66 struct twofish_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
67 u32 *flags = &tfm->base.crt_flags;
68 int err;
69
70 err = xts_verify_key(tfm, key, keylen);
71 if (err)
72 return err;
73
74 /* first half of xts-key is for crypt */
75 err = __twofish_setkey(&ctx->crypt_ctx, key, keylen / 2, flags);
76 if (err)
77 return err;
78
79 /* second half of xts-key is for tweak */
80 return __twofish_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2,
81 flags);
82 }
83
84 static const struct common_glue_ctx twofish_enc = {
85 .num_funcs = 3,
86 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
87
88 .funcs = { {
89 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
90 .fn_u = { .ecb = twofish_ecb_enc_8way }
91 }, {
92 .num_blocks = 3,
93 .fn_u = { .ecb = twofish_enc_blk_3way }
94 }, {
95 .num_blocks = 1,
96 .fn_u = { .ecb = twofish_enc_blk }
97 } }
98 };
99
100 static const struct common_glue_ctx twofish_ctr = {
101 .num_funcs = 3,
102 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
103
104 .funcs = { {
105 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
106 .fn_u = { .ctr = twofish_ctr_8way }
107 }, {
108 .num_blocks = 3,
109 .fn_u = { .ctr = twofish_enc_blk_ctr_3way }
110 }, {
111 .num_blocks = 1,
112 .fn_u = { .ctr = twofish_enc_blk_ctr }
113 } }
114 };
115
116 static const struct common_glue_ctx twofish_enc_xts = {
117 .num_funcs = 2,
118 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
119
120 .funcs = { {
121 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
122 .fn_u = { .xts = twofish_xts_enc_8way }
123 }, {
124 .num_blocks = 1,
125 .fn_u = { .xts = twofish_xts_enc }
126 } }
127 };
128
129 static const struct common_glue_ctx twofish_dec = {
130 .num_funcs = 3,
131 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
132
133 .funcs = { {
134 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
135 .fn_u = { .ecb = twofish_ecb_dec_8way }
136 }, {
137 .num_blocks = 3,
138 .fn_u = { .ecb = twofish_dec_blk_3way }
139 }, {
140 .num_blocks = 1,
141 .fn_u = { .ecb = twofish_dec_blk }
142 } }
143 };
144
145 static const struct common_glue_ctx twofish_dec_cbc = {
146 .num_funcs = 3,
147 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
148
149 .funcs = { {
150 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
151 .fn_u = { .cbc = twofish_cbc_dec_8way }
152 }, {
153 .num_blocks = 3,
154 .fn_u = { .cbc = twofish_dec_blk_cbc_3way }
155 }, {
156 .num_blocks = 1,
157 .fn_u = { .cbc = twofish_dec_blk }
158 } }
159 };
160
161 static const struct common_glue_ctx twofish_dec_xts = {
162 .num_funcs = 2,
163 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
164
165 .funcs = { {
166 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
167 .fn_u = { .xts = twofish_xts_dec_8way }
168 }, {
169 .num_blocks = 1,
170 .fn_u = { .xts = twofish_xts_dec }
171 } }
172 };
173
ecb_encrypt(struct skcipher_request * req)174 static int ecb_encrypt(struct skcipher_request *req)
175 {
176 return glue_ecb_req_128bit(&twofish_enc, req);
177 }
178
ecb_decrypt(struct skcipher_request * req)179 static int ecb_decrypt(struct skcipher_request *req)
180 {
181 return glue_ecb_req_128bit(&twofish_dec, req);
182 }
183
cbc_encrypt(struct skcipher_request * req)184 static int cbc_encrypt(struct skcipher_request *req)
185 {
186 return glue_cbc_encrypt_req_128bit(twofish_enc_blk, req);
187 }
188
cbc_decrypt(struct skcipher_request * req)189 static int cbc_decrypt(struct skcipher_request *req)
190 {
191 return glue_cbc_decrypt_req_128bit(&twofish_dec_cbc, req);
192 }
193
ctr_crypt(struct skcipher_request * req)194 static int ctr_crypt(struct skcipher_request *req)
195 {
196 return glue_ctr_req_128bit(&twofish_ctr, req);
197 }
198
xts_encrypt(struct skcipher_request * req)199 static int xts_encrypt(struct skcipher_request *req)
200 {
201 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
202 struct twofish_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
203
204 return glue_xts_req_128bit(&twofish_enc_xts, req, twofish_enc_blk,
205 &ctx->tweak_ctx, &ctx->crypt_ctx, false);
206 }
207
xts_decrypt(struct skcipher_request * req)208 static int xts_decrypt(struct skcipher_request *req)
209 {
210 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
211 struct twofish_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
212
213 return glue_xts_req_128bit(&twofish_dec_xts, req, twofish_enc_blk,
214 &ctx->tweak_ctx, &ctx->crypt_ctx, true);
215 }
216
217 static struct skcipher_alg twofish_algs[] = {
218 {
219 .base.cra_name = "__ecb(twofish)",
220 .base.cra_driver_name = "__ecb-twofish-avx",
221 .base.cra_priority = 400,
222 .base.cra_flags = CRYPTO_ALG_INTERNAL,
223 .base.cra_blocksize = TF_BLOCK_SIZE,
224 .base.cra_ctxsize = sizeof(struct twofish_ctx),
225 .base.cra_module = THIS_MODULE,
226 .min_keysize = TF_MIN_KEY_SIZE,
227 .max_keysize = TF_MAX_KEY_SIZE,
228 .setkey = twofish_setkey_skcipher,
229 .encrypt = ecb_encrypt,
230 .decrypt = ecb_decrypt,
231 }, {
232 .base.cra_name = "__cbc(twofish)",
233 .base.cra_driver_name = "__cbc-twofish-avx",
234 .base.cra_priority = 400,
235 .base.cra_flags = CRYPTO_ALG_INTERNAL,
236 .base.cra_blocksize = TF_BLOCK_SIZE,
237 .base.cra_ctxsize = sizeof(struct twofish_ctx),
238 .base.cra_module = THIS_MODULE,
239 .min_keysize = TF_MIN_KEY_SIZE,
240 .max_keysize = TF_MAX_KEY_SIZE,
241 .ivsize = TF_BLOCK_SIZE,
242 .setkey = twofish_setkey_skcipher,
243 .encrypt = cbc_encrypt,
244 .decrypt = cbc_decrypt,
245 }, {
246 .base.cra_name = "__ctr(twofish)",
247 .base.cra_driver_name = "__ctr-twofish-avx",
248 .base.cra_priority = 400,
249 .base.cra_flags = CRYPTO_ALG_INTERNAL,
250 .base.cra_blocksize = 1,
251 .base.cra_ctxsize = sizeof(struct twofish_ctx),
252 .base.cra_module = THIS_MODULE,
253 .min_keysize = TF_MIN_KEY_SIZE,
254 .max_keysize = TF_MAX_KEY_SIZE,
255 .ivsize = TF_BLOCK_SIZE,
256 .chunksize = TF_BLOCK_SIZE,
257 .setkey = twofish_setkey_skcipher,
258 .encrypt = ctr_crypt,
259 .decrypt = ctr_crypt,
260 }, {
261 .base.cra_name = "__xts(twofish)",
262 .base.cra_driver_name = "__xts-twofish-avx",
263 .base.cra_priority = 400,
264 .base.cra_flags = CRYPTO_ALG_INTERNAL,
265 .base.cra_blocksize = TF_BLOCK_SIZE,
266 .base.cra_ctxsize = sizeof(struct twofish_xts_ctx),
267 .base.cra_module = THIS_MODULE,
268 .min_keysize = 2 * TF_MIN_KEY_SIZE,
269 .max_keysize = 2 * TF_MAX_KEY_SIZE,
270 .ivsize = TF_BLOCK_SIZE,
271 .setkey = xts_twofish_setkey,
272 .encrypt = xts_encrypt,
273 .decrypt = xts_decrypt,
274 },
275 };
276
277 static struct simd_skcipher_alg *twofish_simd_algs[ARRAY_SIZE(twofish_algs)];
278
twofish_init(void)279 static int __init twofish_init(void)
280 {
281 const char *feature_name;
282
283 if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, &feature_name)) {
284 pr_info("CPU feature '%s' is not supported.\n", feature_name);
285 return -ENODEV;
286 }
287
288 return simd_register_skciphers_compat(twofish_algs,
289 ARRAY_SIZE(twofish_algs),
290 twofish_simd_algs);
291 }
292
twofish_exit(void)293 static void __exit twofish_exit(void)
294 {
295 simd_unregister_skciphers(twofish_algs, ARRAY_SIZE(twofish_algs),
296 twofish_simd_algs);
297 }
298
299 module_init(twofish_init);
300 module_exit(twofish_exit);
301
302 MODULE_DESCRIPTION("Twofish Cipher Algorithm, AVX optimized");
303 MODULE_LICENSE("GPL");
304 MODULE_ALIAS_CRYPTO("twofish");
305