1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * Glue Code for AVX assembler version of Twofish Cipher
4 *
5 * Copyright (C) 2012 Johannes Goetzfried
6 * <Johannes.Goetzfried@informatik.stud.uni-erlangen.de>
7 *
8 * Copyright © 2013 Jussi Kivilinna <jussi.kivilinna@iki.fi>
9 */
10
11 #include <linux/module.h>
12 #include <linux/types.h>
13 #include <linux/crypto.h>
14 #include <linux/err.h>
15 #include <crypto/algapi.h>
16 #include <crypto/internal/simd.h>
17 #include <crypto/twofish.h>
18 #include <crypto/xts.h>
19 #include <asm/crypto/glue_helper.h>
20 #include <asm/crypto/twofish.h>
21
22 #define TWOFISH_PARALLEL_BLOCKS 8
23
24 /* 8-way parallel cipher functions */
25 asmlinkage void twofish_ecb_enc_8way(const void *ctx, u8 *dst, const u8 *src);
26 asmlinkage void twofish_ecb_dec_8way(const void *ctx, u8 *dst, const u8 *src);
27
28 asmlinkage void twofish_cbc_dec_8way(const void *ctx, u8 *dst, const u8 *src);
29 asmlinkage void twofish_ctr_8way(const void *ctx, u8 *dst, const u8 *src,
30 le128 *iv);
31
32 asmlinkage void twofish_xts_enc_8way(const void *ctx, u8 *dst, const u8 *src,
33 le128 *iv);
34 asmlinkage void twofish_xts_dec_8way(const void *ctx, u8 *dst, const u8 *src,
35 le128 *iv);
36
twofish_setkey_skcipher(struct crypto_skcipher * tfm,const u8 * key,unsigned int keylen)37 static int twofish_setkey_skcipher(struct crypto_skcipher *tfm,
38 const u8 *key, unsigned int keylen)
39 {
40 return twofish_setkey(&tfm->base, key, keylen);
41 }
42
twofish_enc_blk_3way(const void * ctx,u8 * dst,const u8 * src)43 static inline void twofish_enc_blk_3way(const void *ctx, u8 *dst, const u8 *src)
44 {
45 __twofish_enc_blk_3way(ctx, dst, src, false);
46 }
47
twofish_xts_enc(const void * ctx,u8 * dst,const u8 * src,le128 * iv)48 static void twofish_xts_enc(const void *ctx, u8 *dst, const u8 *src, le128 *iv)
49 {
50 glue_xts_crypt_128bit_one(ctx, dst, src, iv, twofish_enc_blk);
51 }
52
twofish_xts_dec(const void * ctx,u8 * dst,const u8 * src,le128 * iv)53 static void twofish_xts_dec(const void *ctx, u8 *dst, const u8 *src, le128 *iv)
54 {
55 glue_xts_crypt_128bit_one(ctx, dst, src, iv, twofish_dec_blk);
56 }
57
58 struct twofish_xts_ctx {
59 struct twofish_ctx tweak_ctx;
60 struct twofish_ctx crypt_ctx;
61 };
62
xts_twofish_setkey(struct crypto_skcipher * tfm,const u8 * key,unsigned int keylen)63 static int xts_twofish_setkey(struct crypto_skcipher *tfm, const u8 *key,
64 unsigned int keylen)
65 {
66 struct twofish_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
67 int err;
68
69 err = xts_verify_key(tfm, key, keylen);
70 if (err)
71 return err;
72
73 /* first half of xts-key is for crypt */
74 err = __twofish_setkey(&ctx->crypt_ctx, key, keylen / 2);
75 if (err)
76 return err;
77
78 /* second half of xts-key is for tweak */
79 return __twofish_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2);
80 }
81
82 static const struct common_glue_ctx twofish_enc = {
83 .num_funcs = 3,
84 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
85
86 .funcs = { {
87 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
88 .fn_u = { .ecb = twofish_ecb_enc_8way }
89 }, {
90 .num_blocks = 3,
91 .fn_u = { .ecb = twofish_enc_blk_3way }
92 }, {
93 .num_blocks = 1,
94 .fn_u = { .ecb = twofish_enc_blk }
95 } }
96 };
97
98 static const struct common_glue_ctx twofish_ctr = {
99 .num_funcs = 3,
100 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
101
102 .funcs = { {
103 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
104 .fn_u = { .ctr = twofish_ctr_8way }
105 }, {
106 .num_blocks = 3,
107 .fn_u = { .ctr = twofish_enc_blk_ctr_3way }
108 }, {
109 .num_blocks = 1,
110 .fn_u = { .ctr = twofish_enc_blk_ctr }
111 } }
112 };
113
114 static const struct common_glue_ctx twofish_enc_xts = {
115 .num_funcs = 2,
116 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
117
118 .funcs = { {
119 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
120 .fn_u = { .xts = twofish_xts_enc_8way }
121 }, {
122 .num_blocks = 1,
123 .fn_u = { .xts = twofish_xts_enc }
124 } }
125 };
126
127 static const struct common_glue_ctx twofish_dec = {
128 .num_funcs = 3,
129 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
130
131 .funcs = { {
132 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
133 .fn_u = { .ecb = twofish_ecb_dec_8way }
134 }, {
135 .num_blocks = 3,
136 .fn_u = { .ecb = twofish_dec_blk_3way }
137 }, {
138 .num_blocks = 1,
139 .fn_u = { .ecb = twofish_dec_blk }
140 } }
141 };
142
143 static const struct common_glue_ctx twofish_dec_cbc = {
144 .num_funcs = 3,
145 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
146
147 .funcs = { {
148 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
149 .fn_u = { .cbc = twofish_cbc_dec_8way }
150 }, {
151 .num_blocks = 3,
152 .fn_u = { .cbc = twofish_dec_blk_cbc_3way }
153 }, {
154 .num_blocks = 1,
155 .fn_u = { .cbc = twofish_dec_blk }
156 } }
157 };
158
159 static const struct common_glue_ctx twofish_dec_xts = {
160 .num_funcs = 2,
161 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
162
163 .funcs = { {
164 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
165 .fn_u = { .xts = twofish_xts_dec_8way }
166 }, {
167 .num_blocks = 1,
168 .fn_u = { .xts = twofish_xts_dec }
169 } }
170 };
171
ecb_encrypt(struct skcipher_request * req)172 static int ecb_encrypt(struct skcipher_request *req)
173 {
174 return glue_ecb_req_128bit(&twofish_enc, req);
175 }
176
ecb_decrypt(struct skcipher_request * req)177 static int ecb_decrypt(struct skcipher_request *req)
178 {
179 return glue_ecb_req_128bit(&twofish_dec, req);
180 }
181
cbc_encrypt(struct skcipher_request * req)182 static int cbc_encrypt(struct skcipher_request *req)
183 {
184 return glue_cbc_encrypt_req_128bit(twofish_enc_blk, req);
185 }
186
cbc_decrypt(struct skcipher_request * req)187 static int cbc_decrypt(struct skcipher_request *req)
188 {
189 return glue_cbc_decrypt_req_128bit(&twofish_dec_cbc, req);
190 }
191
ctr_crypt(struct skcipher_request * req)192 static int ctr_crypt(struct skcipher_request *req)
193 {
194 return glue_ctr_req_128bit(&twofish_ctr, req);
195 }
196
xts_encrypt(struct skcipher_request * req)197 static int xts_encrypt(struct skcipher_request *req)
198 {
199 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
200 struct twofish_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
201
202 return glue_xts_req_128bit(&twofish_enc_xts, req, twofish_enc_blk,
203 &ctx->tweak_ctx, &ctx->crypt_ctx, false);
204 }
205
xts_decrypt(struct skcipher_request * req)206 static int xts_decrypt(struct skcipher_request *req)
207 {
208 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
209 struct twofish_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
210
211 return glue_xts_req_128bit(&twofish_dec_xts, req, twofish_enc_blk,
212 &ctx->tweak_ctx, &ctx->crypt_ctx, true);
213 }
214
215 static struct skcipher_alg twofish_algs[] = {
216 {
217 .base.cra_name = "__ecb(twofish)",
218 .base.cra_driver_name = "__ecb-twofish-avx",
219 .base.cra_priority = 400,
220 .base.cra_flags = CRYPTO_ALG_INTERNAL,
221 .base.cra_blocksize = TF_BLOCK_SIZE,
222 .base.cra_ctxsize = sizeof(struct twofish_ctx),
223 .base.cra_module = THIS_MODULE,
224 .min_keysize = TF_MIN_KEY_SIZE,
225 .max_keysize = TF_MAX_KEY_SIZE,
226 .setkey = twofish_setkey_skcipher,
227 .encrypt = ecb_encrypt,
228 .decrypt = ecb_decrypt,
229 }, {
230 .base.cra_name = "__cbc(twofish)",
231 .base.cra_driver_name = "__cbc-twofish-avx",
232 .base.cra_priority = 400,
233 .base.cra_flags = CRYPTO_ALG_INTERNAL,
234 .base.cra_blocksize = TF_BLOCK_SIZE,
235 .base.cra_ctxsize = sizeof(struct twofish_ctx),
236 .base.cra_module = THIS_MODULE,
237 .min_keysize = TF_MIN_KEY_SIZE,
238 .max_keysize = TF_MAX_KEY_SIZE,
239 .ivsize = TF_BLOCK_SIZE,
240 .setkey = twofish_setkey_skcipher,
241 .encrypt = cbc_encrypt,
242 .decrypt = cbc_decrypt,
243 }, {
244 .base.cra_name = "__ctr(twofish)",
245 .base.cra_driver_name = "__ctr-twofish-avx",
246 .base.cra_priority = 400,
247 .base.cra_flags = CRYPTO_ALG_INTERNAL,
248 .base.cra_blocksize = 1,
249 .base.cra_ctxsize = sizeof(struct twofish_ctx),
250 .base.cra_module = THIS_MODULE,
251 .min_keysize = TF_MIN_KEY_SIZE,
252 .max_keysize = TF_MAX_KEY_SIZE,
253 .ivsize = TF_BLOCK_SIZE,
254 .chunksize = TF_BLOCK_SIZE,
255 .setkey = twofish_setkey_skcipher,
256 .encrypt = ctr_crypt,
257 .decrypt = ctr_crypt,
258 }, {
259 .base.cra_name = "__xts(twofish)",
260 .base.cra_driver_name = "__xts-twofish-avx",
261 .base.cra_priority = 400,
262 .base.cra_flags = CRYPTO_ALG_INTERNAL,
263 .base.cra_blocksize = TF_BLOCK_SIZE,
264 .base.cra_ctxsize = sizeof(struct twofish_xts_ctx),
265 .base.cra_module = THIS_MODULE,
266 .min_keysize = 2 * TF_MIN_KEY_SIZE,
267 .max_keysize = 2 * TF_MAX_KEY_SIZE,
268 .ivsize = TF_BLOCK_SIZE,
269 .setkey = xts_twofish_setkey,
270 .encrypt = xts_encrypt,
271 .decrypt = xts_decrypt,
272 },
273 };
274
275 static struct simd_skcipher_alg *twofish_simd_algs[ARRAY_SIZE(twofish_algs)];
276
twofish_init(void)277 static int __init twofish_init(void)
278 {
279 const char *feature_name;
280
281 if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, &feature_name)) {
282 pr_info("CPU feature '%s' is not supported.\n", feature_name);
283 return -ENODEV;
284 }
285
286 return simd_register_skciphers_compat(twofish_algs,
287 ARRAY_SIZE(twofish_algs),
288 twofish_simd_algs);
289 }
290
twofish_exit(void)291 static void __exit twofish_exit(void)
292 {
293 simd_unregister_skciphers(twofish_algs, ARRAY_SIZE(twofish_algs),
294 twofish_simd_algs);
295 }
296
297 module_init(twofish_init);
298 module_exit(twofish_exit);
299
300 MODULE_DESCRIPTION("Twofish Cipher Algorithm, AVX optimized");
301 MODULE_LICENSE("GPL");
302 MODULE_ALIAS_CRYPTO("twofish");
303