1
2 /*
3 * Copyright (C) 2016 Cavium, Inc.
4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms of version 2 of the GNU General Public License
7 * as published by the Free Software Foundation.
8 */
9
10 #include <crypto/aes.h>
11 #include <crypto/algapi.h>
12 #include <crypto/authenc.h>
13 #include <crypto/cryptd.h>
14 #include <crypto/crypto_wq.h>
15 #include <crypto/des.h>
16 #include <crypto/xts.h>
17 #include <linux/crypto.h>
18 #include <linux/err.h>
19 #include <linux/list.h>
20 #include <linux/scatterlist.h>
21
22 #include "cptvf.h"
23 #include "cptvf_algs.h"
24
25 struct cpt_device_handle {
26 void *cdev[MAX_DEVICES];
27 u32 dev_count;
28 };
29
30 static struct cpt_device_handle dev_handle;
31
cvm_callback(u32 status,void * arg)32 static void cvm_callback(u32 status, void *arg)
33 {
34 struct crypto_async_request *req = (struct crypto_async_request *)arg;
35
36 req->complete(req, !status);
37 }
38
update_input_iv(struct cpt_request_info * req_info,u8 * iv,u32 enc_iv_len,u32 * argcnt)39 static inline void update_input_iv(struct cpt_request_info *req_info,
40 u8 *iv, u32 enc_iv_len,
41 u32 *argcnt)
42 {
43 /* Setting the iv information */
44 req_info->in[*argcnt].vptr = (void *)iv;
45 req_info->in[*argcnt].size = enc_iv_len;
46 req_info->req.dlen += enc_iv_len;
47
48 ++(*argcnt);
49 }
50
update_output_iv(struct cpt_request_info * req_info,u8 * iv,u32 enc_iv_len,u32 * argcnt)51 static inline void update_output_iv(struct cpt_request_info *req_info,
52 u8 *iv, u32 enc_iv_len,
53 u32 *argcnt)
54 {
55 /* Setting the iv information */
56 req_info->out[*argcnt].vptr = (void *)iv;
57 req_info->out[*argcnt].size = enc_iv_len;
58 req_info->rlen += enc_iv_len;
59
60 ++(*argcnt);
61 }
62
update_input_data(struct cpt_request_info * req_info,struct scatterlist * inp_sg,u32 nbytes,u32 * argcnt)63 static inline void update_input_data(struct cpt_request_info *req_info,
64 struct scatterlist *inp_sg,
65 u32 nbytes, u32 *argcnt)
66 {
67 req_info->req.dlen += nbytes;
68
69 while (nbytes) {
70 u32 len = min(nbytes, inp_sg->length);
71 u8 *ptr = sg_virt(inp_sg);
72
73 req_info->in[*argcnt].vptr = (void *)ptr;
74 req_info->in[*argcnt].size = len;
75 nbytes -= len;
76
77 ++(*argcnt);
78 ++inp_sg;
79 }
80 }
81
update_output_data(struct cpt_request_info * req_info,struct scatterlist * outp_sg,u32 nbytes,u32 * argcnt)82 static inline void update_output_data(struct cpt_request_info *req_info,
83 struct scatterlist *outp_sg,
84 u32 nbytes, u32 *argcnt)
85 {
86 req_info->rlen += nbytes;
87
88 while (nbytes) {
89 u32 len = min(nbytes, outp_sg->length);
90 u8 *ptr = sg_virt(outp_sg);
91
92 req_info->out[*argcnt].vptr = (void *)ptr;
93 req_info->out[*argcnt].size = len;
94 nbytes -= len;
95 ++(*argcnt);
96 ++outp_sg;
97 }
98 }
99
create_ctx_hdr(struct ablkcipher_request * req,u32 enc,u32 * argcnt)100 static inline u32 create_ctx_hdr(struct ablkcipher_request *req, u32 enc,
101 u32 *argcnt)
102 {
103 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
104 struct cvm_enc_ctx *ctx = crypto_ablkcipher_ctx(tfm);
105 struct cvm_req_ctx *rctx = ablkcipher_request_ctx(req);
106 struct fc_context *fctx = &rctx->fctx;
107 u64 *offset_control = &rctx->control_word;
108 u32 enc_iv_len = crypto_ablkcipher_ivsize(tfm);
109 struct cpt_request_info *req_info = &rctx->cpt_req;
110 u64 *ctrl_flags = NULL;
111
112 req_info->ctrl.s.grp = 0;
113 req_info->ctrl.s.dma_mode = DMA_GATHER_SCATTER;
114 req_info->ctrl.s.se_req = SE_CORE_REQ;
115
116 req_info->req.opcode.s.major = MAJOR_OP_FC |
117 DMA_MODE_FLAG(DMA_GATHER_SCATTER);
118 if (enc)
119 req_info->req.opcode.s.minor = 2;
120 else
121 req_info->req.opcode.s.minor = 3;
122
123 req_info->req.param1 = req->nbytes; /* Encryption Data length */
124 req_info->req.param2 = 0; /*Auth data length */
125
126 fctx->enc.enc_ctrl.e.enc_cipher = ctx->cipher_type;
127 fctx->enc.enc_ctrl.e.aes_key = ctx->key_type;
128 fctx->enc.enc_ctrl.e.iv_source = FROM_DPTR;
129
130 if (ctx->cipher_type == AES_XTS)
131 memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len * 2);
132 else
133 memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len);
134 ctrl_flags = (u64 *)&fctx->enc.enc_ctrl.flags;
135 *ctrl_flags = cpu_to_be64(*ctrl_flags);
136
137 *offset_control = cpu_to_be64(((u64)(enc_iv_len) << 16));
138 /* Storing Packet Data Information in offset
139 * Control Word First 8 bytes
140 */
141 req_info->in[*argcnt].vptr = (u8 *)offset_control;
142 req_info->in[*argcnt].size = CONTROL_WORD_LEN;
143 req_info->req.dlen += CONTROL_WORD_LEN;
144 ++(*argcnt);
145
146 req_info->in[*argcnt].vptr = (u8 *)fctx;
147 req_info->in[*argcnt].size = sizeof(struct fc_context);
148 req_info->req.dlen += sizeof(struct fc_context);
149
150 ++(*argcnt);
151
152 return 0;
153 }
154
create_input_list(struct ablkcipher_request * req,u32 enc,u32 enc_iv_len)155 static inline u32 create_input_list(struct ablkcipher_request *req, u32 enc,
156 u32 enc_iv_len)
157 {
158 struct cvm_req_ctx *rctx = ablkcipher_request_ctx(req);
159 struct cpt_request_info *req_info = &rctx->cpt_req;
160 u32 argcnt = 0;
161
162 create_ctx_hdr(req, enc, &argcnt);
163 update_input_iv(req_info, req->info, enc_iv_len, &argcnt);
164 update_input_data(req_info, req->src, req->nbytes, &argcnt);
165 req_info->incnt = argcnt;
166
167 return 0;
168 }
169
store_cb_info(struct ablkcipher_request * req,struct cpt_request_info * req_info)170 static inline void store_cb_info(struct ablkcipher_request *req,
171 struct cpt_request_info *req_info)
172 {
173 req_info->callback = (void *)cvm_callback;
174 req_info->callback_arg = (void *)&req->base;
175 }
176
create_output_list(struct ablkcipher_request * req,u32 enc_iv_len)177 static inline void create_output_list(struct ablkcipher_request *req,
178 u32 enc_iv_len)
179 {
180 struct cvm_req_ctx *rctx = ablkcipher_request_ctx(req);
181 struct cpt_request_info *req_info = &rctx->cpt_req;
182 u32 argcnt = 0;
183
184 /* OUTPUT Buffer Processing
185 * AES encryption/decryption output would be
186 * received in the following format
187 *
188 * ------IV--------|------ENCRYPTED/DECRYPTED DATA-----|
189 * [ 16 Bytes/ [ Request Enc/Dec/ DATA Len AES CBC ]
190 */
191 /* Reading IV information */
192 update_output_iv(req_info, req->info, enc_iv_len, &argcnt);
193 update_output_data(req_info, req->dst, req->nbytes, &argcnt);
194 req_info->outcnt = argcnt;
195 }
196
cvm_enc_dec(struct ablkcipher_request * req,u32 enc)197 static inline int cvm_enc_dec(struct ablkcipher_request *req, u32 enc)
198 {
199 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
200 struct cvm_req_ctx *rctx = ablkcipher_request_ctx(req);
201 u32 enc_iv_len = crypto_ablkcipher_ivsize(tfm);
202 struct fc_context *fctx = &rctx->fctx;
203 struct cpt_request_info *req_info = &rctx->cpt_req;
204 void *cdev = NULL;
205 int status;
206
207 memset(req_info, 0, sizeof(struct cpt_request_info));
208 req_info->may_sleep = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) != 0;
209 memset(fctx, 0, sizeof(struct fc_context));
210 create_input_list(req, enc, enc_iv_len);
211 create_output_list(req, enc_iv_len);
212 store_cb_info(req, req_info);
213 cdev = dev_handle.cdev[smp_processor_id()];
214 status = cptvf_do_request(cdev, req_info);
215 /* We perform an asynchronous send and once
216 * the request is completed the driver would
217 * intimate through registered call back functions
218 */
219
220 if (status)
221 return status;
222 else
223 return -EINPROGRESS;
224 }
225
cvm_encrypt(struct ablkcipher_request * req)226 static int cvm_encrypt(struct ablkcipher_request *req)
227 {
228 return cvm_enc_dec(req, true);
229 }
230
cvm_decrypt(struct ablkcipher_request * req)231 static int cvm_decrypt(struct ablkcipher_request *req)
232 {
233 return cvm_enc_dec(req, false);
234 }
235
cvm_xts_setkey(struct crypto_ablkcipher * cipher,const u8 * key,u32 keylen)236 static int cvm_xts_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
237 u32 keylen)
238 {
239 struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
240 struct cvm_enc_ctx *ctx = crypto_tfm_ctx(tfm);
241 int err;
242 const u8 *key1 = key;
243 const u8 *key2 = key + (keylen / 2);
244
245 err = xts_check_key(tfm, key, keylen);
246 if (err)
247 return err;
248 ctx->key_len = keylen;
249 memcpy(ctx->enc_key, key1, keylen / 2);
250 memcpy(ctx->enc_key + KEY2_OFFSET, key2, keylen / 2);
251 ctx->cipher_type = AES_XTS;
252 switch (ctx->key_len) {
253 case 32:
254 ctx->key_type = AES_128_BIT;
255 break;
256 case 64:
257 ctx->key_type = AES_256_BIT;
258 break;
259 default:
260 return -EINVAL;
261 }
262
263 return 0;
264 }
265
cvm_validate_keylen(struct cvm_enc_ctx * ctx,u32 keylen)266 static int cvm_validate_keylen(struct cvm_enc_ctx *ctx, u32 keylen)
267 {
268 if ((keylen == 16) || (keylen == 24) || (keylen == 32)) {
269 ctx->key_len = keylen;
270 switch (ctx->key_len) {
271 case 16:
272 ctx->key_type = AES_128_BIT;
273 break;
274 case 24:
275 ctx->key_type = AES_192_BIT;
276 break;
277 case 32:
278 ctx->key_type = AES_256_BIT;
279 break;
280 default:
281 return -EINVAL;
282 }
283
284 if (ctx->cipher_type == DES3_CBC)
285 ctx->key_type = 0;
286
287 return 0;
288 }
289
290 return -EINVAL;
291 }
292
cvm_setkey(struct crypto_ablkcipher * cipher,const u8 * key,u32 keylen,u8 cipher_type)293 static int cvm_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
294 u32 keylen, u8 cipher_type)
295 {
296 struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
297 struct cvm_enc_ctx *ctx = crypto_tfm_ctx(tfm);
298
299 ctx->cipher_type = cipher_type;
300 if (!cvm_validate_keylen(ctx, keylen)) {
301 memcpy(ctx->enc_key, key, keylen);
302 return 0;
303 } else {
304 crypto_ablkcipher_set_flags(cipher,
305 CRYPTO_TFM_RES_BAD_KEY_LEN);
306 return -EINVAL;
307 }
308 }
309
cvm_cbc_aes_setkey(struct crypto_ablkcipher * cipher,const u8 * key,u32 keylen)310 static int cvm_cbc_aes_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
311 u32 keylen)
312 {
313 return cvm_setkey(cipher, key, keylen, AES_CBC);
314 }
315
cvm_ecb_aes_setkey(struct crypto_ablkcipher * cipher,const u8 * key,u32 keylen)316 static int cvm_ecb_aes_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
317 u32 keylen)
318 {
319 return cvm_setkey(cipher, key, keylen, AES_ECB);
320 }
321
cvm_cfb_aes_setkey(struct crypto_ablkcipher * cipher,const u8 * key,u32 keylen)322 static int cvm_cfb_aes_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
323 u32 keylen)
324 {
325 return cvm_setkey(cipher, key, keylen, AES_CFB);
326 }
327
cvm_cbc_des3_setkey(struct crypto_ablkcipher * cipher,const u8 * key,u32 keylen)328 static int cvm_cbc_des3_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
329 u32 keylen)
330 {
331 return cvm_setkey(cipher, key, keylen, DES3_CBC);
332 }
333
cvm_ecb_des3_setkey(struct crypto_ablkcipher * cipher,const u8 * key,u32 keylen)334 static int cvm_ecb_des3_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
335 u32 keylen)
336 {
337 return cvm_setkey(cipher, key, keylen, DES3_ECB);
338 }
339
cvm_enc_dec_init(struct crypto_tfm * tfm)340 static int cvm_enc_dec_init(struct crypto_tfm *tfm)
341 {
342 struct cvm_enc_ctx *ctx = crypto_tfm_ctx(tfm);
343
344 memset(ctx, 0, sizeof(*ctx));
345 tfm->crt_ablkcipher.reqsize = sizeof(struct cvm_req_ctx) +
346 sizeof(struct ablkcipher_request);
347 /* Additional memory for ablkcipher_request is
348 * allocated since the cryptd daemon uses
349 * this memory for request_ctx information
350 */
351
352 return 0;
353 }
354
355 static struct crypto_alg algs[] = { {
356 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
357 .cra_blocksize = AES_BLOCK_SIZE,
358 .cra_ctxsize = sizeof(struct cvm_enc_ctx),
359 .cra_alignmask = 7,
360 .cra_priority = 4001,
361 .cra_name = "xts(aes)",
362 .cra_driver_name = "cavium-xts-aes",
363 .cra_type = &crypto_ablkcipher_type,
364 .cra_u = {
365 .ablkcipher = {
366 .ivsize = AES_BLOCK_SIZE,
367 .min_keysize = 2 * AES_MIN_KEY_SIZE,
368 .max_keysize = 2 * AES_MAX_KEY_SIZE,
369 .setkey = cvm_xts_setkey,
370 .encrypt = cvm_encrypt,
371 .decrypt = cvm_decrypt,
372 },
373 },
374 .cra_init = cvm_enc_dec_init,
375 .cra_module = THIS_MODULE,
376 }, {
377 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
378 .cra_blocksize = AES_BLOCK_SIZE,
379 .cra_ctxsize = sizeof(struct cvm_enc_ctx),
380 .cra_alignmask = 7,
381 .cra_priority = 4001,
382 .cra_name = "cbc(aes)",
383 .cra_driver_name = "cavium-cbc-aes",
384 .cra_type = &crypto_ablkcipher_type,
385 .cra_u = {
386 .ablkcipher = {
387 .ivsize = AES_BLOCK_SIZE,
388 .min_keysize = AES_MIN_KEY_SIZE,
389 .max_keysize = AES_MAX_KEY_SIZE,
390 .setkey = cvm_cbc_aes_setkey,
391 .encrypt = cvm_encrypt,
392 .decrypt = cvm_decrypt,
393 },
394 },
395 .cra_init = cvm_enc_dec_init,
396 .cra_module = THIS_MODULE,
397 }, {
398 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
399 .cra_blocksize = AES_BLOCK_SIZE,
400 .cra_ctxsize = sizeof(struct cvm_enc_ctx),
401 .cra_alignmask = 7,
402 .cra_priority = 4001,
403 .cra_name = "ecb(aes)",
404 .cra_driver_name = "cavium-ecb-aes",
405 .cra_type = &crypto_ablkcipher_type,
406 .cra_u = {
407 .ablkcipher = {
408 .ivsize = AES_BLOCK_SIZE,
409 .min_keysize = AES_MIN_KEY_SIZE,
410 .max_keysize = AES_MAX_KEY_SIZE,
411 .setkey = cvm_ecb_aes_setkey,
412 .encrypt = cvm_encrypt,
413 .decrypt = cvm_decrypt,
414 },
415 },
416 .cra_init = cvm_enc_dec_init,
417 .cra_module = THIS_MODULE,
418 }, {
419 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
420 .cra_blocksize = AES_BLOCK_SIZE,
421 .cra_ctxsize = sizeof(struct cvm_enc_ctx),
422 .cra_alignmask = 7,
423 .cra_priority = 4001,
424 .cra_name = "cfb(aes)",
425 .cra_driver_name = "cavium-cfb-aes",
426 .cra_type = &crypto_ablkcipher_type,
427 .cra_u = {
428 .ablkcipher = {
429 .ivsize = AES_BLOCK_SIZE,
430 .min_keysize = AES_MIN_KEY_SIZE,
431 .max_keysize = AES_MAX_KEY_SIZE,
432 .setkey = cvm_cfb_aes_setkey,
433 .encrypt = cvm_encrypt,
434 .decrypt = cvm_decrypt,
435 },
436 },
437 .cra_init = cvm_enc_dec_init,
438 .cra_module = THIS_MODULE,
439 }, {
440 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
441 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
442 .cra_ctxsize = sizeof(struct cvm_des3_ctx),
443 .cra_alignmask = 7,
444 .cra_priority = 4001,
445 .cra_name = "cbc(des3_ede)",
446 .cra_driver_name = "cavium-cbc-des3_ede",
447 .cra_type = &crypto_ablkcipher_type,
448 .cra_u = {
449 .ablkcipher = {
450 .min_keysize = DES3_EDE_KEY_SIZE,
451 .max_keysize = DES3_EDE_KEY_SIZE,
452 .ivsize = DES_BLOCK_SIZE,
453 .setkey = cvm_cbc_des3_setkey,
454 .encrypt = cvm_encrypt,
455 .decrypt = cvm_decrypt,
456 },
457 },
458 .cra_init = cvm_enc_dec_init,
459 .cra_module = THIS_MODULE,
460 }, {
461 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
462 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
463 .cra_ctxsize = sizeof(struct cvm_des3_ctx),
464 .cra_alignmask = 7,
465 .cra_priority = 4001,
466 .cra_name = "ecb(des3_ede)",
467 .cra_driver_name = "cavium-ecb-des3_ede",
468 .cra_type = &crypto_ablkcipher_type,
469 .cra_u = {
470 .ablkcipher = {
471 .min_keysize = DES3_EDE_KEY_SIZE,
472 .max_keysize = DES3_EDE_KEY_SIZE,
473 .ivsize = DES_BLOCK_SIZE,
474 .setkey = cvm_ecb_des3_setkey,
475 .encrypt = cvm_encrypt,
476 .decrypt = cvm_decrypt,
477 },
478 },
479 .cra_init = cvm_enc_dec_init,
480 .cra_module = THIS_MODULE,
481 } };
482
cav_register_algs(void)483 static inline int cav_register_algs(void)
484 {
485 int err = 0;
486
487 err = crypto_register_algs(algs, ARRAY_SIZE(algs));
488 if (err)
489 return err;
490
491 return 0;
492 }
493
cav_unregister_algs(void)494 static inline void cav_unregister_algs(void)
495 {
496 crypto_unregister_algs(algs, ARRAY_SIZE(algs));
497 }
498
cvm_crypto_init(struct cpt_vf * cptvf)499 int cvm_crypto_init(struct cpt_vf *cptvf)
500 {
501 struct pci_dev *pdev = cptvf->pdev;
502 u32 dev_count;
503
504 dev_count = dev_handle.dev_count;
505 dev_handle.cdev[dev_count] = cptvf;
506 dev_handle.dev_count++;
507
508 if (dev_count == 3) {
509 if (cav_register_algs()) {
510 dev_err(&pdev->dev, "Error in registering crypto algorithms\n");
511 return -EINVAL;
512 }
513 }
514
515 return 0;
516 }
517
cvm_crypto_exit(void)518 void cvm_crypto_exit(void)
519 {
520 u32 dev_count;
521
522 dev_count = --dev_handle.dev_count;
523 if (!dev_count)
524 cav_unregister_algs();
525 }
526