1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2019 HiSilicon Limited. */
3 #include <crypto/akcipher.h>
4 #include <crypto/curve25519.h>
5 #include <crypto/dh.h>
6 #include <crypto/ecc_curve.h>
7 #include <crypto/ecdh.h>
8 #include <crypto/rng.h>
9 #include <crypto/internal/akcipher.h>
10 #include <crypto/internal/kpp.h>
11 #include <crypto/internal/rsa.h>
12 #include <crypto/kpp.h>
13 #include <crypto/scatterwalk.h>
14 #include <linux/dma-mapping.h>
15 #include <linux/fips.h>
16 #include <linux/module.h>
17 #include <linux/time.h>
18 #include "hpre.h"
19
20 struct hpre_ctx;
21
22 #define HPRE_CRYPTO_ALG_PRI 1000
23 #define HPRE_ALIGN_SZ 64
24 #define HPRE_BITS_2_BYTES_SHIFT 3
25 #define HPRE_RSA_512BITS_KSZ 64
26 #define HPRE_RSA_1536BITS_KSZ 192
27 #define HPRE_CRT_PRMS 5
28 #define HPRE_CRT_Q 2
29 #define HPRE_CRT_P 3
30 #define HPRE_CRT_INV 4
31 #define HPRE_DH_G_FLAG 0x02
32 #define HPRE_TRY_SEND_TIMES 100
33 #define HPRE_INVLD_REQ_ID (-1)
34
35 #define HPRE_SQE_ALG_BITS 5
36 #define HPRE_SQE_DONE_SHIFT 30
37 #define HPRE_DH_MAX_P_SZ 512
38
39 #define HPRE_DFX_SEC_TO_US 1000000
40 #define HPRE_DFX_US_TO_NS 1000
41
42 /* due to nist p521 */
43 #define HPRE_ECC_MAX_KSZ 66
44
45 /* size in bytes of the n prime */
46 #define HPRE_ECC_NIST_P192_N_SIZE 24
47 #define HPRE_ECC_NIST_P256_N_SIZE 32
48 #define HPRE_ECC_NIST_P384_N_SIZE 48
49
50 /* size in bytes */
51 #define HPRE_ECC_HW256_KSZ_B 32
52 #define HPRE_ECC_HW384_KSZ_B 48
53
54 /* capability register mask of driver */
55 #define HPRE_DRV_RSA_MASK_CAP BIT(0)
56 #define HPRE_DRV_DH_MASK_CAP BIT(1)
57 #define HPRE_DRV_ECDH_MASK_CAP BIT(2)
58 #define HPRE_DRV_X25519_MASK_CAP BIT(5)
59
60 static DEFINE_MUTEX(hpre_algs_lock);
61 static unsigned int hpre_available_devs;
62
63 typedef void (*hpre_cb)(struct hpre_ctx *ctx, void *sqe);
64
65 struct hpre_rsa_ctx {
66 /* low address: e--->n */
67 char *pubkey;
68 dma_addr_t dma_pubkey;
69
70 /* low address: d--->n */
71 char *prikey;
72 dma_addr_t dma_prikey;
73
74 /* low address: dq->dp->q->p->qinv */
75 char *crt_prikey;
76 dma_addr_t dma_crt_prikey;
77
78 struct crypto_akcipher *soft_tfm;
79 };
80
81 struct hpre_dh_ctx {
82 /*
83 * If base is g we compute the public key
84 * ya = g^xa mod p; [RFC2631 sec 2.1.1]
85 * else if base if the counterpart public key we
86 * compute the shared secret
87 * ZZ = yb^xa mod p; [RFC2631 sec 2.1.1]
88 * low address: d--->n, please refer to Hisilicon HPRE UM
89 */
90 char *xa_p;
91 dma_addr_t dma_xa_p;
92
93 char *g; /* m */
94 dma_addr_t dma_g;
95 };
96
97 struct hpre_ecdh_ctx {
98 /* low address: p->a->k->b */
99 unsigned char *p;
100 dma_addr_t dma_p;
101
102 /* low address: x->y */
103 unsigned char *g;
104 dma_addr_t dma_g;
105 };
106
107 struct hpre_curve25519_ctx {
108 /* low address: p->a->k */
109 unsigned char *p;
110 dma_addr_t dma_p;
111
112 /* gx coordinate */
113 unsigned char *g;
114 dma_addr_t dma_g;
115 };
116
117 struct hpre_ctx {
118 struct hisi_qp *qp;
119 struct device *dev;
120 struct hpre_asym_request **req_list;
121 struct hpre *hpre;
122 spinlock_t req_lock;
123 unsigned int key_sz;
124 bool crt_g2_mode;
125 struct idr req_idr;
126 union {
127 struct hpre_rsa_ctx rsa;
128 struct hpre_dh_ctx dh;
129 struct hpre_ecdh_ctx ecdh;
130 struct hpre_curve25519_ctx curve25519;
131 };
132 /* for ecc algorithms */
133 unsigned int curve_id;
134 };
135
136 struct hpre_asym_request {
137 char *src;
138 char *dst;
139 struct hpre_sqe req;
140 struct hpre_ctx *ctx;
141 union {
142 struct akcipher_request *rsa;
143 struct kpp_request *dh;
144 struct kpp_request *ecdh;
145 struct kpp_request *curve25519;
146 } areq;
147 int err;
148 int req_id;
149 hpre_cb cb;
150 struct timespec64 req_time;
151 };
152
hpre_align_sz(void)153 static inline unsigned int hpre_align_sz(void)
154 {
155 return ((crypto_dma_align() - 1) | (HPRE_ALIGN_SZ - 1)) + 1;
156 }
157
hpre_align_pd(void)158 static inline unsigned int hpre_align_pd(void)
159 {
160 return (hpre_align_sz() - 1) & ~(crypto_tfm_ctx_alignment() - 1);
161 }
162
hpre_alloc_req_id(struct hpre_ctx * ctx)163 static int hpre_alloc_req_id(struct hpre_ctx *ctx)
164 {
165 unsigned long flags;
166 int id;
167
168 spin_lock_irqsave(&ctx->req_lock, flags);
169 id = idr_alloc(&ctx->req_idr, NULL, 0, ctx->qp->sq_depth, GFP_ATOMIC);
170 spin_unlock_irqrestore(&ctx->req_lock, flags);
171
172 return id;
173 }
174
hpre_free_req_id(struct hpre_ctx * ctx,int req_id)175 static void hpre_free_req_id(struct hpre_ctx *ctx, int req_id)
176 {
177 unsigned long flags;
178
179 spin_lock_irqsave(&ctx->req_lock, flags);
180 idr_remove(&ctx->req_idr, req_id);
181 spin_unlock_irqrestore(&ctx->req_lock, flags);
182 }
183
hpre_add_req_to_ctx(struct hpre_asym_request * hpre_req)184 static int hpre_add_req_to_ctx(struct hpre_asym_request *hpre_req)
185 {
186 struct hpre_ctx *ctx;
187 struct hpre_dfx *dfx;
188 int id;
189
190 ctx = hpre_req->ctx;
191 id = hpre_alloc_req_id(ctx);
192 if (unlikely(id < 0))
193 return -EINVAL;
194
195 ctx->req_list[id] = hpre_req;
196 hpre_req->req_id = id;
197
198 dfx = ctx->hpre->debug.dfx;
199 if (atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value))
200 ktime_get_ts64(&hpre_req->req_time);
201
202 return id;
203 }
204
hpre_rm_req_from_ctx(struct hpre_asym_request * hpre_req)205 static void hpre_rm_req_from_ctx(struct hpre_asym_request *hpre_req)
206 {
207 struct hpre_ctx *ctx = hpre_req->ctx;
208 int id = hpre_req->req_id;
209
210 if (hpre_req->req_id >= 0) {
211 hpre_req->req_id = HPRE_INVLD_REQ_ID;
212 ctx->req_list[id] = NULL;
213 hpre_free_req_id(ctx, id);
214 }
215 }
216
hpre_get_qp_and_start(u8 type)217 static struct hisi_qp *hpre_get_qp_and_start(u8 type)
218 {
219 struct hisi_qp *qp;
220 int ret;
221
222 qp = hpre_create_qp(type);
223 if (!qp) {
224 pr_err("Can not create hpre qp!\n");
225 return ERR_PTR(-ENODEV);
226 }
227
228 ret = hisi_qm_start_qp(qp, 0);
229 if (ret < 0) {
230 hisi_qm_free_qps(&qp, 1);
231 pci_err(qp->qm->pdev, "Can not start qp!\n");
232 return ERR_PTR(-EINVAL);
233 }
234
235 return qp;
236 }
237
hpre_get_data_dma_addr(struct hpre_asym_request * hpre_req,struct scatterlist * data,unsigned int len,int is_src,dma_addr_t * tmp)238 static int hpre_get_data_dma_addr(struct hpre_asym_request *hpre_req,
239 struct scatterlist *data, unsigned int len,
240 int is_src, dma_addr_t *tmp)
241 {
242 struct device *dev = hpre_req->ctx->dev;
243 enum dma_data_direction dma_dir;
244
245 if (is_src) {
246 hpre_req->src = NULL;
247 dma_dir = DMA_TO_DEVICE;
248 } else {
249 hpre_req->dst = NULL;
250 dma_dir = DMA_FROM_DEVICE;
251 }
252 *tmp = dma_map_single(dev, sg_virt(data), len, dma_dir);
253 if (unlikely(dma_mapping_error(dev, *tmp))) {
254 dev_err(dev, "dma map data err!\n");
255 return -ENOMEM;
256 }
257
258 return 0;
259 }
260
hpre_prepare_dma_buf(struct hpre_asym_request * hpre_req,struct scatterlist * data,unsigned int len,int is_src,dma_addr_t * tmp)261 static int hpre_prepare_dma_buf(struct hpre_asym_request *hpre_req,
262 struct scatterlist *data, unsigned int len,
263 int is_src, dma_addr_t *tmp)
264 {
265 struct hpre_ctx *ctx = hpre_req->ctx;
266 struct device *dev = ctx->dev;
267 void *ptr;
268 int shift;
269
270 shift = ctx->key_sz - len;
271 if (unlikely(shift < 0))
272 return -EINVAL;
273
274 ptr = dma_alloc_coherent(dev, ctx->key_sz, tmp, GFP_ATOMIC);
275 if (unlikely(!ptr))
276 return -ENOMEM;
277
278 if (is_src) {
279 scatterwalk_map_and_copy(ptr + shift, data, 0, len, 0);
280 hpre_req->src = ptr;
281 } else {
282 hpre_req->dst = ptr;
283 }
284
285 return 0;
286 }
287
hpre_hw_data_init(struct hpre_asym_request * hpre_req,struct scatterlist * data,unsigned int len,int is_src,int is_dh)288 static int hpre_hw_data_init(struct hpre_asym_request *hpre_req,
289 struct scatterlist *data, unsigned int len,
290 int is_src, int is_dh)
291 {
292 struct hpre_sqe *msg = &hpre_req->req;
293 struct hpre_ctx *ctx = hpre_req->ctx;
294 dma_addr_t tmp = 0;
295 int ret;
296
297 /* when the data is dh's source, we should format it */
298 if ((sg_is_last(data) && len == ctx->key_sz) &&
299 ((is_dh && !is_src) || !is_dh))
300 ret = hpre_get_data_dma_addr(hpre_req, data, len, is_src, &tmp);
301 else
302 ret = hpre_prepare_dma_buf(hpre_req, data, len, is_src, &tmp);
303
304 if (unlikely(ret))
305 return ret;
306
307 if (is_src)
308 msg->in = cpu_to_le64(tmp);
309 else
310 msg->out = cpu_to_le64(tmp);
311
312 return 0;
313 }
314
hpre_hw_data_clr_all(struct hpre_ctx * ctx,struct hpre_asym_request * req,struct scatterlist * dst,struct scatterlist * src)315 static void hpre_hw_data_clr_all(struct hpre_ctx *ctx,
316 struct hpre_asym_request *req,
317 struct scatterlist *dst,
318 struct scatterlist *src)
319 {
320 struct device *dev = ctx->dev;
321 struct hpre_sqe *sqe = &req->req;
322 dma_addr_t tmp;
323
324 tmp = le64_to_cpu(sqe->in);
325 if (unlikely(dma_mapping_error(dev, tmp)))
326 return;
327
328 if (src) {
329 if (req->src)
330 dma_free_coherent(dev, ctx->key_sz, req->src, tmp);
331 else
332 dma_unmap_single(dev, tmp, ctx->key_sz, DMA_TO_DEVICE);
333 }
334
335 tmp = le64_to_cpu(sqe->out);
336 if (unlikely(dma_mapping_error(dev, tmp)))
337 return;
338
339 if (req->dst) {
340 if (dst)
341 scatterwalk_map_and_copy(req->dst, dst, 0,
342 ctx->key_sz, 1);
343 dma_free_coherent(dev, ctx->key_sz, req->dst, tmp);
344 } else {
345 dma_unmap_single(dev, tmp, ctx->key_sz, DMA_FROM_DEVICE);
346 }
347 }
348
hpre_alg_res_post_hf(struct hpre_ctx * ctx,struct hpre_sqe * sqe,void ** kreq)349 static int hpre_alg_res_post_hf(struct hpre_ctx *ctx, struct hpre_sqe *sqe,
350 void **kreq)
351 {
352 struct hpre_asym_request *req;
353 unsigned int err, done, alg;
354 int id;
355
356 #define HPRE_NO_HW_ERR 0
357 #define HPRE_HW_TASK_DONE 3
358 #define HREE_HW_ERR_MASK GENMASK(10, 0)
359 #define HREE_SQE_DONE_MASK GENMASK(1, 0)
360 #define HREE_ALG_TYPE_MASK GENMASK(4, 0)
361 id = (int)le16_to_cpu(sqe->tag);
362 req = ctx->req_list[id];
363 hpre_rm_req_from_ctx(req);
364 *kreq = req;
365
366 err = (le32_to_cpu(sqe->dw0) >> HPRE_SQE_ALG_BITS) &
367 HREE_HW_ERR_MASK;
368
369 done = (le32_to_cpu(sqe->dw0) >> HPRE_SQE_DONE_SHIFT) &
370 HREE_SQE_DONE_MASK;
371
372 if (likely(err == HPRE_NO_HW_ERR && done == HPRE_HW_TASK_DONE))
373 return 0;
374
375 alg = le32_to_cpu(sqe->dw0) & HREE_ALG_TYPE_MASK;
376 dev_err_ratelimited(ctx->dev, "alg[0x%x] error: done[0x%x], etype[0x%x]\n",
377 alg, done, err);
378
379 return -EINVAL;
380 }
381
hpre_ctx_set(struct hpre_ctx * ctx,struct hisi_qp * qp,int qlen)382 static int hpre_ctx_set(struct hpre_ctx *ctx, struct hisi_qp *qp, int qlen)
383 {
384 struct hpre *hpre;
385
386 if (!ctx || !qp || qlen < 0)
387 return -EINVAL;
388
389 spin_lock_init(&ctx->req_lock);
390 ctx->qp = qp;
391 ctx->dev = &qp->qm->pdev->dev;
392
393 hpre = container_of(ctx->qp->qm, struct hpre, qm);
394 ctx->hpre = hpre;
395 ctx->req_list = kcalloc(qlen, sizeof(void *), GFP_KERNEL);
396 if (!ctx->req_list)
397 return -ENOMEM;
398 ctx->key_sz = 0;
399 ctx->crt_g2_mode = false;
400 idr_init(&ctx->req_idr);
401
402 return 0;
403 }
404
hpre_ctx_clear(struct hpre_ctx * ctx,bool is_clear_all)405 static void hpre_ctx_clear(struct hpre_ctx *ctx, bool is_clear_all)
406 {
407 if (is_clear_all) {
408 idr_destroy(&ctx->req_idr);
409 kfree(ctx->req_list);
410 hisi_qm_free_qps(&ctx->qp, 1);
411 }
412
413 ctx->crt_g2_mode = false;
414 ctx->key_sz = 0;
415 }
416
hpre_is_bd_timeout(struct hpre_asym_request * req,u64 overtime_thrhld)417 static bool hpre_is_bd_timeout(struct hpre_asym_request *req,
418 u64 overtime_thrhld)
419 {
420 struct timespec64 reply_time;
421 u64 time_use_us;
422
423 ktime_get_ts64(&reply_time);
424 time_use_us = (reply_time.tv_sec - req->req_time.tv_sec) *
425 HPRE_DFX_SEC_TO_US +
426 (reply_time.tv_nsec - req->req_time.tv_nsec) /
427 HPRE_DFX_US_TO_NS;
428
429 if (time_use_us <= overtime_thrhld)
430 return false;
431
432 return true;
433 }
434
hpre_dh_cb(struct hpre_ctx * ctx,void * resp)435 static void hpre_dh_cb(struct hpre_ctx *ctx, void *resp)
436 {
437 struct hpre_dfx *dfx = ctx->hpre->debug.dfx;
438 struct hpre_asym_request *req;
439 struct kpp_request *areq;
440 u64 overtime_thrhld;
441 int ret;
442
443 ret = hpre_alg_res_post_hf(ctx, resp, (void **)&req);
444 areq = req->areq.dh;
445 areq->dst_len = ctx->key_sz;
446
447 overtime_thrhld = atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value);
448 if (overtime_thrhld && hpre_is_bd_timeout(req, overtime_thrhld))
449 atomic64_inc(&dfx[HPRE_OVER_THRHLD_CNT].value);
450
451 hpre_hw_data_clr_all(ctx, req, areq->dst, areq->src);
452 kpp_request_complete(areq, ret);
453 atomic64_inc(&dfx[HPRE_RECV_CNT].value);
454 }
455
hpre_rsa_cb(struct hpre_ctx * ctx,void * resp)456 static void hpre_rsa_cb(struct hpre_ctx *ctx, void *resp)
457 {
458 struct hpre_dfx *dfx = ctx->hpre->debug.dfx;
459 struct hpre_asym_request *req;
460 struct akcipher_request *areq;
461 u64 overtime_thrhld;
462 int ret;
463
464 ret = hpre_alg_res_post_hf(ctx, resp, (void **)&req);
465
466 overtime_thrhld = atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value);
467 if (overtime_thrhld && hpre_is_bd_timeout(req, overtime_thrhld))
468 atomic64_inc(&dfx[HPRE_OVER_THRHLD_CNT].value);
469
470 areq = req->areq.rsa;
471 areq->dst_len = ctx->key_sz;
472 hpre_hw_data_clr_all(ctx, req, areq->dst, areq->src);
473 akcipher_request_complete(areq, ret);
474 atomic64_inc(&dfx[HPRE_RECV_CNT].value);
475 }
476
hpre_alg_cb(struct hisi_qp * qp,void * resp)477 static void hpre_alg_cb(struct hisi_qp *qp, void *resp)
478 {
479 struct hpre_ctx *ctx = qp->qp_ctx;
480 struct hpre_dfx *dfx = ctx->hpre->debug.dfx;
481 struct hpre_sqe *sqe = resp;
482 struct hpre_asym_request *req = ctx->req_list[le16_to_cpu(sqe->tag)];
483
484 if (unlikely(!req)) {
485 atomic64_inc(&dfx[HPRE_INVALID_REQ_CNT].value);
486 return;
487 }
488
489 req->cb(ctx, resp);
490 }
491
hpre_stop_qp_and_put(struct hisi_qp * qp)492 static void hpre_stop_qp_and_put(struct hisi_qp *qp)
493 {
494 hisi_qm_stop_qp(qp);
495 hisi_qm_free_qps(&qp, 1);
496 }
497
hpre_ctx_init(struct hpre_ctx * ctx,u8 type)498 static int hpre_ctx_init(struct hpre_ctx *ctx, u8 type)
499 {
500 struct hisi_qp *qp;
501 int ret;
502
503 qp = hpre_get_qp_and_start(type);
504 if (IS_ERR(qp))
505 return PTR_ERR(qp);
506
507 qp->qp_ctx = ctx;
508 qp->req_cb = hpre_alg_cb;
509
510 ret = hpre_ctx_set(ctx, qp, qp->sq_depth);
511 if (ret)
512 hpre_stop_qp_and_put(qp);
513
514 return ret;
515 }
516
hpre_msg_request_set(struct hpre_ctx * ctx,void * req,bool is_rsa)517 static int hpre_msg_request_set(struct hpre_ctx *ctx, void *req, bool is_rsa)
518 {
519 struct hpre_asym_request *h_req;
520 struct hpre_sqe *msg;
521 int req_id;
522 void *tmp;
523
524 if (is_rsa) {
525 struct akcipher_request *akreq = req;
526
527 if (akreq->dst_len < ctx->key_sz) {
528 akreq->dst_len = ctx->key_sz;
529 return -EOVERFLOW;
530 }
531
532 tmp = akcipher_request_ctx(akreq);
533 h_req = PTR_ALIGN(tmp, hpre_align_sz());
534 h_req->cb = hpre_rsa_cb;
535 h_req->areq.rsa = akreq;
536 msg = &h_req->req;
537 memset(msg, 0, sizeof(*msg));
538 } else {
539 struct kpp_request *kreq = req;
540
541 if (kreq->dst_len < ctx->key_sz) {
542 kreq->dst_len = ctx->key_sz;
543 return -EOVERFLOW;
544 }
545
546 tmp = kpp_request_ctx(kreq);
547 h_req = PTR_ALIGN(tmp, hpre_align_sz());
548 h_req->cb = hpre_dh_cb;
549 h_req->areq.dh = kreq;
550 msg = &h_req->req;
551 memset(msg, 0, sizeof(*msg));
552 msg->key = cpu_to_le64(ctx->dh.dma_xa_p);
553 }
554
555 msg->in = cpu_to_le64(DMA_MAPPING_ERROR);
556 msg->out = cpu_to_le64(DMA_MAPPING_ERROR);
557 msg->dw0 |= cpu_to_le32(0x1 << HPRE_SQE_DONE_SHIFT);
558 msg->task_len1 = (ctx->key_sz >> HPRE_BITS_2_BYTES_SHIFT) - 1;
559 h_req->ctx = ctx;
560
561 req_id = hpre_add_req_to_ctx(h_req);
562 if (req_id < 0)
563 return -EBUSY;
564
565 msg->tag = cpu_to_le16((u16)req_id);
566
567 return 0;
568 }
569
hpre_send(struct hpre_ctx * ctx,struct hpre_sqe * msg)570 static int hpre_send(struct hpre_ctx *ctx, struct hpre_sqe *msg)
571 {
572 struct hpre_dfx *dfx = ctx->hpre->debug.dfx;
573 int ctr = 0;
574 int ret;
575
576 do {
577 atomic64_inc(&dfx[HPRE_SEND_CNT].value);
578 spin_lock_bh(&ctx->req_lock);
579 ret = hisi_qp_send(ctx->qp, msg);
580 spin_unlock_bh(&ctx->req_lock);
581 if (ret != -EBUSY)
582 break;
583 atomic64_inc(&dfx[HPRE_SEND_BUSY_CNT].value);
584 } while (ctr++ < HPRE_TRY_SEND_TIMES);
585
586 if (likely(!ret))
587 return ret;
588
589 if (ret != -EBUSY)
590 atomic64_inc(&dfx[HPRE_SEND_FAIL_CNT].value);
591
592 return ret;
593 }
594
hpre_dh_compute_value(struct kpp_request * req)595 static int hpre_dh_compute_value(struct kpp_request *req)
596 {
597 struct crypto_kpp *tfm = crypto_kpp_reqtfm(req);
598 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
599 void *tmp = kpp_request_ctx(req);
600 struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, hpre_align_sz());
601 struct hpre_sqe *msg = &hpre_req->req;
602 int ret;
603
604 ret = hpre_msg_request_set(ctx, req, false);
605 if (unlikely(ret))
606 return ret;
607
608 if (req->src) {
609 ret = hpre_hw_data_init(hpre_req, req->src, req->src_len, 1, 1);
610 if (unlikely(ret))
611 goto clear_all;
612 } else {
613 msg->in = cpu_to_le64(ctx->dh.dma_g);
614 }
615
616 ret = hpre_hw_data_init(hpre_req, req->dst, req->dst_len, 0, 1);
617 if (unlikely(ret))
618 goto clear_all;
619
620 if (ctx->crt_g2_mode && !req->src)
621 msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | HPRE_ALG_DH_G2);
622 else
623 msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | HPRE_ALG_DH);
624
625 /* success */
626 ret = hpre_send(ctx, msg);
627 if (likely(!ret))
628 return -EINPROGRESS;
629
630 clear_all:
631 hpre_rm_req_from_ctx(hpre_req);
632 hpre_hw_data_clr_all(ctx, hpre_req, req->dst, req->src);
633
634 return ret;
635 }
636
hpre_is_dh_params_length_valid(unsigned int key_sz)637 static int hpre_is_dh_params_length_valid(unsigned int key_sz)
638 {
639 #define _HPRE_DH_GRP1 768
640 #define _HPRE_DH_GRP2 1024
641 #define _HPRE_DH_GRP5 1536
642 #define _HPRE_DH_GRP14 2048
643 #define _HPRE_DH_GRP15 3072
644 #define _HPRE_DH_GRP16 4096
645 switch (key_sz) {
646 case _HPRE_DH_GRP1:
647 case _HPRE_DH_GRP2:
648 case _HPRE_DH_GRP5:
649 case _HPRE_DH_GRP14:
650 case _HPRE_DH_GRP15:
651 case _HPRE_DH_GRP16:
652 return 0;
653 default:
654 return -EINVAL;
655 }
656 }
657
hpre_dh_set_params(struct hpre_ctx * ctx,struct dh * params)658 static int hpre_dh_set_params(struct hpre_ctx *ctx, struct dh *params)
659 {
660 struct device *dev = ctx->dev;
661 unsigned int sz;
662
663 if (params->p_size > HPRE_DH_MAX_P_SZ)
664 return -EINVAL;
665
666 if (hpre_is_dh_params_length_valid(params->p_size <<
667 HPRE_BITS_2_BYTES_SHIFT))
668 return -EINVAL;
669
670 sz = ctx->key_sz = params->p_size;
671 ctx->dh.xa_p = dma_alloc_coherent(dev, sz << 1,
672 &ctx->dh.dma_xa_p, GFP_KERNEL);
673 if (!ctx->dh.xa_p)
674 return -ENOMEM;
675
676 memcpy(ctx->dh.xa_p + sz, params->p, sz);
677
678 /* If g equals 2 don't copy it */
679 if (params->g_size == 1 && *(char *)params->g == HPRE_DH_G_FLAG) {
680 ctx->crt_g2_mode = true;
681 return 0;
682 }
683
684 ctx->dh.g = dma_alloc_coherent(dev, sz, &ctx->dh.dma_g, GFP_KERNEL);
685 if (!ctx->dh.g) {
686 dma_free_coherent(dev, sz << 1, ctx->dh.xa_p,
687 ctx->dh.dma_xa_p);
688 ctx->dh.xa_p = NULL;
689 return -ENOMEM;
690 }
691
692 memcpy(ctx->dh.g + (sz - params->g_size), params->g, params->g_size);
693
694 return 0;
695 }
696
hpre_dh_clear_ctx(struct hpre_ctx * ctx,bool is_clear_all)697 static void hpre_dh_clear_ctx(struct hpre_ctx *ctx, bool is_clear_all)
698 {
699 struct device *dev = ctx->dev;
700 unsigned int sz = ctx->key_sz;
701
702 if (is_clear_all)
703 hisi_qm_stop_qp(ctx->qp);
704
705 if (ctx->dh.g) {
706 dma_free_coherent(dev, sz, ctx->dh.g, ctx->dh.dma_g);
707 ctx->dh.g = NULL;
708 }
709
710 if (ctx->dh.xa_p) {
711 memzero_explicit(ctx->dh.xa_p, sz);
712 dma_free_coherent(dev, sz << 1, ctx->dh.xa_p,
713 ctx->dh.dma_xa_p);
714 ctx->dh.xa_p = NULL;
715 }
716
717 hpre_ctx_clear(ctx, is_clear_all);
718 }
719
hpre_dh_set_secret(struct crypto_kpp * tfm,const void * buf,unsigned int len)720 static int hpre_dh_set_secret(struct crypto_kpp *tfm, const void *buf,
721 unsigned int len)
722 {
723 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
724 struct dh params;
725 int ret;
726
727 if (crypto_dh_decode_key(buf, len, ¶ms) < 0)
728 return -EINVAL;
729
730 /* Free old secret if any */
731 hpre_dh_clear_ctx(ctx, false);
732
733 ret = hpre_dh_set_params(ctx, ¶ms);
734 if (ret < 0)
735 goto err_clear_ctx;
736
737 memcpy(ctx->dh.xa_p + (ctx->key_sz - params.key_size), params.key,
738 params.key_size);
739
740 return 0;
741
742 err_clear_ctx:
743 hpre_dh_clear_ctx(ctx, false);
744 return ret;
745 }
746
hpre_dh_max_size(struct crypto_kpp * tfm)747 static unsigned int hpre_dh_max_size(struct crypto_kpp *tfm)
748 {
749 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
750
751 return ctx->key_sz;
752 }
753
hpre_dh_init_tfm(struct crypto_kpp * tfm)754 static int hpre_dh_init_tfm(struct crypto_kpp *tfm)
755 {
756 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
757
758 kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd());
759
760 return hpre_ctx_init(ctx, HPRE_V2_ALG_TYPE);
761 }
762
hpre_dh_exit_tfm(struct crypto_kpp * tfm)763 static void hpre_dh_exit_tfm(struct crypto_kpp *tfm)
764 {
765 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
766
767 hpre_dh_clear_ctx(ctx, true);
768 }
769
hpre_rsa_drop_leading_zeros(const char ** ptr,size_t * len)770 static void hpre_rsa_drop_leading_zeros(const char **ptr, size_t *len)
771 {
772 while (!**ptr && *len) {
773 (*ptr)++;
774 (*len)--;
775 }
776 }
777
hpre_rsa_key_size_is_support(unsigned int len)778 static bool hpre_rsa_key_size_is_support(unsigned int len)
779 {
780 unsigned int bits = len << HPRE_BITS_2_BYTES_SHIFT;
781
782 #define _RSA_1024BITS_KEY_WDTH 1024
783 #define _RSA_2048BITS_KEY_WDTH 2048
784 #define _RSA_3072BITS_KEY_WDTH 3072
785 #define _RSA_4096BITS_KEY_WDTH 4096
786
787 switch (bits) {
788 case _RSA_1024BITS_KEY_WDTH:
789 case _RSA_2048BITS_KEY_WDTH:
790 case _RSA_3072BITS_KEY_WDTH:
791 case _RSA_4096BITS_KEY_WDTH:
792 return true;
793 default:
794 return false;
795 }
796 }
797
hpre_rsa_enc(struct akcipher_request * req)798 static int hpre_rsa_enc(struct akcipher_request *req)
799 {
800 struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
801 struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm);
802 void *tmp = akcipher_request_ctx(req);
803 struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, hpre_align_sz());
804 struct hpre_sqe *msg = &hpre_req->req;
805 int ret;
806
807 /* For 512 and 1536 bits key size, use soft tfm instead */
808 if (ctx->key_sz == HPRE_RSA_512BITS_KSZ ||
809 ctx->key_sz == HPRE_RSA_1536BITS_KSZ) {
810 akcipher_request_set_tfm(req, ctx->rsa.soft_tfm);
811 ret = crypto_akcipher_encrypt(req);
812 akcipher_request_set_tfm(req, tfm);
813 return ret;
814 }
815
816 if (unlikely(!ctx->rsa.pubkey))
817 return -EINVAL;
818
819 ret = hpre_msg_request_set(ctx, req, true);
820 if (unlikely(ret))
821 return ret;
822
823 msg->dw0 |= cpu_to_le32(HPRE_ALG_NC_NCRT);
824 msg->key = cpu_to_le64(ctx->rsa.dma_pubkey);
825
826 ret = hpre_hw_data_init(hpre_req, req->src, req->src_len, 1, 0);
827 if (unlikely(ret))
828 goto clear_all;
829
830 ret = hpre_hw_data_init(hpre_req, req->dst, req->dst_len, 0, 0);
831 if (unlikely(ret))
832 goto clear_all;
833
834 /* success */
835 ret = hpre_send(ctx, msg);
836 if (likely(!ret))
837 return -EINPROGRESS;
838
839 clear_all:
840 hpre_rm_req_from_ctx(hpre_req);
841 hpre_hw_data_clr_all(ctx, hpre_req, req->dst, req->src);
842
843 return ret;
844 }
845
hpre_rsa_dec(struct akcipher_request * req)846 static int hpre_rsa_dec(struct akcipher_request *req)
847 {
848 struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
849 struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm);
850 void *tmp = akcipher_request_ctx(req);
851 struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, hpre_align_sz());
852 struct hpre_sqe *msg = &hpre_req->req;
853 int ret;
854
855 /* For 512 and 1536 bits key size, use soft tfm instead */
856 if (ctx->key_sz == HPRE_RSA_512BITS_KSZ ||
857 ctx->key_sz == HPRE_RSA_1536BITS_KSZ) {
858 akcipher_request_set_tfm(req, ctx->rsa.soft_tfm);
859 ret = crypto_akcipher_decrypt(req);
860 akcipher_request_set_tfm(req, tfm);
861 return ret;
862 }
863
864 if (unlikely(!ctx->rsa.prikey))
865 return -EINVAL;
866
867 ret = hpre_msg_request_set(ctx, req, true);
868 if (unlikely(ret))
869 return ret;
870
871 if (ctx->crt_g2_mode) {
872 msg->key = cpu_to_le64(ctx->rsa.dma_crt_prikey);
873 msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) |
874 HPRE_ALG_NC_CRT);
875 } else {
876 msg->key = cpu_to_le64(ctx->rsa.dma_prikey);
877 msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) |
878 HPRE_ALG_NC_NCRT);
879 }
880
881 ret = hpre_hw_data_init(hpre_req, req->src, req->src_len, 1, 0);
882 if (unlikely(ret))
883 goto clear_all;
884
885 ret = hpre_hw_data_init(hpre_req, req->dst, req->dst_len, 0, 0);
886 if (unlikely(ret))
887 goto clear_all;
888
889 /* success */
890 ret = hpre_send(ctx, msg);
891 if (likely(!ret))
892 return -EINPROGRESS;
893
894 clear_all:
895 hpre_rm_req_from_ctx(hpre_req);
896 hpre_hw_data_clr_all(ctx, hpre_req, req->dst, req->src);
897
898 return ret;
899 }
900
hpre_rsa_set_n(struct hpre_ctx * ctx,const char * value,size_t vlen,bool private)901 static int hpre_rsa_set_n(struct hpre_ctx *ctx, const char *value,
902 size_t vlen, bool private)
903 {
904 const char *ptr = value;
905
906 hpre_rsa_drop_leading_zeros(&ptr, &vlen);
907
908 ctx->key_sz = vlen;
909
910 /* if invalid key size provided, we use software tfm */
911 if (!hpre_rsa_key_size_is_support(ctx->key_sz))
912 return 0;
913
914 ctx->rsa.pubkey = dma_alloc_coherent(ctx->dev, vlen << 1,
915 &ctx->rsa.dma_pubkey,
916 GFP_KERNEL);
917 if (!ctx->rsa.pubkey)
918 return -ENOMEM;
919
920 if (private) {
921 ctx->rsa.prikey = dma_alloc_coherent(ctx->dev, vlen << 1,
922 &ctx->rsa.dma_prikey,
923 GFP_KERNEL);
924 if (!ctx->rsa.prikey) {
925 dma_free_coherent(ctx->dev, vlen << 1,
926 ctx->rsa.pubkey,
927 ctx->rsa.dma_pubkey);
928 ctx->rsa.pubkey = NULL;
929 return -ENOMEM;
930 }
931 memcpy(ctx->rsa.prikey + vlen, ptr, vlen);
932 }
933 memcpy(ctx->rsa.pubkey + vlen, ptr, vlen);
934
935 /* Using hardware HPRE to do RSA */
936 return 1;
937 }
938
hpre_rsa_set_e(struct hpre_ctx * ctx,const char * value,size_t vlen)939 static int hpre_rsa_set_e(struct hpre_ctx *ctx, const char *value,
940 size_t vlen)
941 {
942 const char *ptr = value;
943
944 hpre_rsa_drop_leading_zeros(&ptr, &vlen);
945
946 if (!ctx->key_sz || !vlen || vlen > ctx->key_sz)
947 return -EINVAL;
948
949 memcpy(ctx->rsa.pubkey + ctx->key_sz - vlen, ptr, vlen);
950
951 return 0;
952 }
953
hpre_rsa_set_d(struct hpre_ctx * ctx,const char * value,size_t vlen)954 static int hpre_rsa_set_d(struct hpre_ctx *ctx, const char *value,
955 size_t vlen)
956 {
957 const char *ptr = value;
958
959 hpre_rsa_drop_leading_zeros(&ptr, &vlen);
960
961 if (!ctx->key_sz || !vlen || vlen > ctx->key_sz)
962 return -EINVAL;
963
964 memcpy(ctx->rsa.prikey + ctx->key_sz - vlen, ptr, vlen);
965
966 return 0;
967 }
968
hpre_crt_para_get(char * para,size_t para_sz,const char * raw,size_t raw_sz)969 static int hpre_crt_para_get(char *para, size_t para_sz,
970 const char *raw, size_t raw_sz)
971 {
972 const char *ptr = raw;
973 size_t len = raw_sz;
974
975 hpre_rsa_drop_leading_zeros(&ptr, &len);
976 if (!len || len > para_sz)
977 return -EINVAL;
978
979 memcpy(para + para_sz - len, ptr, len);
980
981 return 0;
982 }
983
hpre_rsa_setkey_crt(struct hpre_ctx * ctx,struct rsa_key * rsa_key)984 static int hpre_rsa_setkey_crt(struct hpre_ctx *ctx, struct rsa_key *rsa_key)
985 {
986 unsigned int hlf_ksz = ctx->key_sz >> 1;
987 struct device *dev = ctx->dev;
988 u64 offset;
989 int ret;
990
991 ctx->rsa.crt_prikey = dma_alloc_coherent(dev, hlf_ksz * HPRE_CRT_PRMS,
992 &ctx->rsa.dma_crt_prikey,
993 GFP_KERNEL);
994 if (!ctx->rsa.crt_prikey)
995 return -ENOMEM;
996
997 ret = hpre_crt_para_get(ctx->rsa.crt_prikey, hlf_ksz,
998 rsa_key->dq, rsa_key->dq_sz);
999 if (ret)
1000 goto free_key;
1001
1002 offset = hlf_ksz;
1003 ret = hpre_crt_para_get(ctx->rsa.crt_prikey + offset, hlf_ksz,
1004 rsa_key->dp, rsa_key->dp_sz);
1005 if (ret)
1006 goto free_key;
1007
1008 offset = hlf_ksz * HPRE_CRT_Q;
1009 ret = hpre_crt_para_get(ctx->rsa.crt_prikey + offset, hlf_ksz,
1010 rsa_key->q, rsa_key->q_sz);
1011 if (ret)
1012 goto free_key;
1013
1014 offset = hlf_ksz * HPRE_CRT_P;
1015 ret = hpre_crt_para_get(ctx->rsa.crt_prikey + offset, hlf_ksz,
1016 rsa_key->p, rsa_key->p_sz);
1017 if (ret)
1018 goto free_key;
1019
1020 offset = hlf_ksz * HPRE_CRT_INV;
1021 ret = hpre_crt_para_get(ctx->rsa.crt_prikey + offset, hlf_ksz,
1022 rsa_key->qinv, rsa_key->qinv_sz);
1023 if (ret)
1024 goto free_key;
1025
1026 ctx->crt_g2_mode = true;
1027
1028 return 0;
1029
1030 free_key:
1031 offset = hlf_ksz * HPRE_CRT_PRMS;
1032 memzero_explicit(ctx->rsa.crt_prikey, offset);
1033 dma_free_coherent(dev, hlf_ksz * HPRE_CRT_PRMS, ctx->rsa.crt_prikey,
1034 ctx->rsa.dma_crt_prikey);
1035 ctx->rsa.crt_prikey = NULL;
1036 ctx->crt_g2_mode = false;
1037
1038 return ret;
1039 }
1040
1041 /* If it is clear all, all the resources of the QP will be cleaned. */
hpre_rsa_clear_ctx(struct hpre_ctx * ctx,bool is_clear_all)1042 static void hpre_rsa_clear_ctx(struct hpre_ctx *ctx, bool is_clear_all)
1043 {
1044 unsigned int half_key_sz = ctx->key_sz >> 1;
1045 struct device *dev = ctx->dev;
1046
1047 if (is_clear_all)
1048 hisi_qm_stop_qp(ctx->qp);
1049
1050 if (ctx->rsa.pubkey) {
1051 dma_free_coherent(dev, ctx->key_sz << 1,
1052 ctx->rsa.pubkey, ctx->rsa.dma_pubkey);
1053 ctx->rsa.pubkey = NULL;
1054 }
1055
1056 if (ctx->rsa.crt_prikey) {
1057 memzero_explicit(ctx->rsa.crt_prikey,
1058 half_key_sz * HPRE_CRT_PRMS);
1059 dma_free_coherent(dev, half_key_sz * HPRE_CRT_PRMS,
1060 ctx->rsa.crt_prikey, ctx->rsa.dma_crt_prikey);
1061 ctx->rsa.crt_prikey = NULL;
1062 }
1063
1064 if (ctx->rsa.prikey) {
1065 memzero_explicit(ctx->rsa.prikey, ctx->key_sz);
1066 dma_free_coherent(dev, ctx->key_sz << 1, ctx->rsa.prikey,
1067 ctx->rsa.dma_prikey);
1068 ctx->rsa.prikey = NULL;
1069 }
1070
1071 hpre_ctx_clear(ctx, is_clear_all);
1072 }
1073
1074 /*
1075 * we should judge if it is CRT or not,
1076 * CRT: return true, N-CRT: return false .
1077 */
hpre_is_crt_key(struct rsa_key * key)1078 static bool hpre_is_crt_key(struct rsa_key *key)
1079 {
1080 u16 len = key->p_sz + key->q_sz + key->dp_sz + key->dq_sz +
1081 key->qinv_sz;
1082
1083 #define LEN_OF_NCRT_PARA 5
1084
1085 /* N-CRT less than 5 parameters */
1086 return len > LEN_OF_NCRT_PARA;
1087 }
1088
hpre_rsa_setkey(struct hpre_ctx * ctx,const void * key,unsigned int keylen,bool private)1089 static int hpre_rsa_setkey(struct hpre_ctx *ctx, const void *key,
1090 unsigned int keylen, bool private)
1091 {
1092 struct rsa_key rsa_key;
1093 int ret;
1094
1095 hpre_rsa_clear_ctx(ctx, false);
1096
1097 if (private)
1098 ret = rsa_parse_priv_key(&rsa_key, key, keylen);
1099 else
1100 ret = rsa_parse_pub_key(&rsa_key, key, keylen);
1101 if (ret < 0)
1102 return ret;
1103
1104 ret = hpre_rsa_set_n(ctx, rsa_key.n, rsa_key.n_sz, private);
1105 if (ret <= 0)
1106 return ret;
1107
1108 if (private) {
1109 ret = hpre_rsa_set_d(ctx, rsa_key.d, rsa_key.d_sz);
1110 if (ret < 0)
1111 goto free;
1112
1113 if (hpre_is_crt_key(&rsa_key)) {
1114 ret = hpre_rsa_setkey_crt(ctx, &rsa_key);
1115 if (ret < 0)
1116 goto free;
1117 }
1118 }
1119
1120 ret = hpre_rsa_set_e(ctx, rsa_key.e, rsa_key.e_sz);
1121 if (ret < 0)
1122 goto free;
1123
1124 if ((private && !ctx->rsa.prikey) || !ctx->rsa.pubkey) {
1125 ret = -EINVAL;
1126 goto free;
1127 }
1128
1129 return 0;
1130
1131 free:
1132 hpre_rsa_clear_ctx(ctx, false);
1133 return ret;
1134 }
1135
hpre_rsa_setpubkey(struct crypto_akcipher * tfm,const void * key,unsigned int keylen)1136 static int hpre_rsa_setpubkey(struct crypto_akcipher *tfm, const void *key,
1137 unsigned int keylen)
1138 {
1139 struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm);
1140 int ret;
1141
1142 ret = crypto_akcipher_set_pub_key(ctx->rsa.soft_tfm, key, keylen);
1143 if (ret)
1144 return ret;
1145
1146 return hpre_rsa_setkey(ctx, key, keylen, false);
1147 }
1148
hpre_rsa_setprivkey(struct crypto_akcipher * tfm,const void * key,unsigned int keylen)1149 static int hpre_rsa_setprivkey(struct crypto_akcipher *tfm, const void *key,
1150 unsigned int keylen)
1151 {
1152 struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm);
1153 int ret;
1154
1155 ret = crypto_akcipher_set_priv_key(ctx->rsa.soft_tfm, key, keylen);
1156 if (ret)
1157 return ret;
1158
1159 return hpre_rsa_setkey(ctx, key, keylen, true);
1160 }
1161
hpre_rsa_max_size(struct crypto_akcipher * tfm)1162 static unsigned int hpre_rsa_max_size(struct crypto_akcipher *tfm)
1163 {
1164 struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm);
1165
1166 /* For 512 and 1536 bits key size, use soft tfm instead */
1167 if (ctx->key_sz == HPRE_RSA_512BITS_KSZ ||
1168 ctx->key_sz == HPRE_RSA_1536BITS_KSZ)
1169 return crypto_akcipher_maxsize(ctx->rsa.soft_tfm);
1170
1171 return ctx->key_sz;
1172 }
1173
hpre_rsa_init_tfm(struct crypto_akcipher * tfm)1174 static int hpre_rsa_init_tfm(struct crypto_akcipher *tfm)
1175 {
1176 struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm);
1177 int ret;
1178
1179 ctx->rsa.soft_tfm = crypto_alloc_akcipher("rsa-generic", 0, 0);
1180 if (IS_ERR(ctx->rsa.soft_tfm)) {
1181 pr_err("Can not alloc_akcipher!\n");
1182 return PTR_ERR(ctx->rsa.soft_tfm);
1183 }
1184
1185 akcipher_set_reqsize(tfm, sizeof(struct hpre_asym_request) +
1186 hpre_align_pd());
1187
1188 ret = hpre_ctx_init(ctx, HPRE_V2_ALG_TYPE);
1189 if (ret)
1190 crypto_free_akcipher(ctx->rsa.soft_tfm);
1191
1192 return ret;
1193 }
1194
hpre_rsa_exit_tfm(struct crypto_akcipher * tfm)1195 static void hpre_rsa_exit_tfm(struct crypto_akcipher *tfm)
1196 {
1197 struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm);
1198
1199 hpre_rsa_clear_ctx(ctx, true);
1200 crypto_free_akcipher(ctx->rsa.soft_tfm);
1201 }
1202
hpre_key_to_big_end(u8 * data,int len)1203 static void hpre_key_to_big_end(u8 *data, int len)
1204 {
1205 int i, j;
1206
1207 for (i = 0; i < len / 2; i++) {
1208 j = len - i - 1;
1209 swap(data[j], data[i]);
1210 }
1211 }
1212
hpre_ecc_clear_ctx(struct hpre_ctx * ctx,bool is_clear_all,bool is_ecdh)1213 static void hpre_ecc_clear_ctx(struct hpre_ctx *ctx, bool is_clear_all,
1214 bool is_ecdh)
1215 {
1216 struct device *dev = ctx->dev;
1217 unsigned int sz = ctx->key_sz;
1218 unsigned int shift = sz << 1;
1219
1220 if (is_clear_all)
1221 hisi_qm_stop_qp(ctx->qp);
1222
1223 if (is_ecdh && ctx->ecdh.p) {
1224 /* ecdh: p->a->k->b */
1225 memzero_explicit(ctx->ecdh.p + shift, sz);
1226 dma_free_coherent(dev, sz << 3, ctx->ecdh.p, ctx->ecdh.dma_p);
1227 ctx->ecdh.p = NULL;
1228 } else if (!is_ecdh && ctx->curve25519.p) {
1229 /* curve25519: p->a->k */
1230 memzero_explicit(ctx->curve25519.p + shift, sz);
1231 dma_free_coherent(dev, sz << 2, ctx->curve25519.p,
1232 ctx->curve25519.dma_p);
1233 ctx->curve25519.p = NULL;
1234 }
1235
1236 hpre_ctx_clear(ctx, is_clear_all);
1237 }
1238
1239 /*
1240 * The bits of 192/224/256/384/521 are supported by HPRE,
1241 * and convert the bits like:
1242 * bits<=256, bits=256; 256<bits<=384, bits=384; 384<bits<=576, bits=576;
1243 * If the parameter bit width is insufficient, then we fill in the
1244 * high-order zeros by soft, so TASK_LENGTH1 is 0x3/0x5/0x8;
1245 */
hpre_ecdh_supported_curve(unsigned short id)1246 static unsigned int hpre_ecdh_supported_curve(unsigned short id)
1247 {
1248 switch (id) {
1249 case ECC_CURVE_NIST_P192:
1250 case ECC_CURVE_NIST_P256:
1251 return HPRE_ECC_HW256_KSZ_B;
1252 case ECC_CURVE_NIST_P384:
1253 return HPRE_ECC_HW384_KSZ_B;
1254 default:
1255 break;
1256 }
1257
1258 return 0;
1259 }
1260
fill_curve_param(void * addr,u64 * param,unsigned int cur_sz,u8 ndigits)1261 static void fill_curve_param(void *addr, u64 *param, unsigned int cur_sz, u8 ndigits)
1262 {
1263 unsigned int sz = cur_sz - (ndigits - 1) * sizeof(u64);
1264 u8 i = 0;
1265
1266 while (i < ndigits - 1) {
1267 memcpy(addr + sizeof(u64) * i, ¶m[i], sizeof(u64));
1268 i++;
1269 }
1270
1271 memcpy(addr + sizeof(u64) * i, ¶m[ndigits - 1], sz);
1272 hpre_key_to_big_end((u8 *)addr, cur_sz);
1273 }
1274
hpre_ecdh_fill_curve(struct hpre_ctx * ctx,struct ecdh * params,unsigned int cur_sz)1275 static int hpre_ecdh_fill_curve(struct hpre_ctx *ctx, struct ecdh *params,
1276 unsigned int cur_sz)
1277 {
1278 unsigned int shifta = ctx->key_sz << 1;
1279 unsigned int shiftb = ctx->key_sz << 2;
1280 void *p = ctx->ecdh.p + ctx->key_sz - cur_sz;
1281 void *a = ctx->ecdh.p + shifta - cur_sz;
1282 void *b = ctx->ecdh.p + shiftb - cur_sz;
1283 void *x = ctx->ecdh.g + ctx->key_sz - cur_sz;
1284 void *y = ctx->ecdh.g + shifta - cur_sz;
1285 const struct ecc_curve *curve = ecc_get_curve(ctx->curve_id);
1286 char *n;
1287
1288 if (unlikely(!curve))
1289 return -EINVAL;
1290
1291 n = kzalloc(ctx->key_sz, GFP_KERNEL);
1292 if (!n)
1293 return -ENOMEM;
1294
1295 fill_curve_param(p, curve->p, cur_sz, curve->g.ndigits);
1296 fill_curve_param(a, curve->a, cur_sz, curve->g.ndigits);
1297 fill_curve_param(b, curve->b, cur_sz, curve->g.ndigits);
1298 fill_curve_param(x, curve->g.x, cur_sz, curve->g.ndigits);
1299 fill_curve_param(y, curve->g.y, cur_sz, curve->g.ndigits);
1300 fill_curve_param(n, curve->n, cur_sz, curve->g.ndigits);
1301
1302 if (params->key_size == cur_sz && memcmp(params->key, n, cur_sz) >= 0) {
1303 kfree(n);
1304 return -EINVAL;
1305 }
1306
1307 kfree(n);
1308 return 0;
1309 }
1310
hpre_ecdh_get_curvesz(unsigned short id)1311 static unsigned int hpre_ecdh_get_curvesz(unsigned short id)
1312 {
1313 switch (id) {
1314 case ECC_CURVE_NIST_P192:
1315 return HPRE_ECC_NIST_P192_N_SIZE;
1316 case ECC_CURVE_NIST_P256:
1317 return HPRE_ECC_NIST_P256_N_SIZE;
1318 case ECC_CURVE_NIST_P384:
1319 return HPRE_ECC_NIST_P384_N_SIZE;
1320 default:
1321 break;
1322 }
1323
1324 return 0;
1325 }
1326
hpre_ecdh_set_param(struct hpre_ctx * ctx,struct ecdh * params)1327 static int hpre_ecdh_set_param(struct hpre_ctx *ctx, struct ecdh *params)
1328 {
1329 struct device *dev = ctx->dev;
1330 unsigned int sz, shift, curve_sz;
1331 int ret;
1332
1333 ctx->key_sz = hpre_ecdh_supported_curve(ctx->curve_id);
1334 if (!ctx->key_sz)
1335 return -EINVAL;
1336
1337 curve_sz = hpre_ecdh_get_curvesz(ctx->curve_id);
1338 if (!curve_sz || params->key_size > curve_sz)
1339 return -EINVAL;
1340
1341 sz = ctx->key_sz;
1342
1343 if (!ctx->ecdh.p) {
1344 ctx->ecdh.p = dma_alloc_coherent(dev, sz << 3, &ctx->ecdh.dma_p,
1345 GFP_KERNEL);
1346 if (!ctx->ecdh.p)
1347 return -ENOMEM;
1348 }
1349
1350 shift = sz << 2;
1351 ctx->ecdh.g = ctx->ecdh.p + shift;
1352 ctx->ecdh.dma_g = ctx->ecdh.dma_p + shift;
1353
1354 ret = hpre_ecdh_fill_curve(ctx, params, curve_sz);
1355 if (ret) {
1356 dev_err(dev, "failed to fill curve_param, ret = %d!\n", ret);
1357 dma_free_coherent(dev, sz << 3, ctx->ecdh.p, ctx->ecdh.dma_p);
1358 ctx->ecdh.p = NULL;
1359 return ret;
1360 }
1361
1362 return 0;
1363 }
1364
hpre_key_is_zero(char * key,unsigned short key_sz)1365 static bool hpre_key_is_zero(char *key, unsigned short key_sz)
1366 {
1367 int i;
1368
1369 for (i = 0; i < key_sz; i++)
1370 if (key[i])
1371 return false;
1372
1373 return true;
1374 }
1375
ecdh_gen_privkey(struct hpre_ctx * ctx,struct ecdh * params)1376 static int ecdh_gen_privkey(struct hpre_ctx *ctx, struct ecdh *params)
1377 {
1378 struct device *dev = ctx->dev;
1379 int ret;
1380
1381 ret = crypto_get_default_rng();
1382 if (ret) {
1383 dev_err(dev, "failed to get default rng, ret = %d!\n", ret);
1384 return ret;
1385 }
1386
1387 ret = crypto_rng_get_bytes(crypto_default_rng, (u8 *)params->key,
1388 params->key_size);
1389 crypto_put_default_rng();
1390 if (ret)
1391 dev_err(dev, "failed to get rng, ret = %d!\n", ret);
1392
1393 return ret;
1394 }
1395
hpre_ecdh_set_secret(struct crypto_kpp * tfm,const void * buf,unsigned int len)1396 static int hpre_ecdh_set_secret(struct crypto_kpp *tfm, const void *buf,
1397 unsigned int len)
1398 {
1399 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
1400 unsigned int sz, sz_shift, curve_sz;
1401 struct device *dev = ctx->dev;
1402 char key[HPRE_ECC_MAX_KSZ];
1403 struct ecdh params;
1404 int ret;
1405
1406 if (crypto_ecdh_decode_key(buf, len, ¶ms) < 0) {
1407 dev_err(dev, "failed to decode ecdh key!\n");
1408 return -EINVAL;
1409 }
1410
1411 /* Use stdrng to generate private key */
1412 if (!params.key || !params.key_size) {
1413 params.key = key;
1414 curve_sz = hpre_ecdh_get_curvesz(ctx->curve_id);
1415 if (!curve_sz) {
1416 dev_err(dev, "Invalid curve size!\n");
1417 return -EINVAL;
1418 }
1419
1420 params.key_size = curve_sz - 1;
1421 ret = ecdh_gen_privkey(ctx, ¶ms);
1422 if (ret)
1423 return ret;
1424 }
1425
1426 if (hpre_key_is_zero(params.key, params.key_size)) {
1427 dev_err(dev, "Invalid hpre key!\n");
1428 return -EINVAL;
1429 }
1430
1431 hpre_ecc_clear_ctx(ctx, false, true);
1432
1433 ret = hpre_ecdh_set_param(ctx, ¶ms);
1434 if (ret < 0) {
1435 dev_err(dev, "failed to set hpre param, ret = %d!\n", ret);
1436 return ret;
1437 }
1438
1439 sz = ctx->key_sz;
1440 sz_shift = (sz << 1) + sz - params.key_size;
1441 memcpy(ctx->ecdh.p + sz_shift, params.key, params.key_size);
1442
1443 return 0;
1444 }
1445
hpre_ecdh_hw_data_clr_all(struct hpre_ctx * ctx,struct hpre_asym_request * req,struct scatterlist * dst,struct scatterlist * src)1446 static void hpre_ecdh_hw_data_clr_all(struct hpre_ctx *ctx,
1447 struct hpre_asym_request *req,
1448 struct scatterlist *dst,
1449 struct scatterlist *src)
1450 {
1451 struct device *dev = ctx->dev;
1452 struct hpre_sqe *sqe = &req->req;
1453 dma_addr_t dma;
1454
1455 dma = le64_to_cpu(sqe->in);
1456 if (unlikely(dma_mapping_error(dev, dma)))
1457 return;
1458
1459 if (src && req->src)
1460 dma_free_coherent(dev, ctx->key_sz << 2, req->src, dma);
1461
1462 dma = le64_to_cpu(sqe->out);
1463 if (unlikely(dma_mapping_error(dev, dma)))
1464 return;
1465
1466 if (req->dst)
1467 dma_free_coherent(dev, ctx->key_sz << 1, req->dst, dma);
1468 if (dst)
1469 dma_unmap_single(dev, dma, ctx->key_sz << 1, DMA_FROM_DEVICE);
1470 }
1471
hpre_ecdh_cb(struct hpre_ctx * ctx,void * resp)1472 static void hpre_ecdh_cb(struct hpre_ctx *ctx, void *resp)
1473 {
1474 unsigned int curve_sz = hpre_ecdh_get_curvesz(ctx->curve_id);
1475 struct hpre_dfx *dfx = ctx->hpre->debug.dfx;
1476 struct hpre_asym_request *req = NULL;
1477 struct kpp_request *areq;
1478 u64 overtime_thrhld;
1479 char *p;
1480 int ret;
1481
1482 ret = hpre_alg_res_post_hf(ctx, resp, (void **)&req);
1483 areq = req->areq.ecdh;
1484 areq->dst_len = ctx->key_sz << 1;
1485
1486 overtime_thrhld = atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value);
1487 if (overtime_thrhld && hpre_is_bd_timeout(req, overtime_thrhld))
1488 atomic64_inc(&dfx[HPRE_OVER_THRHLD_CNT].value);
1489
1490 /* Do unmap before data processing */
1491 hpre_ecdh_hw_data_clr_all(ctx, req, areq->dst, areq->src);
1492
1493 p = sg_virt(areq->dst);
1494 memmove(p, p + ctx->key_sz - curve_sz, curve_sz);
1495 memmove(p + curve_sz, p + areq->dst_len - curve_sz, curve_sz);
1496
1497 kpp_request_complete(areq, ret);
1498
1499 atomic64_inc(&dfx[HPRE_RECV_CNT].value);
1500 }
1501
hpre_ecdh_msg_request_set(struct hpre_ctx * ctx,struct kpp_request * req)1502 static int hpre_ecdh_msg_request_set(struct hpre_ctx *ctx,
1503 struct kpp_request *req)
1504 {
1505 struct hpre_asym_request *h_req;
1506 struct hpre_sqe *msg;
1507 int req_id;
1508 void *tmp;
1509
1510 if (req->dst_len < ctx->key_sz << 1) {
1511 req->dst_len = ctx->key_sz << 1;
1512 return -EINVAL;
1513 }
1514
1515 tmp = kpp_request_ctx(req);
1516 h_req = PTR_ALIGN(tmp, hpre_align_sz());
1517 h_req->cb = hpre_ecdh_cb;
1518 h_req->areq.ecdh = req;
1519 msg = &h_req->req;
1520 memset(msg, 0, sizeof(*msg));
1521 msg->in = cpu_to_le64(DMA_MAPPING_ERROR);
1522 msg->out = cpu_to_le64(DMA_MAPPING_ERROR);
1523 msg->key = cpu_to_le64(ctx->ecdh.dma_p);
1524
1525 msg->dw0 |= cpu_to_le32(0x1U << HPRE_SQE_DONE_SHIFT);
1526 msg->task_len1 = (ctx->key_sz >> HPRE_BITS_2_BYTES_SHIFT) - 1;
1527 h_req->ctx = ctx;
1528
1529 req_id = hpre_add_req_to_ctx(h_req);
1530 if (req_id < 0)
1531 return -EBUSY;
1532
1533 msg->tag = cpu_to_le16((u16)req_id);
1534 return 0;
1535 }
1536
hpre_ecdh_src_data_init(struct hpre_asym_request * hpre_req,struct scatterlist * data,unsigned int len)1537 static int hpre_ecdh_src_data_init(struct hpre_asym_request *hpre_req,
1538 struct scatterlist *data, unsigned int len)
1539 {
1540 struct hpre_sqe *msg = &hpre_req->req;
1541 struct hpre_ctx *ctx = hpre_req->ctx;
1542 struct device *dev = ctx->dev;
1543 unsigned int tmpshift;
1544 dma_addr_t dma = 0;
1545 void *ptr;
1546 int shift;
1547
1548 /* Src_data include gx and gy. */
1549 shift = ctx->key_sz - (len >> 1);
1550 if (unlikely(shift < 0))
1551 return -EINVAL;
1552
1553 ptr = dma_alloc_coherent(dev, ctx->key_sz << 2, &dma, GFP_KERNEL);
1554 if (unlikely(!ptr))
1555 return -ENOMEM;
1556
1557 tmpshift = ctx->key_sz << 1;
1558 scatterwalk_map_and_copy(ptr + tmpshift, data, 0, len, 0);
1559 memcpy(ptr + shift, ptr + tmpshift, len >> 1);
1560 memcpy(ptr + ctx->key_sz + shift, ptr + tmpshift + (len >> 1), len >> 1);
1561
1562 hpre_req->src = ptr;
1563 msg->in = cpu_to_le64(dma);
1564 return 0;
1565 }
1566
hpre_ecdh_dst_data_init(struct hpre_asym_request * hpre_req,struct scatterlist * data,unsigned int len)1567 static int hpre_ecdh_dst_data_init(struct hpre_asym_request *hpre_req,
1568 struct scatterlist *data, unsigned int len)
1569 {
1570 struct hpre_sqe *msg = &hpre_req->req;
1571 struct hpre_ctx *ctx = hpre_req->ctx;
1572 struct device *dev = ctx->dev;
1573 dma_addr_t dma;
1574
1575 if (unlikely(!data || !sg_is_last(data) || len != ctx->key_sz << 1)) {
1576 dev_err(dev, "data or data length is illegal!\n");
1577 return -EINVAL;
1578 }
1579
1580 hpre_req->dst = NULL;
1581 dma = dma_map_single(dev, sg_virt(data), len, DMA_FROM_DEVICE);
1582 if (unlikely(dma_mapping_error(dev, dma))) {
1583 dev_err(dev, "dma map data err!\n");
1584 return -ENOMEM;
1585 }
1586
1587 msg->out = cpu_to_le64(dma);
1588 return 0;
1589 }
1590
hpre_ecdh_compute_value(struct kpp_request * req)1591 static int hpre_ecdh_compute_value(struct kpp_request *req)
1592 {
1593 struct crypto_kpp *tfm = crypto_kpp_reqtfm(req);
1594 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
1595 struct device *dev = ctx->dev;
1596 void *tmp = kpp_request_ctx(req);
1597 struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, hpre_align_sz());
1598 struct hpre_sqe *msg = &hpre_req->req;
1599 int ret;
1600
1601 ret = hpre_ecdh_msg_request_set(ctx, req);
1602 if (unlikely(ret)) {
1603 dev_err(dev, "failed to set ecdh request, ret = %d!\n", ret);
1604 return ret;
1605 }
1606
1607 if (req->src) {
1608 ret = hpre_ecdh_src_data_init(hpre_req, req->src, req->src_len);
1609 if (unlikely(ret)) {
1610 dev_err(dev, "failed to init src data, ret = %d!\n", ret);
1611 goto clear_all;
1612 }
1613 } else {
1614 msg->in = cpu_to_le64(ctx->ecdh.dma_g);
1615 }
1616
1617 ret = hpre_ecdh_dst_data_init(hpre_req, req->dst, req->dst_len);
1618 if (unlikely(ret)) {
1619 dev_err(dev, "failed to init dst data, ret = %d!\n", ret);
1620 goto clear_all;
1621 }
1622
1623 msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | HPRE_ALG_ECC_MUL);
1624 ret = hpre_send(ctx, msg);
1625 if (likely(!ret))
1626 return -EINPROGRESS;
1627
1628 clear_all:
1629 hpre_rm_req_from_ctx(hpre_req);
1630 hpre_ecdh_hw_data_clr_all(ctx, hpre_req, req->dst, req->src);
1631 return ret;
1632 }
1633
hpre_ecdh_max_size(struct crypto_kpp * tfm)1634 static unsigned int hpre_ecdh_max_size(struct crypto_kpp *tfm)
1635 {
1636 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
1637
1638 /* max size is the pub_key_size, include x and y */
1639 return ctx->key_sz << 1;
1640 }
1641
hpre_ecdh_nist_p192_init_tfm(struct crypto_kpp * tfm)1642 static int hpre_ecdh_nist_p192_init_tfm(struct crypto_kpp *tfm)
1643 {
1644 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
1645
1646 ctx->curve_id = ECC_CURVE_NIST_P192;
1647
1648 kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd());
1649
1650 return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE);
1651 }
1652
hpre_ecdh_nist_p256_init_tfm(struct crypto_kpp * tfm)1653 static int hpre_ecdh_nist_p256_init_tfm(struct crypto_kpp *tfm)
1654 {
1655 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
1656
1657 ctx->curve_id = ECC_CURVE_NIST_P256;
1658
1659 kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd());
1660
1661 return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE);
1662 }
1663
hpre_ecdh_nist_p384_init_tfm(struct crypto_kpp * tfm)1664 static int hpre_ecdh_nist_p384_init_tfm(struct crypto_kpp *tfm)
1665 {
1666 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
1667
1668 ctx->curve_id = ECC_CURVE_NIST_P384;
1669
1670 kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd());
1671
1672 return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE);
1673 }
1674
hpre_ecdh_exit_tfm(struct crypto_kpp * tfm)1675 static void hpre_ecdh_exit_tfm(struct crypto_kpp *tfm)
1676 {
1677 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
1678
1679 hpre_ecc_clear_ctx(ctx, true, true);
1680 }
1681
hpre_curve25519_fill_curve(struct hpre_ctx * ctx,const void * buf,unsigned int len)1682 static void hpre_curve25519_fill_curve(struct hpre_ctx *ctx, const void *buf,
1683 unsigned int len)
1684 {
1685 u8 secret[CURVE25519_KEY_SIZE] = { 0 };
1686 unsigned int sz = ctx->key_sz;
1687 const struct ecc_curve *curve;
1688 unsigned int shift = sz << 1;
1689 void *p;
1690
1691 /*
1692 * The key from 'buf' is in little-endian, we should preprocess it as
1693 * the description in rfc7748: "k[0] &= 248, k[31] &= 127, k[31] |= 64",
1694 * then convert it to big endian. Only in this way, the result can be
1695 * the same as the software curve-25519 that exists in crypto.
1696 */
1697 memcpy(secret, buf, len);
1698 curve25519_clamp_secret(secret);
1699 hpre_key_to_big_end(secret, CURVE25519_KEY_SIZE);
1700
1701 p = ctx->curve25519.p + sz - len;
1702
1703 curve = ecc_get_curve25519();
1704
1705 /* fill curve parameters */
1706 fill_curve_param(p, curve->p, len, curve->g.ndigits);
1707 fill_curve_param(p + sz, curve->a, len, curve->g.ndigits);
1708 memcpy(p + shift, secret, len);
1709 fill_curve_param(p + shift + sz, curve->g.x, len, curve->g.ndigits);
1710 memzero_explicit(secret, CURVE25519_KEY_SIZE);
1711 }
1712
hpre_curve25519_set_param(struct hpre_ctx * ctx,const void * buf,unsigned int len)1713 static int hpre_curve25519_set_param(struct hpre_ctx *ctx, const void *buf,
1714 unsigned int len)
1715 {
1716 struct device *dev = ctx->dev;
1717 unsigned int sz = ctx->key_sz;
1718 unsigned int shift = sz << 1;
1719
1720 /* p->a->k->gx */
1721 if (!ctx->curve25519.p) {
1722 ctx->curve25519.p = dma_alloc_coherent(dev, sz << 2,
1723 &ctx->curve25519.dma_p,
1724 GFP_KERNEL);
1725 if (!ctx->curve25519.p)
1726 return -ENOMEM;
1727 }
1728
1729 ctx->curve25519.g = ctx->curve25519.p + shift + sz;
1730 ctx->curve25519.dma_g = ctx->curve25519.dma_p + shift + sz;
1731
1732 hpre_curve25519_fill_curve(ctx, buf, len);
1733
1734 return 0;
1735 }
1736
hpre_curve25519_set_secret(struct crypto_kpp * tfm,const void * buf,unsigned int len)1737 static int hpre_curve25519_set_secret(struct crypto_kpp *tfm, const void *buf,
1738 unsigned int len)
1739 {
1740 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
1741 struct device *dev = ctx->dev;
1742 int ret = -EINVAL;
1743
1744 if (len != CURVE25519_KEY_SIZE ||
1745 !crypto_memneq(buf, curve25519_null_point, CURVE25519_KEY_SIZE)) {
1746 dev_err(dev, "key is null or key len is not 32bytes!\n");
1747 return ret;
1748 }
1749
1750 /* Free old secret if any */
1751 hpre_ecc_clear_ctx(ctx, false, false);
1752
1753 ctx->key_sz = CURVE25519_KEY_SIZE;
1754 ret = hpre_curve25519_set_param(ctx, buf, CURVE25519_KEY_SIZE);
1755 if (ret) {
1756 dev_err(dev, "failed to set curve25519 param, ret = %d!\n", ret);
1757 hpre_ecc_clear_ctx(ctx, false, false);
1758 return ret;
1759 }
1760
1761 return 0;
1762 }
1763
hpre_curve25519_hw_data_clr_all(struct hpre_ctx * ctx,struct hpre_asym_request * req,struct scatterlist * dst,struct scatterlist * src)1764 static void hpre_curve25519_hw_data_clr_all(struct hpre_ctx *ctx,
1765 struct hpre_asym_request *req,
1766 struct scatterlist *dst,
1767 struct scatterlist *src)
1768 {
1769 struct device *dev = ctx->dev;
1770 struct hpre_sqe *sqe = &req->req;
1771 dma_addr_t dma;
1772
1773 dma = le64_to_cpu(sqe->in);
1774 if (unlikely(dma_mapping_error(dev, dma)))
1775 return;
1776
1777 if (src && req->src)
1778 dma_free_coherent(dev, ctx->key_sz, req->src, dma);
1779
1780 dma = le64_to_cpu(sqe->out);
1781 if (unlikely(dma_mapping_error(dev, dma)))
1782 return;
1783
1784 if (req->dst)
1785 dma_free_coherent(dev, ctx->key_sz, req->dst, dma);
1786 if (dst)
1787 dma_unmap_single(dev, dma, ctx->key_sz, DMA_FROM_DEVICE);
1788 }
1789
hpre_curve25519_cb(struct hpre_ctx * ctx,void * resp)1790 static void hpre_curve25519_cb(struct hpre_ctx *ctx, void *resp)
1791 {
1792 struct hpre_dfx *dfx = ctx->hpre->debug.dfx;
1793 struct hpre_asym_request *req = NULL;
1794 struct kpp_request *areq;
1795 u64 overtime_thrhld;
1796 int ret;
1797
1798 ret = hpre_alg_res_post_hf(ctx, resp, (void **)&req);
1799 areq = req->areq.curve25519;
1800 areq->dst_len = ctx->key_sz;
1801
1802 overtime_thrhld = atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value);
1803 if (overtime_thrhld && hpre_is_bd_timeout(req, overtime_thrhld))
1804 atomic64_inc(&dfx[HPRE_OVER_THRHLD_CNT].value);
1805
1806 /* Do unmap before data processing */
1807 hpre_curve25519_hw_data_clr_all(ctx, req, areq->dst, areq->src);
1808
1809 hpre_key_to_big_end(sg_virt(areq->dst), CURVE25519_KEY_SIZE);
1810
1811 kpp_request_complete(areq, ret);
1812
1813 atomic64_inc(&dfx[HPRE_RECV_CNT].value);
1814 }
1815
hpre_curve25519_msg_request_set(struct hpre_ctx * ctx,struct kpp_request * req)1816 static int hpre_curve25519_msg_request_set(struct hpre_ctx *ctx,
1817 struct kpp_request *req)
1818 {
1819 struct hpre_asym_request *h_req;
1820 struct hpre_sqe *msg;
1821 int req_id;
1822 void *tmp;
1823
1824 if (unlikely(req->dst_len < ctx->key_sz)) {
1825 req->dst_len = ctx->key_sz;
1826 return -EINVAL;
1827 }
1828
1829 tmp = kpp_request_ctx(req);
1830 h_req = PTR_ALIGN(tmp, hpre_align_sz());
1831 h_req->cb = hpre_curve25519_cb;
1832 h_req->areq.curve25519 = req;
1833 msg = &h_req->req;
1834 memset(msg, 0, sizeof(*msg));
1835 msg->in = cpu_to_le64(DMA_MAPPING_ERROR);
1836 msg->out = cpu_to_le64(DMA_MAPPING_ERROR);
1837 msg->key = cpu_to_le64(ctx->curve25519.dma_p);
1838
1839 msg->dw0 |= cpu_to_le32(0x1U << HPRE_SQE_DONE_SHIFT);
1840 msg->task_len1 = (ctx->key_sz >> HPRE_BITS_2_BYTES_SHIFT) - 1;
1841 h_req->ctx = ctx;
1842
1843 req_id = hpre_add_req_to_ctx(h_req);
1844 if (req_id < 0)
1845 return -EBUSY;
1846
1847 msg->tag = cpu_to_le16((u16)req_id);
1848 return 0;
1849 }
1850
hpre_curve25519_src_modulo_p(u8 * ptr)1851 static void hpre_curve25519_src_modulo_p(u8 *ptr)
1852 {
1853 int i;
1854
1855 for (i = 0; i < CURVE25519_KEY_SIZE - 1; i++)
1856 ptr[i] = 0;
1857
1858 /* The modulus is ptr's last byte minus '0xed'(last byte of p) */
1859 ptr[i] -= 0xed;
1860 }
1861
hpre_curve25519_src_init(struct hpre_asym_request * hpre_req,struct scatterlist * data,unsigned int len)1862 static int hpre_curve25519_src_init(struct hpre_asym_request *hpre_req,
1863 struct scatterlist *data, unsigned int len)
1864 {
1865 struct hpre_sqe *msg = &hpre_req->req;
1866 struct hpre_ctx *ctx = hpre_req->ctx;
1867 struct device *dev = ctx->dev;
1868 u8 p[CURVE25519_KEY_SIZE] = { 0 };
1869 const struct ecc_curve *curve;
1870 dma_addr_t dma = 0;
1871 u8 *ptr;
1872
1873 if (len != CURVE25519_KEY_SIZE) {
1874 dev_err(dev, "sourc_data len is not 32bytes, len = %u!\n", len);
1875 return -EINVAL;
1876 }
1877
1878 ptr = dma_alloc_coherent(dev, ctx->key_sz, &dma, GFP_KERNEL);
1879 if (unlikely(!ptr))
1880 return -ENOMEM;
1881
1882 scatterwalk_map_and_copy(ptr, data, 0, len, 0);
1883
1884 if (!crypto_memneq(ptr, curve25519_null_point, CURVE25519_KEY_SIZE)) {
1885 dev_err(dev, "gx is null!\n");
1886 goto err;
1887 }
1888
1889 /*
1890 * Src_data(gx) is in little-endian order, MSB in the final byte should
1891 * be masked as described in RFC7748, then transform it to big-endian
1892 * form, then hisi_hpre can use the data.
1893 */
1894 ptr[31] &= 0x7f;
1895 hpre_key_to_big_end(ptr, CURVE25519_KEY_SIZE);
1896
1897 curve = ecc_get_curve25519();
1898
1899 fill_curve_param(p, curve->p, CURVE25519_KEY_SIZE, curve->g.ndigits);
1900
1901 /*
1902 * When src_data equals (2^255 - 19) ~ (2^255 - 1), it is out of p,
1903 * we get its modulus to p, and then use it.
1904 */
1905 if (memcmp(ptr, p, ctx->key_sz) == 0) {
1906 dev_err(dev, "gx is p!\n");
1907 goto err;
1908 } else if (memcmp(ptr, p, ctx->key_sz) > 0) {
1909 hpre_curve25519_src_modulo_p(ptr);
1910 }
1911
1912 hpre_req->src = ptr;
1913 msg->in = cpu_to_le64(dma);
1914 return 0;
1915
1916 err:
1917 dma_free_coherent(dev, ctx->key_sz, ptr, dma);
1918 return -EINVAL;
1919 }
1920
hpre_curve25519_dst_init(struct hpre_asym_request * hpre_req,struct scatterlist * data,unsigned int len)1921 static int hpre_curve25519_dst_init(struct hpre_asym_request *hpre_req,
1922 struct scatterlist *data, unsigned int len)
1923 {
1924 struct hpre_sqe *msg = &hpre_req->req;
1925 struct hpre_ctx *ctx = hpre_req->ctx;
1926 struct device *dev = ctx->dev;
1927 dma_addr_t dma;
1928
1929 if (!data || !sg_is_last(data) || len != ctx->key_sz) {
1930 dev_err(dev, "data or data length is illegal!\n");
1931 return -EINVAL;
1932 }
1933
1934 hpre_req->dst = NULL;
1935 dma = dma_map_single(dev, sg_virt(data), len, DMA_FROM_DEVICE);
1936 if (unlikely(dma_mapping_error(dev, dma))) {
1937 dev_err(dev, "dma map data err!\n");
1938 return -ENOMEM;
1939 }
1940
1941 msg->out = cpu_to_le64(dma);
1942 return 0;
1943 }
1944
hpre_curve25519_compute_value(struct kpp_request * req)1945 static int hpre_curve25519_compute_value(struct kpp_request *req)
1946 {
1947 struct crypto_kpp *tfm = crypto_kpp_reqtfm(req);
1948 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
1949 struct device *dev = ctx->dev;
1950 void *tmp = kpp_request_ctx(req);
1951 struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, hpre_align_sz());
1952 struct hpre_sqe *msg = &hpre_req->req;
1953 int ret;
1954
1955 ret = hpre_curve25519_msg_request_set(ctx, req);
1956 if (unlikely(ret)) {
1957 dev_err(dev, "failed to set curve25519 request, ret = %d!\n", ret);
1958 return ret;
1959 }
1960
1961 if (req->src) {
1962 ret = hpre_curve25519_src_init(hpre_req, req->src, req->src_len);
1963 if (unlikely(ret)) {
1964 dev_err(dev, "failed to init src data, ret = %d!\n",
1965 ret);
1966 goto clear_all;
1967 }
1968 } else {
1969 msg->in = cpu_to_le64(ctx->curve25519.dma_g);
1970 }
1971
1972 ret = hpre_curve25519_dst_init(hpre_req, req->dst, req->dst_len);
1973 if (unlikely(ret)) {
1974 dev_err(dev, "failed to init dst data, ret = %d!\n", ret);
1975 goto clear_all;
1976 }
1977
1978 msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | HPRE_ALG_CURVE25519_MUL);
1979 ret = hpre_send(ctx, msg);
1980 if (likely(!ret))
1981 return -EINPROGRESS;
1982
1983 clear_all:
1984 hpre_rm_req_from_ctx(hpre_req);
1985 hpre_curve25519_hw_data_clr_all(ctx, hpre_req, req->dst, req->src);
1986 return ret;
1987 }
1988
hpre_curve25519_max_size(struct crypto_kpp * tfm)1989 static unsigned int hpre_curve25519_max_size(struct crypto_kpp *tfm)
1990 {
1991 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
1992
1993 return ctx->key_sz;
1994 }
1995
hpre_curve25519_init_tfm(struct crypto_kpp * tfm)1996 static int hpre_curve25519_init_tfm(struct crypto_kpp *tfm)
1997 {
1998 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
1999
2000 kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd());
2001
2002 return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE);
2003 }
2004
hpre_curve25519_exit_tfm(struct crypto_kpp * tfm)2005 static void hpre_curve25519_exit_tfm(struct crypto_kpp *tfm)
2006 {
2007 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
2008
2009 hpre_ecc_clear_ctx(ctx, true, false);
2010 }
2011
2012 static struct akcipher_alg rsa = {
2013 .sign = hpre_rsa_dec,
2014 .verify = hpre_rsa_enc,
2015 .encrypt = hpre_rsa_enc,
2016 .decrypt = hpre_rsa_dec,
2017 .set_pub_key = hpre_rsa_setpubkey,
2018 .set_priv_key = hpre_rsa_setprivkey,
2019 .max_size = hpre_rsa_max_size,
2020 .init = hpre_rsa_init_tfm,
2021 .exit = hpre_rsa_exit_tfm,
2022 .base = {
2023 .cra_ctxsize = sizeof(struct hpre_ctx),
2024 .cra_priority = HPRE_CRYPTO_ALG_PRI,
2025 .cra_name = "rsa",
2026 .cra_driver_name = "hpre-rsa",
2027 .cra_module = THIS_MODULE,
2028 },
2029 };
2030
2031 static struct kpp_alg dh = {
2032 .set_secret = hpre_dh_set_secret,
2033 .generate_public_key = hpre_dh_compute_value,
2034 .compute_shared_secret = hpre_dh_compute_value,
2035 .max_size = hpre_dh_max_size,
2036 .init = hpre_dh_init_tfm,
2037 .exit = hpre_dh_exit_tfm,
2038 .base = {
2039 .cra_ctxsize = sizeof(struct hpre_ctx),
2040 .cra_priority = HPRE_CRYPTO_ALG_PRI,
2041 .cra_name = "dh",
2042 .cra_driver_name = "hpre-dh",
2043 .cra_module = THIS_MODULE,
2044 },
2045 };
2046
2047 static struct kpp_alg ecdh_curves[] = {
2048 {
2049 .set_secret = hpre_ecdh_set_secret,
2050 .generate_public_key = hpre_ecdh_compute_value,
2051 .compute_shared_secret = hpre_ecdh_compute_value,
2052 .max_size = hpre_ecdh_max_size,
2053 .init = hpre_ecdh_nist_p192_init_tfm,
2054 .exit = hpre_ecdh_exit_tfm,
2055 .base = {
2056 .cra_ctxsize = sizeof(struct hpre_ctx),
2057 .cra_priority = HPRE_CRYPTO_ALG_PRI,
2058 .cra_name = "ecdh-nist-p192",
2059 .cra_driver_name = "hpre-ecdh-nist-p192",
2060 .cra_module = THIS_MODULE,
2061 },
2062 }, {
2063 .set_secret = hpre_ecdh_set_secret,
2064 .generate_public_key = hpre_ecdh_compute_value,
2065 .compute_shared_secret = hpre_ecdh_compute_value,
2066 .max_size = hpre_ecdh_max_size,
2067 .init = hpre_ecdh_nist_p256_init_tfm,
2068 .exit = hpre_ecdh_exit_tfm,
2069 .base = {
2070 .cra_ctxsize = sizeof(struct hpre_ctx),
2071 .cra_priority = HPRE_CRYPTO_ALG_PRI,
2072 .cra_name = "ecdh-nist-p256",
2073 .cra_driver_name = "hpre-ecdh-nist-p256",
2074 .cra_module = THIS_MODULE,
2075 },
2076 }, {
2077 .set_secret = hpre_ecdh_set_secret,
2078 .generate_public_key = hpre_ecdh_compute_value,
2079 .compute_shared_secret = hpre_ecdh_compute_value,
2080 .max_size = hpre_ecdh_max_size,
2081 .init = hpre_ecdh_nist_p384_init_tfm,
2082 .exit = hpre_ecdh_exit_tfm,
2083 .base = {
2084 .cra_ctxsize = sizeof(struct hpre_ctx),
2085 .cra_priority = HPRE_CRYPTO_ALG_PRI,
2086 .cra_name = "ecdh-nist-p384",
2087 .cra_driver_name = "hpre-ecdh-nist-p384",
2088 .cra_module = THIS_MODULE,
2089 },
2090 }
2091 };
2092
2093 static struct kpp_alg curve25519_alg = {
2094 .set_secret = hpre_curve25519_set_secret,
2095 .generate_public_key = hpre_curve25519_compute_value,
2096 .compute_shared_secret = hpre_curve25519_compute_value,
2097 .max_size = hpre_curve25519_max_size,
2098 .init = hpre_curve25519_init_tfm,
2099 .exit = hpre_curve25519_exit_tfm,
2100 .base = {
2101 .cra_ctxsize = sizeof(struct hpre_ctx),
2102 .cra_priority = HPRE_CRYPTO_ALG_PRI,
2103 .cra_name = "curve25519",
2104 .cra_driver_name = "hpre-curve25519",
2105 .cra_module = THIS_MODULE,
2106 },
2107 };
2108
hpre_register_rsa(struct hisi_qm * qm)2109 static int hpre_register_rsa(struct hisi_qm *qm)
2110 {
2111 int ret;
2112
2113 if (!hpre_check_alg_support(qm, HPRE_DRV_RSA_MASK_CAP))
2114 return 0;
2115
2116 rsa.base.cra_flags = 0;
2117 ret = crypto_register_akcipher(&rsa);
2118 if (ret)
2119 dev_err(&qm->pdev->dev, "failed to register rsa (%d)!\n", ret);
2120
2121 return ret;
2122 }
2123
hpre_unregister_rsa(struct hisi_qm * qm)2124 static void hpre_unregister_rsa(struct hisi_qm *qm)
2125 {
2126 if (!hpre_check_alg_support(qm, HPRE_DRV_RSA_MASK_CAP))
2127 return;
2128
2129 crypto_unregister_akcipher(&rsa);
2130 }
2131
hpre_register_dh(struct hisi_qm * qm)2132 static int hpre_register_dh(struct hisi_qm *qm)
2133 {
2134 int ret;
2135
2136 if (!hpre_check_alg_support(qm, HPRE_DRV_DH_MASK_CAP))
2137 return 0;
2138
2139 ret = crypto_register_kpp(&dh);
2140 if (ret)
2141 dev_err(&qm->pdev->dev, "failed to register dh (%d)!\n", ret);
2142
2143 return ret;
2144 }
2145
hpre_unregister_dh(struct hisi_qm * qm)2146 static void hpre_unregister_dh(struct hisi_qm *qm)
2147 {
2148 if (!hpre_check_alg_support(qm, HPRE_DRV_DH_MASK_CAP))
2149 return;
2150
2151 crypto_unregister_kpp(&dh);
2152 }
2153
hpre_register_ecdh(struct hisi_qm * qm)2154 static int hpre_register_ecdh(struct hisi_qm *qm)
2155 {
2156 int ret, i;
2157
2158 if (!hpre_check_alg_support(qm, HPRE_DRV_ECDH_MASK_CAP))
2159 return 0;
2160
2161 for (i = 0; i < ARRAY_SIZE(ecdh_curves); i++) {
2162 ret = crypto_register_kpp(&ecdh_curves[i]);
2163 if (ret) {
2164 dev_err(&qm->pdev->dev, "failed to register %s (%d)!\n",
2165 ecdh_curves[i].base.cra_name, ret);
2166 goto unreg_kpp;
2167 }
2168 }
2169
2170 return 0;
2171
2172 unreg_kpp:
2173 for (--i; i >= 0; --i)
2174 crypto_unregister_kpp(&ecdh_curves[i]);
2175
2176 return ret;
2177 }
2178
hpre_unregister_ecdh(struct hisi_qm * qm)2179 static void hpre_unregister_ecdh(struct hisi_qm *qm)
2180 {
2181 int i;
2182
2183 if (!hpre_check_alg_support(qm, HPRE_DRV_ECDH_MASK_CAP))
2184 return;
2185
2186 for (i = ARRAY_SIZE(ecdh_curves) - 1; i >= 0; --i)
2187 crypto_unregister_kpp(&ecdh_curves[i]);
2188 }
2189
hpre_register_x25519(struct hisi_qm * qm)2190 static int hpre_register_x25519(struct hisi_qm *qm)
2191 {
2192 int ret;
2193
2194 if (!hpre_check_alg_support(qm, HPRE_DRV_X25519_MASK_CAP))
2195 return 0;
2196
2197 ret = crypto_register_kpp(&curve25519_alg);
2198 if (ret)
2199 dev_err(&qm->pdev->dev, "failed to register x25519 (%d)!\n", ret);
2200
2201 return ret;
2202 }
2203
hpre_unregister_x25519(struct hisi_qm * qm)2204 static void hpre_unregister_x25519(struct hisi_qm *qm)
2205 {
2206 if (!hpre_check_alg_support(qm, HPRE_DRV_X25519_MASK_CAP))
2207 return;
2208
2209 crypto_unregister_kpp(&curve25519_alg);
2210 }
2211
hpre_algs_register(struct hisi_qm * qm)2212 int hpre_algs_register(struct hisi_qm *qm)
2213 {
2214 int ret = 0;
2215
2216 mutex_lock(&hpre_algs_lock);
2217 if (hpre_available_devs) {
2218 hpre_available_devs++;
2219 goto unlock;
2220 }
2221
2222 ret = hpre_register_rsa(qm);
2223 if (ret)
2224 goto unlock;
2225
2226 ret = hpre_register_dh(qm);
2227 if (ret)
2228 goto unreg_rsa;
2229
2230 ret = hpre_register_ecdh(qm);
2231 if (ret)
2232 goto unreg_dh;
2233
2234 ret = hpre_register_x25519(qm);
2235 if (ret)
2236 goto unreg_ecdh;
2237
2238 hpre_available_devs++;
2239 mutex_unlock(&hpre_algs_lock);
2240
2241 return ret;
2242
2243 unreg_ecdh:
2244 hpre_unregister_ecdh(qm);
2245 unreg_dh:
2246 hpre_unregister_dh(qm);
2247 unreg_rsa:
2248 hpre_unregister_rsa(qm);
2249 unlock:
2250 mutex_unlock(&hpre_algs_lock);
2251 return ret;
2252 }
2253
hpre_algs_unregister(struct hisi_qm * qm)2254 void hpre_algs_unregister(struct hisi_qm *qm)
2255 {
2256 mutex_lock(&hpre_algs_lock);
2257 if (--hpre_available_devs)
2258 goto unlock;
2259
2260 hpre_unregister_x25519(qm);
2261 hpre_unregister_ecdh(qm);
2262 hpre_unregister_dh(qm);
2263 hpre_unregister_rsa(qm);
2264
2265 unlock:
2266 mutex_unlock(&hpre_algs_lock);
2267 }
2268