1 /*
2 * Software multibuffer async crypto daemon.
3 *
4 * Copyright (c) 2014 Tim Chen <tim.c.chen@linux.intel.com>
5 *
6 * Adapted from crypto daemon.
7 *
8 * This program is free software; you can redistribute it and/or modify it
9 * under the terms of the GNU General Public License as published by the Free
10 * Software Foundation; either version 2 of the License, or (at your option)
11 * any later version.
12 *
13 */
14
15 #include <crypto/algapi.h>
16 #include <crypto/internal/hash.h>
17 #include <crypto/internal/aead.h>
18 #include <crypto/mcryptd.h>
19 #include <crypto/crypto_wq.h>
20 #include <linux/err.h>
21 #include <linux/init.h>
22 #include <linux/kernel.h>
23 #include <linux/list.h>
24 #include <linux/module.h>
25 #include <linux/scatterlist.h>
26 #include <linux/sched.h>
27 #include <linux/sched/stat.h>
28 #include <linux/slab.h>
29 #include <linux/hardirq.h>
30
31 #define MCRYPTD_MAX_CPU_QLEN 100
32 #define MCRYPTD_BATCH 9
33
34 static void *mcryptd_alloc_instance(struct crypto_alg *alg, unsigned int head,
35 unsigned int tail);
36
37 struct mcryptd_flush_list {
38 struct list_head list;
39 struct mutex lock;
40 };
41
42 static struct mcryptd_flush_list __percpu *mcryptd_flist;
43
44 struct hashd_instance_ctx {
45 struct crypto_ahash_spawn spawn;
46 struct mcryptd_queue *queue;
47 };
48
49 static void mcryptd_queue_worker(struct work_struct *work);
50
mcryptd_arm_flusher(struct mcryptd_alg_cstate * cstate,unsigned long delay)51 void mcryptd_arm_flusher(struct mcryptd_alg_cstate *cstate, unsigned long delay)
52 {
53 struct mcryptd_flush_list *flist;
54
55 if (!cstate->flusher_engaged) {
56 /* put the flusher on the flush list */
57 flist = per_cpu_ptr(mcryptd_flist, smp_processor_id());
58 mutex_lock(&flist->lock);
59 list_add_tail(&cstate->flush_list, &flist->list);
60 cstate->flusher_engaged = true;
61 cstate->next_flush = jiffies + delay;
62 queue_delayed_work_on(smp_processor_id(), kcrypto_wq,
63 &cstate->flush, delay);
64 mutex_unlock(&flist->lock);
65 }
66 }
67 EXPORT_SYMBOL(mcryptd_arm_flusher);
68
mcryptd_init_queue(struct mcryptd_queue * queue,unsigned int max_cpu_qlen)69 static int mcryptd_init_queue(struct mcryptd_queue *queue,
70 unsigned int max_cpu_qlen)
71 {
72 int cpu;
73 struct mcryptd_cpu_queue *cpu_queue;
74
75 queue->cpu_queue = alloc_percpu(struct mcryptd_cpu_queue);
76 pr_debug("mqueue:%p mcryptd_cpu_queue %p\n", queue, queue->cpu_queue);
77 if (!queue->cpu_queue)
78 return -ENOMEM;
79 for_each_possible_cpu(cpu) {
80 cpu_queue = per_cpu_ptr(queue->cpu_queue, cpu);
81 pr_debug("cpu_queue #%d %p\n", cpu, queue->cpu_queue);
82 crypto_init_queue(&cpu_queue->queue, max_cpu_qlen);
83 INIT_WORK(&cpu_queue->work, mcryptd_queue_worker);
84 spin_lock_init(&cpu_queue->q_lock);
85 }
86 return 0;
87 }
88
mcryptd_fini_queue(struct mcryptd_queue * queue)89 static void mcryptd_fini_queue(struct mcryptd_queue *queue)
90 {
91 int cpu;
92 struct mcryptd_cpu_queue *cpu_queue;
93
94 for_each_possible_cpu(cpu) {
95 cpu_queue = per_cpu_ptr(queue->cpu_queue, cpu);
96 BUG_ON(cpu_queue->queue.qlen);
97 }
98 free_percpu(queue->cpu_queue);
99 }
100
mcryptd_enqueue_request(struct mcryptd_queue * queue,struct crypto_async_request * request,struct mcryptd_hash_request_ctx * rctx)101 static int mcryptd_enqueue_request(struct mcryptd_queue *queue,
102 struct crypto_async_request *request,
103 struct mcryptd_hash_request_ctx *rctx)
104 {
105 int cpu, err;
106 struct mcryptd_cpu_queue *cpu_queue;
107
108 cpu_queue = raw_cpu_ptr(queue->cpu_queue);
109 spin_lock(&cpu_queue->q_lock);
110 cpu = smp_processor_id();
111 rctx->tag.cpu = smp_processor_id();
112
113 err = crypto_enqueue_request(&cpu_queue->queue, request);
114 pr_debug("enqueue request: cpu %d cpu_queue %p request %p\n",
115 cpu, cpu_queue, request);
116 spin_unlock(&cpu_queue->q_lock);
117 queue_work_on(cpu, kcrypto_wq, &cpu_queue->work);
118
119 return err;
120 }
121
122 /*
123 * Try to opportunisticlly flush the partially completed jobs if
124 * crypto daemon is the only task running.
125 */
mcryptd_opportunistic_flush(void)126 static void mcryptd_opportunistic_flush(void)
127 {
128 struct mcryptd_flush_list *flist;
129 struct mcryptd_alg_cstate *cstate;
130
131 flist = per_cpu_ptr(mcryptd_flist, smp_processor_id());
132 while (single_task_running()) {
133 mutex_lock(&flist->lock);
134 cstate = list_first_entry_or_null(&flist->list,
135 struct mcryptd_alg_cstate, flush_list);
136 if (!cstate || !cstate->flusher_engaged) {
137 mutex_unlock(&flist->lock);
138 return;
139 }
140 list_del(&cstate->flush_list);
141 cstate->flusher_engaged = false;
142 mutex_unlock(&flist->lock);
143 cstate->alg_state->flusher(cstate);
144 }
145 }
146
147 /*
148 * Called in workqueue context, do one real cryption work (via
149 * req->complete) and reschedule itself if there are more work to
150 * do.
151 */
mcryptd_queue_worker(struct work_struct * work)152 static void mcryptd_queue_worker(struct work_struct *work)
153 {
154 struct mcryptd_cpu_queue *cpu_queue;
155 struct crypto_async_request *req, *backlog;
156 int i;
157
158 /*
159 * Need to loop through more than once for multi-buffer to
160 * be effective.
161 */
162
163 cpu_queue = container_of(work, struct mcryptd_cpu_queue, work);
164 i = 0;
165 while (i < MCRYPTD_BATCH || single_task_running()) {
166
167 spin_lock_bh(&cpu_queue->q_lock);
168 backlog = crypto_get_backlog(&cpu_queue->queue);
169 req = crypto_dequeue_request(&cpu_queue->queue);
170 spin_unlock_bh(&cpu_queue->q_lock);
171
172 if (!req) {
173 mcryptd_opportunistic_flush();
174 return;
175 }
176
177 if (backlog)
178 backlog->complete(backlog, -EINPROGRESS);
179 req->complete(req, 0);
180 if (!cpu_queue->queue.qlen)
181 return;
182 ++i;
183 }
184 if (cpu_queue->queue.qlen)
185 queue_work_on(smp_processor_id(), kcrypto_wq, &cpu_queue->work);
186 }
187
mcryptd_flusher(struct work_struct * __work)188 void mcryptd_flusher(struct work_struct *__work)
189 {
190 struct mcryptd_alg_cstate *alg_cpu_state;
191 struct mcryptd_alg_state *alg_state;
192 struct mcryptd_flush_list *flist;
193 int cpu;
194
195 cpu = smp_processor_id();
196 alg_cpu_state = container_of(to_delayed_work(__work),
197 struct mcryptd_alg_cstate, flush);
198 alg_state = alg_cpu_state->alg_state;
199 if (alg_cpu_state->cpu != cpu)
200 pr_debug("mcryptd error: work on cpu %d, should be cpu %d\n",
201 cpu, alg_cpu_state->cpu);
202
203 if (alg_cpu_state->flusher_engaged) {
204 flist = per_cpu_ptr(mcryptd_flist, cpu);
205 mutex_lock(&flist->lock);
206 list_del(&alg_cpu_state->flush_list);
207 alg_cpu_state->flusher_engaged = false;
208 mutex_unlock(&flist->lock);
209 alg_state->flusher(alg_cpu_state);
210 }
211 }
212 EXPORT_SYMBOL_GPL(mcryptd_flusher);
213
mcryptd_get_queue(struct crypto_tfm * tfm)214 static inline struct mcryptd_queue *mcryptd_get_queue(struct crypto_tfm *tfm)
215 {
216 struct crypto_instance *inst = crypto_tfm_alg_instance(tfm);
217 struct mcryptd_instance_ctx *ictx = crypto_instance_ctx(inst);
218
219 return ictx->queue;
220 }
221
mcryptd_alloc_instance(struct crypto_alg * alg,unsigned int head,unsigned int tail)222 static void *mcryptd_alloc_instance(struct crypto_alg *alg, unsigned int head,
223 unsigned int tail)
224 {
225 char *p;
226 struct crypto_instance *inst;
227 int err;
228
229 p = kzalloc(head + sizeof(*inst) + tail, GFP_KERNEL);
230 if (!p)
231 return ERR_PTR(-ENOMEM);
232
233 inst = (void *)(p + head);
234
235 err = -ENAMETOOLONG;
236 if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME,
237 "mcryptd(%s)", alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
238 goto out_free_inst;
239
240 memcpy(inst->alg.cra_name, alg->cra_name, CRYPTO_MAX_ALG_NAME);
241
242 inst->alg.cra_priority = alg->cra_priority + 50;
243 inst->alg.cra_blocksize = alg->cra_blocksize;
244 inst->alg.cra_alignmask = alg->cra_alignmask;
245
246 out:
247 return p;
248
249 out_free_inst:
250 kfree(p);
251 p = ERR_PTR(err);
252 goto out;
253 }
254
mcryptd_check_internal(struct rtattr ** tb,u32 * type,u32 * mask)255 static inline bool mcryptd_check_internal(struct rtattr **tb, u32 *type,
256 u32 *mask)
257 {
258 struct crypto_attr_type *algt;
259
260 algt = crypto_get_attr_type(tb);
261 if (IS_ERR(algt))
262 return false;
263
264 *type |= algt->type & CRYPTO_ALG_INTERNAL;
265 *mask |= algt->mask & CRYPTO_ALG_INTERNAL;
266
267 if (*type & *mask & CRYPTO_ALG_INTERNAL)
268 return true;
269 else
270 return false;
271 }
272
mcryptd_hash_init_tfm(struct crypto_tfm * tfm)273 static int mcryptd_hash_init_tfm(struct crypto_tfm *tfm)
274 {
275 struct crypto_instance *inst = crypto_tfm_alg_instance(tfm);
276 struct hashd_instance_ctx *ictx = crypto_instance_ctx(inst);
277 struct crypto_ahash_spawn *spawn = &ictx->spawn;
278 struct mcryptd_hash_ctx *ctx = crypto_tfm_ctx(tfm);
279 struct crypto_ahash *hash;
280
281 hash = crypto_spawn_ahash(spawn);
282 if (IS_ERR(hash))
283 return PTR_ERR(hash);
284
285 ctx->child = hash;
286 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
287 sizeof(struct mcryptd_hash_request_ctx) +
288 crypto_ahash_reqsize(hash));
289 return 0;
290 }
291
mcryptd_hash_exit_tfm(struct crypto_tfm * tfm)292 static void mcryptd_hash_exit_tfm(struct crypto_tfm *tfm)
293 {
294 struct mcryptd_hash_ctx *ctx = crypto_tfm_ctx(tfm);
295
296 crypto_free_ahash(ctx->child);
297 }
298
mcryptd_hash_setkey(struct crypto_ahash * parent,const u8 * key,unsigned int keylen)299 static int mcryptd_hash_setkey(struct crypto_ahash *parent,
300 const u8 *key, unsigned int keylen)
301 {
302 struct mcryptd_hash_ctx *ctx = crypto_ahash_ctx(parent);
303 struct crypto_ahash *child = ctx->child;
304 int err;
305
306 crypto_ahash_clear_flags(child, CRYPTO_TFM_REQ_MASK);
307 crypto_ahash_set_flags(child, crypto_ahash_get_flags(parent) &
308 CRYPTO_TFM_REQ_MASK);
309 err = crypto_ahash_setkey(child, key, keylen);
310 crypto_ahash_set_flags(parent, crypto_ahash_get_flags(child) &
311 CRYPTO_TFM_RES_MASK);
312 return err;
313 }
314
mcryptd_hash_enqueue(struct ahash_request * req,crypto_completion_t complete)315 static int mcryptd_hash_enqueue(struct ahash_request *req,
316 crypto_completion_t complete)
317 {
318 int ret;
319
320 struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
321 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
322 struct mcryptd_queue *queue =
323 mcryptd_get_queue(crypto_ahash_tfm(tfm));
324
325 rctx->complete = req->base.complete;
326 req->base.complete = complete;
327
328 ret = mcryptd_enqueue_request(queue, &req->base, rctx);
329
330 return ret;
331 }
332
mcryptd_hash_init(struct crypto_async_request * req_async,int err)333 static void mcryptd_hash_init(struct crypto_async_request *req_async, int err)
334 {
335 struct mcryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm);
336 struct crypto_ahash *child = ctx->child;
337 struct ahash_request *req = ahash_request_cast(req_async);
338 struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
339 struct ahash_request *desc = &rctx->areq;
340
341 if (unlikely(err == -EINPROGRESS))
342 goto out;
343
344 ahash_request_set_tfm(desc, child);
345 ahash_request_set_callback(desc, CRYPTO_TFM_REQ_MAY_SLEEP,
346 rctx->complete, req_async);
347
348 rctx->out = req->result;
349 err = crypto_ahash_init(desc);
350
351 out:
352 local_bh_disable();
353 rctx->complete(&req->base, err);
354 local_bh_enable();
355 }
356
mcryptd_hash_init_enqueue(struct ahash_request * req)357 static int mcryptd_hash_init_enqueue(struct ahash_request *req)
358 {
359 return mcryptd_hash_enqueue(req, mcryptd_hash_init);
360 }
361
mcryptd_hash_update(struct crypto_async_request * req_async,int err)362 static void mcryptd_hash_update(struct crypto_async_request *req_async, int err)
363 {
364 struct ahash_request *req = ahash_request_cast(req_async);
365 struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
366
367 if (unlikely(err == -EINPROGRESS))
368 goto out;
369
370 rctx->out = req->result;
371 err = ahash_mcryptd_update(&rctx->areq);
372 if (err) {
373 req->base.complete = rctx->complete;
374 goto out;
375 }
376
377 return;
378 out:
379 local_bh_disable();
380 rctx->complete(&req->base, err);
381 local_bh_enable();
382 }
383
mcryptd_hash_update_enqueue(struct ahash_request * req)384 static int mcryptd_hash_update_enqueue(struct ahash_request *req)
385 {
386 return mcryptd_hash_enqueue(req, mcryptd_hash_update);
387 }
388
mcryptd_hash_final(struct crypto_async_request * req_async,int err)389 static void mcryptd_hash_final(struct crypto_async_request *req_async, int err)
390 {
391 struct ahash_request *req = ahash_request_cast(req_async);
392 struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
393
394 if (unlikely(err == -EINPROGRESS))
395 goto out;
396
397 rctx->out = req->result;
398 err = ahash_mcryptd_final(&rctx->areq);
399 if (err) {
400 req->base.complete = rctx->complete;
401 goto out;
402 }
403
404 return;
405 out:
406 local_bh_disable();
407 rctx->complete(&req->base, err);
408 local_bh_enable();
409 }
410
mcryptd_hash_final_enqueue(struct ahash_request * req)411 static int mcryptd_hash_final_enqueue(struct ahash_request *req)
412 {
413 return mcryptd_hash_enqueue(req, mcryptd_hash_final);
414 }
415
mcryptd_hash_finup(struct crypto_async_request * req_async,int err)416 static void mcryptd_hash_finup(struct crypto_async_request *req_async, int err)
417 {
418 struct ahash_request *req = ahash_request_cast(req_async);
419 struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
420
421 if (unlikely(err == -EINPROGRESS))
422 goto out;
423 rctx->out = req->result;
424 err = ahash_mcryptd_finup(&rctx->areq);
425
426 if (err) {
427 req->base.complete = rctx->complete;
428 goto out;
429 }
430
431 return;
432 out:
433 local_bh_disable();
434 rctx->complete(&req->base, err);
435 local_bh_enable();
436 }
437
mcryptd_hash_finup_enqueue(struct ahash_request * req)438 static int mcryptd_hash_finup_enqueue(struct ahash_request *req)
439 {
440 return mcryptd_hash_enqueue(req, mcryptd_hash_finup);
441 }
442
mcryptd_hash_digest(struct crypto_async_request * req_async,int err)443 static void mcryptd_hash_digest(struct crypto_async_request *req_async, int err)
444 {
445 struct mcryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm);
446 struct crypto_ahash *child = ctx->child;
447 struct ahash_request *req = ahash_request_cast(req_async);
448 struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
449 struct ahash_request *desc = &rctx->areq;
450
451 if (unlikely(err == -EINPROGRESS))
452 goto out;
453
454 ahash_request_set_tfm(desc, child);
455 ahash_request_set_callback(desc, CRYPTO_TFM_REQ_MAY_SLEEP,
456 rctx->complete, req_async);
457
458 rctx->out = req->result;
459 err = ahash_mcryptd_digest(desc);
460
461 out:
462 local_bh_disable();
463 rctx->complete(&req->base, err);
464 local_bh_enable();
465 }
466
mcryptd_hash_digest_enqueue(struct ahash_request * req)467 static int mcryptd_hash_digest_enqueue(struct ahash_request *req)
468 {
469 return mcryptd_hash_enqueue(req, mcryptd_hash_digest);
470 }
471
mcryptd_hash_export(struct ahash_request * req,void * out)472 static int mcryptd_hash_export(struct ahash_request *req, void *out)
473 {
474 struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
475
476 return crypto_ahash_export(&rctx->areq, out);
477 }
478
mcryptd_hash_import(struct ahash_request * req,const void * in)479 static int mcryptd_hash_import(struct ahash_request *req, const void *in)
480 {
481 struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
482
483 return crypto_ahash_import(&rctx->areq, in);
484 }
485
mcryptd_create_hash(struct crypto_template * tmpl,struct rtattr ** tb,struct mcryptd_queue * queue)486 static int mcryptd_create_hash(struct crypto_template *tmpl, struct rtattr **tb,
487 struct mcryptd_queue *queue)
488 {
489 struct hashd_instance_ctx *ctx;
490 struct ahash_instance *inst;
491 struct hash_alg_common *halg;
492 struct crypto_alg *alg;
493 u32 type = 0;
494 u32 mask = 0;
495 int err;
496
497 if (!mcryptd_check_internal(tb, &type, &mask))
498 return -EINVAL;
499
500 halg = ahash_attr_alg(tb[1], type, mask);
501 if (IS_ERR(halg))
502 return PTR_ERR(halg);
503
504 alg = &halg->base;
505 pr_debug("crypto: mcryptd hash alg: %s\n", alg->cra_name);
506 inst = mcryptd_alloc_instance(alg, ahash_instance_headroom(),
507 sizeof(*ctx));
508 err = PTR_ERR(inst);
509 if (IS_ERR(inst))
510 goto out_put_alg;
511
512 ctx = ahash_instance_ctx(inst);
513 ctx->queue = queue;
514
515 err = crypto_init_ahash_spawn(&ctx->spawn, halg,
516 ahash_crypto_instance(inst));
517 if (err)
518 goto out_free_inst;
519
520 inst->alg.halg.base.cra_flags = CRYPTO_ALG_ASYNC |
521 (alg->cra_flags & (CRYPTO_ALG_INTERNAL |
522 CRYPTO_ALG_OPTIONAL_KEY));
523
524 inst->alg.halg.digestsize = halg->digestsize;
525 inst->alg.halg.statesize = halg->statesize;
526 inst->alg.halg.base.cra_ctxsize = sizeof(struct mcryptd_hash_ctx);
527
528 inst->alg.halg.base.cra_init = mcryptd_hash_init_tfm;
529 inst->alg.halg.base.cra_exit = mcryptd_hash_exit_tfm;
530
531 inst->alg.init = mcryptd_hash_init_enqueue;
532 inst->alg.update = mcryptd_hash_update_enqueue;
533 inst->alg.final = mcryptd_hash_final_enqueue;
534 inst->alg.finup = mcryptd_hash_finup_enqueue;
535 inst->alg.export = mcryptd_hash_export;
536 inst->alg.import = mcryptd_hash_import;
537 if (crypto_hash_alg_has_setkey(halg))
538 inst->alg.setkey = mcryptd_hash_setkey;
539 inst->alg.digest = mcryptd_hash_digest_enqueue;
540
541 err = ahash_register_instance(tmpl, inst);
542 if (err) {
543 crypto_drop_ahash(&ctx->spawn);
544 out_free_inst:
545 kfree(inst);
546 }
547
548 out_put_alg:
549 crypto_mod_put(alg);
550 return err;
551 }
552
553 static struct mcryptd_queue mqueue;
554
mcryptd_create(struct crypto_template * tmpl,struct rtattr ** tb)555 static int mcryptd_create(struct crypto_template *tmpl, struct rtattr **tb)
556 {
557 struct crypto_attr_type *algt;
558
559 algt = crypto_get_attr_type(tb);
560 if (IS_ERR(algt))
561 return PTR_ERR(algt);
562
563 switch (algt->type & algt->mask & CRYPTO_ALG_TYPE_MASK) {
564 case CRYPTO_ALG_TYPE_DIGEST:
565 return mcryptd_create_hash(tmpl, tb, &mqueue);
566 break;
567 }
568
569 return -EINVAL;
570 }
571
mcryptd_free(struct crypto_instance * inst)572 static void mcryptd_free(struct crypto_instance *inst)
573 {
574 struct mcryptd_instance_ctx *ctx = crypto_instance_ctx(inst);
575 struct hashd_instance_ctx *hctx = crypto_instance_ctx(inst);
576
577 switch (inst->alg.cra_flags & CRYPTO_ALG_TYPE_MASK) {
578 case CRYPTO_ALG_TYPE_AHASH:
579 crypto_drop_ahash(&hctx->spawn);
580 kfree(ahash_instance(inst));
581 return;
582 default:
583 crypto_drop_spawn(&ctx->spawn);
584 kfree(inst);
585 }
586 }
587
588 static struct crypto_template mcryptd_tmpl = {
589 .name = "mcryptd",
590 .create = mcryptd_create,
591 .free = mcryptd_free,
592 .module = THIS_MODULE,
593 };
594
mcryptd_alloc_ahash(const char * alg_name,u32 type,u32 mask)595 struct mcryptd_ahash *mcryptd_alloc_ahash(const char *alg_name,
596 u32 type, u32 mask)
597 {
598 char mcryptd_alg_name[CRYPTO_MAX_ALG_NAME];
599 struct crypto_ahash *tfm;
600
601 if (snprintf(mcryptd_alg_name, CRYPTO_MAX_ALG_NAME,
602 "mcryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME)
603 return ERR_PTR(-EINVAL);
604 tfm = crypto_alloc_ahash(mcryptd_alg_name, type, mask);
605 if (IS_ERR(tfm))
606 return ERR_CAST(tfm);
607 if (tfm->base.__crt_alg->cra_module != THIS_MODULE) {
608 crypto_free_ahash(tfm);
609 return ERR_PTR(-EINVAL);
610 }
611
612 return __mcryptd_ahash_cast(tfm);
613 }
614 EXPORT_SYMBOL_GPL(mcryptd_alloc_ahash);
615
ahash_mcryptd_digest(struct ahash_request * desc)616 int ahash_mcryptd_digest(struct ahash_request *desc)
617 {
618 return crypto_ahash_init(desc) ?: ahash_mcryptd_finup(desc);
619 }
620
ahash_mcryptd_update(struct ahash_request * desc)621 int ahash_mcryptd_update(struct ahash_request *desc)
622 {
623 /* alignment is to be done by multi-buffer crypto algorithm if needed */
624
625 return crypto_ahash_update(desc);
626 }
627
ahash_mcryptd_finup(struct ahash_request * desc)628 int ahash_mcryptd_finup(struct ahash_request *desc)
629 {
630 /* alignment is to be done by multi-buffer crypto algorithm if needed */
631
632 return crypto_ahash_finup(desc);
633 }
634
ahash_mcryptd_final(struct ahash_request * desc)635 int ahash_mcryptd_final(struct ahash_request *desc)
636 {
637 /* alignment is to be done by multi-buffer crypto algorithm if needed */
638
639 return crypto_ahash_final(desc);
640 }
641
mcryptd_ahash_child(struct mcryptd_ahash * tfm)642 struct crypto_ahash *mcryptd_ahash_child(struct mcryptd_ahash *tfm)
643 {
644 struct mcryptd_hash_ctx *ctx = crypto_ahash_ctx(&tfm->base);
645
646 return ctx->child;
647 }
648 EXPORT_SYMBOL_GPL(mcryptd_ahash_child);
649
mcryptd_ahash_desc(struct ahash_request * req)650 struct ahash_request *mcryptd_ahash_desc(struct ahash_request *req)
651 {
652 struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
653 return &rctx->areq;
654 }
655 EXPORT_SYMBOL_GPL(mcryptd_ahash_desc);
656
mcryptd_free_ahash(struct mcryptd_ahash * tfm)657 void mcryptd_free_ahash(struct mcryptd_ahash *tfm)
658 {
659 crypto_free_ahash(&tfm->base);
660 }
661 EXPORT_SYMBOL_GPL(mcryptd_free_ahash);
662
mcryptd_init(void)663 static int __init mcryptd_init(void)
664 {
665 int err, cpu;
666 struct mcryptd_flush_list *flist;
667
668 mcryptd_flist = alloc_percpu(struct mcryptd_flush_list);
669 for_each_possible_cpu(cpu) {
670 flist = per_cpu_ptr(mcryptd_flist, cpu);
671 INIT_LIST_HEAD(&flist->list);
672 mutex_init(&flist->lock);
673 }
674
675 err = mcryptd_init_queue(&mqueue, MCRYPTD_MAX_CPU_QLEN);
676 if (err) {
677 free_percpu(mcryptd_flist);
678 return err;
679 }
680
681 err = crypto_register_template(&mcryptd_tmpl);
682 if (err) {
683 mcryptd_fini_queue(&mqueue);
684 free_percpu(mcryptd_flist);
685 }
686
687 return err;
688 }
689
mcryptd_exit(void)690 static void __exit mcryptd_exit(void)
691 {
692 mcryptd_fini_queue(&mqueue);
693 crypto_unregister_template(&mcryptd_tmpl);
694 free_percpu(mcryptd_flist);
695 }
696
697 subsys_initcall(mcryptd_init);
698 module_exit(mcryptd_exit);
699
700 MODULE_LICENSE("GPL");
701 MODULE_DESCRIPTION("Software async multibuffer crypto daemon");
702 MODULE_ALIAS_CRYPTO("mcryptd");
703