167 int padata_do_parallel(struct padata_shell *ps,  in padata_do_parallel()  argument
170 	struct padata_instance *pinst = ps->pinst;  in padata_do_parallel()
177 	pd = rcu_dereference_bh(ps->pd);  in padata_do_parallel()
278 	struct padata_instance *pinst = pd->ps->pinst;  in padata_reorder()
436 	struct padata_mt_job_state *ps = pw->pw_data;  in padata_mt_helper()  local
437 	struct padata_mt_job *job = ps->job;  in padata_mt_helper()
440 	spin_lock(&ps->lock);  in padata_mt_helper()
447 		size = roundup(start + 1, ps->chunk_size) - start;  in padata_mt_helper()
454 		spin_unlock(&ps->lock);  in padata_mt_helper()
456 		spin_lock(&ps->lock);  in padata_mt_helper()
459 	++ps->nworks_fini;  in padata_mt_helper()
460 	done = (ps->nworks_fini == ps->nworks);  in padata_mt_helper()
461 	spin_unlock(&ps->lock);  in padata_mt_helper()
464 		complete(&ps->completion);  in padata_mt_helper()
478 	struct padata_mt_job_state ps;  in padata_do_multithreaded()  local
495 	spin_lock_init(&ps.lock);  in padata_do_multithreaded()
496 	init_completion(&ps.completion);  in padata_do_multithreaded()
497 	ps.job	       = job;  in padata_do_multithreaded()
498 	ps.nworks      = padata_work_alloc_mt(nworks, &ps, &works);  in padata_do_multithreaded()
499 	ps.nworks_fini = 0;  in padata_do_multithreaded()
507 	ps.chunk_size = job->size / (ps.nworks * load_balance_factor);  in padata_do_multithreaded()
508 	ps.chunk_size = max(ps.chunk_size, job->min_chunk);  in padata_do_multithreaded()
509 	ps.chunk_size = roundup(ps.chunk_size, job->align);  in padata_do_multithreaded()
515 	padata_work_init(&my_work, padata_mt_helper, &ps, PADATA_WORK_ONSTACK);  in padata_do_multithreaded()
519 	wait_for_completion(&ps.completion);  in padata_do_multithreaded()
558 static struct parallel_data *padata_alloc_pd(struct padata_shell *ps)  in padata_alloc_pd()  argument
560 	struct padata_instance *pinst = ps->pinst;  in padata_alloc_pd()
575 	pd->ps = ps;  in padata_alloc_pd()
632 static int padata_replace_one(struct padata_shell *ps)  in padata_replace_one()  argument
636 	pd_new = padata_alloc_pd(ps);  in padata_replace_one()
640 	ps->opd = rcu_dereference_protected(ps->pd, 1);  in padata_replace_one()
641 	rcu_assign_pointer(ps->pd, pd_new);  in padata_replace_one()
648 	struct padata_shell *ps;  in padata_replace()  local
653 	list_for_each_entry(ps, &pinst->pslist, list) {  in padata_replace()
654 		err = padata_replace_one(ps);  in padata_replace()
661 	list_for_each_entry_continue_reverse(ps, &pinst->pslist, list)  in padata_replace()
662 		if (refcount_dec_and_test(&ps->opd->refcnt))  in padata_replace()
663 			padata_free_pd(ps->opd);  in padata_replace()
1061 	struct padata_shell *ps;  in padata_alloc_shell()  local
1063 	ps = kzalloc(sizeof(*ps), GFP_KERNEL);  in padata_alloc_shell()
1064 	if (!ps)  in padata_alloc_shell()
1067 	ps->pinst = pinst;  in padata_alloc_shell()
1070 	pd = padata_alloc_pd(ps);  in padata_alloc_shell()
1077 	RCU_INIT_POINTER(ps->pd, pd);  in padata_alloc_shell()
1078 	list_add(&ps->list, &pinst->pslist);  in padata_alloc_shell()
1081 	return ps;  in padata_alloc_shell()
1084 	kfree(ps);  in padata_alloc_shell()
1095 void padata_free_shell(struct padata_shell *ps)  in padata_free_shell()  argument
1099 	if (!ps)  in padata_free_shell()
1102 	mutex_lock(&ps->pinst->lock);  in padata_free_shell()
1103 	list_del(&ps->list);  in padata_free_shell()
1104 	pd = rcu_dereference_protected(ps->pd, 1);  in padata_free_shell()
1107 	mutex_unlock(&ps->pinst->lock);  in padata_free_shell()
1109 	kfree(ps);  in padata_free_shell()