392 	struct kcopyd_job *job = NULL;  in pop()  local
398 		job = list_entry(jobs->next, struct kcopyd_job, list);  in pop()
399 		list_del(&job->list);  in pop()
403 	return job;  in pop()
406 static void push(struct list_head *jobs, struct kcopyd_job *job)  in push()  argument
409 	struct dm_kcopyd_client *kc = job->kc;  in push()
412 	list_add_tail(&job->list, jobs);  in push()
417 static void push_head(struct list_head *jobs, struct kcopyd_job *job)  in push_head()  argument
420 	struct dm_kcopyd_client *kc = job->kc;  in push_head()
423 	list_add(&job->list, jobs);  in push_head()
436 static int run_complete_job(struct kcopyd_job *job)  in run_complete_job()  argument
438 	void *context = job->context;  in run_complete_job()
439 	int read_err = job->read_err;  in run_complete_job()
440 	unsigned long write_err = job->write_err;  in run_complete_job()
441 	dm_kcopyd_notify_fn fn = job->fn;  in run_complete_job()
442 	struct dm_kcopyd_client *kc = job->kc;  in run_complete_job()
444 	if (job->pages && job->pages != &zero_page_list)  in run_complete_job()
445 		kcopyd_put_pages(kc, job->pages);  in run_complete_job()
450 	if (job->master_job == job)  in run_complete_job()
451 		mempool_free(job, kc->job_pool);  in run_complete_job()
462 	struct kcopyd_job *job = (struct kcopyd_job *) context;  in complete_io()  local
463 	struct dm_kcopyd_client *kc = job->kc;  in complete_io()
468 		if (job->rw & WRITE)  in complete_io()
469 			job->write_err |= error;  in complete_io()
471 			job->read_err = 1;  in complete_io()
473 		if (!test_bit(DM_KCOPYD_IGNORE_ERROR, &job->flags)) {  in complete_io()
474 			push(&kc->complete_jobs, job);  in complete_io()
480 	if (job->rw & WRITE)  in complete_io()
481 		push(&kc->complete_jobs, job);  in complete_io()
484 		job->rw = WRITE;  in complete_io()
485 		push(&kc->io_jobs, job);  in complete_io()
495 static int run_io_job(struct kcopyd_job *job)  in run_io_job()  argument
499 		.bi_rw = job->rw,  in run_io_job()
501 		.mem.ptr.pl = job->pages,  in run_io_job()
504 		.notify.context = job,  in run_io_job()
505 		.client = job->kc->io_client,  in run_io_job()
508 	io_job_start(job->kc->throttle);  in run_io_job()
510 	if (job->rw == READ)  in run_io_job()
511 		r = dm_io(&io_req, 1, &job->source, NULL);  in run_io_job()
513 		r = dm_io(&io_req, job->num_dests, job->dests, NULL);  in run_io_job()
518 static int run_pages_job(struct kcopyd_job *job)  in run_pages_job()  argument
521 	unsigned nr_pages = dm_div_up(job->dests[0].count, PAGE_SIZE >> 9);  in run_pages_job()
523 	r = kcopyd_get_pages(job->kc, nr_pages, &job->pages);  in run_pages_job()
526 		push(&job->kc->io_jobs, job);  in run_pages_job()
544 	struct kcopyd_job *job;  in process_jobs()  local
547 	while ((job = pop(jobs, kc))) {  in process_jobs()
549 		r = fn(job);  in process_jobs()
553 			if (job->rw & WRITE)  in process_jobs()
554 				job->write_err = (unsigned long) -1L;  in process_jobs()
556 				job->read_err = 1;  in process_jobs()
557 			push(&kc->complete_jobs, job);  in process_jobs()
566 			push_head(jobs, job);  in process_jobs()
604 static void dispatch_job(struct kcopyd_job *job)  in dispatch_job()  argument
606 	struct dm_kcopyd_client *kc = job->kc;  in dispatch_job()
608 	if (unlikely(!job->source.count))  in dispatch_job()
609 		push(&kc->complete_jobs, job);  in dispatch_job()
610 	else if (job->pages == &zero_page_list)  in dispatch_job()
611 		push(&kc->io_jobs, job);  in dispatch_job()
613 		push(&kc->pages_jobs, job);  in dispatch_job()
624 	struct kcopyd_job *job = sub_job->master_job;  in segment_complete()  local
625 	struct dm_kcopyd_client *kc = job->kc;  in segment_complete()
627 	mutex_lock(&job->lock);  in segment_complete()
631 		job->read_err = 1;  in segment_complete()
634 		job->write_err |= write_err;  in segment_complete()
639 	if ((!job->read_err && !job->write_err) ||  in segment_complete()
640 	    test_bit(DM_KCOPYD_IGNORE_ERROR, &job->flags)) {  in segment_complete()
642 		progress = job->progress;  in segment_complete()
643 		count = job->source.count - progress;  in segment_complete()
648 			job->progress += count;  in segment_complete()
651 	mutex_unlock(&job->lock);  in segment_complete()
656 		*sub_job = *job;  in segment_complete()
660 		for (i = 0; i < job->num_dests; i++) {  in segment_complete()
669 	} else if (atomic_dec_and_test(&job->sub_jobs)) {  in segment_complete()
680 		push(&kc->complete_jobs, job);  in segment_complete()
705 	struct kcopyd_job *job;  in dm_kcopyd_copy()  local
712 	job = mempool_alloc(kc->job_pool, GFP_NOIO);  in dm_kcopyd_copy()
717 	job->kc = kc;  in dm_kcopyd_copy()
718 	job->flags = flags;  in dm_kcopyd_copy()
719 	job->read_err = 0;  in dm_kcopyd_copy()
720 	job->write_err = 0;  in dm_kcopyd_copy()
722 	job->num_dests = num_dests;  in dm_kcopyd_copy()
723 	memcpy(&job->dests, dests, sizeof(*dests) * num_dests);  in dm_kcopyd_copy()
726 		job->source = *from;  in dm_kcopyd_copy()
727 		job->pages = NULL;  in dm_kcopyd_copy()
728 		job->rw = READ;  in dm_kcopyd_copy()
730 		memset(&job->source, 0, sizeof job->source);  in dm_kcopyd_copy()
731 		job->source.count = job->dests[0].count;  in dm_kcopyd_copy()
732 		job->pages = &zero_page_list;  in dm_kcopyd_copy()
737 		job->rw = WRITE | REQ_WRITE_SAME;  in dm_kcopyd_copy()
738 		for (i = 0; i < job->num_dests; i++)  in dm_kcopyd_copy()
739 			if (!bdev_write_same(job->dests[i].bdev)) {  in dm_kcopyd_copy()
740 				job->rw = WRITE;  in dm_kcopyd_copy()
745 	job->fn = fn;  in dm_kcopyd_copy()
746 	job->context = context;  in dm_kcopyd_copy()
747 	job->master_job = job;  in dm_kcopyd_copy()
749 	if (job->source.count <= SUB_JOB_SIZE)  in dm_kcopyd_copy()
750 		dispatch_job(job);  in dm_kcopyd_copy()
752 		mutex_init(&job->lock);  in dm_kcopyd_copy()
753 		job->progress = 0;  in dm_kcopyd_copy()
754 		split_job(job);  in dm_kcopyd_copy()
772 	struct kcopyd_job *job;  in dm_kcopyd_prepare_callback()  local
774 	job = mempool_alloc(kc->job_pool, GFP_NOIO);  in dm_kcopyd_prepare_callback()
776 	memset(job, 0, sizeof(struct kcopyd_job));  in dm_kcopyd_prepare_callback()
777 	job->kc = kc;  in dm_kcopyd_prepare_callback()
778 	job->fn = fn;  in dm_kcopyd_prepare_callback()
779 	job->context = context;  in dm_kcopyd_prepare_callback()
780 	job->master_job = job;  in dm_kcopyd_prepare_callback()
784 	return job;  in dm_kcopyd_prepare_callback()
790 	struct kcopyd_job *job = j;  in dm_kcopyd_do_callback()  local
791 	struct dm_kcopyd_client *kc = job->kc;  in dm_kcopyd_do_callback()
793 	job->read_err = read_err;  in dm_kcopyd_do_callback()
794 	job->write_err = write_err;  in dm_kcopyd_do_callback()
796 	push(&kc->complete_jobs, job);  in dm_kcopyd_do_callback()
806 int kcopyd_cancel(struct kcopyd_job *job, int block)