Lines Matching refs:kc
187 static void wake(struct dm_kcopyd_client *kc) in wake() argument
189 queue_work(kc->kcopyd_wq, &kc->kcopyd_work); in wake()
222 static void kcopyd_put_pages(struct dm_kcopyd_client *kc, struct page_list *pl) in kcopyd_put_pages() argument
229 if (kc->nr_free_pages >= kc->nr_reserved_pages) in kcopyd_put_pages()
232 pl->next = kc->pages; in kcopyd_put_pages()
233 kc->pages = pl; in kcopyd_put_pages()
234 kc->nr_free_pages++; in kcopyd_put_pages()
241 static int kcopyd_get_pages(struct dm_kcopyd_client *kc, in kcopyd_get_pages() argument
252 pl = kc->pages; in kcopyd_get_pages()
255 kc->pages = pl->next; in kcopyd_get_pages()
256 kc->nr_free_pages--; in kcopyd_get_pages()
266 kcopyd_put_pages(kc, *pages); in kcopyd_get_pages()
287 static int client_reserve_pages(struct dm_kcopyd_client *kc, unsigned nr_pages) in client_reserve_pages() argument
303 kc->nr_reserved_pages += nr_pages; in client_reserve_pages()
304 kcopyd_put_pages(kc, pl); in client_reserve_pages()
309 static void client_free_pages(struct dm_kcopyd_client *kc) in client_free_pages() argument
311 BUG_ON(kc->nr_free_pages != kc->nr_reserved_pages); in client_free_pages()
312 drop_pages(kc->pages); in client_free_pages()
313 kc->pages = NULL; in client_free_pages()
314 kc->nr_free_pages = kc->nr_reserved_pages = 0; in client_free_pages()
323 struct dm_kcopyd_client *kc; member
392 struct dm_kcopyd_client *kc) in pop() argument
397 spin_lock_irqsave(&kc->job_lock, flags); in pop()
403 spin_unlock_irqrestore(&kc->job_lock, flags); in pop()
411 struct dm_kcopyd_client *kc = job->kc; in push() local
413 spin_lock_irqsave(&kc->job_lock, flags); in push()
415 spin_unlock_irqrestore(&kc->job_lock, flags); in push()
422 struct dm_kcopyd_client *kc = job->kc; in push_head() local
424 spin_lock_irqsave(&kc->job_lock, flags); in push_head()
426 spin_unlock_irqrestore(&kc->job_lock, flags); in push_head()
444 struct dm_kcopyd_client *kc = job->kc; in run_complete_job() local
447 kcopyd_put_pages(kc, job->pages); in run_complete_job()
453 mempool_free(job, kc->job_pool); in run_complete_job()
456 if (atomic_dec_and_test(&kc->nr_jobs)) in run_complete_job()
457 wake_up(&kc->destroyq); in run_complete_job()
467 struct dm_kcopyd_client *kc = job->kc; in complete_io() local
469 io_job_finish(kc->throttle); in complete_io()
478 push(&kc->complete_jobs, job); in complete_io()
479 wake(kc); in complete_io()
485 push(&kc->complete_jobs, job); in complete_io()
489 push(&kc->io_jobs, job); in complete_io()
492 wake(kc); in complete_io()
509 .client = job->kc->io_client, in run_io_job()
512 io_job_start(job->kc->throttle); in run_io_job()
527 r = kcopyd_get_pages(job->kc, nr_pages, &job->pages); in run_pages_job()
530 push(&job->kc->io_jobs, job); in run_pages_job()
545 static int process_jobs(struct list_head *jobs, struct dm_kcopyd_client *kc, in process_jobs() argument
551 while ((job = pop(jobs, kc))) { in process_jobs()
561 push(&kc->complete_jobs, job); in process_jobs()
585 struct dm_kcopyd_client *kc = container_of(work, in do_work() local
597 spin_lock_irqsave(&kc->job_lock, flags); in do_work()
598 list_splice_tail_init(&kc->callback_jobs, &kc->complete_jobs); in do_work()
599 spin_unlock_irqrestore(&kc->job_lock, flags); in do_work()
602 process_jobs(&kc->complete_jobs, kc, run_complete_job); in do_work()
603 process_jobs(&kc->pages_jobs, kc, run_pages_job); in do_work()
604 process_jobs(&kc->io_jobs, kc, run_io_job); in do_work()
615 struct dm_kcopyd_client *kc = job->kc; in dispatch_job() local
616 atomic_inc(&kc->nr_jobs); in dispatch_job()
618 push(&kc->callback_jobs, job); in dispatch_job()
620 push(&kc->io_jobs, job); in dispatch_job()
622 push(&kc->pages_jobs, job); in dispatch_job()
623 wake(kc); in dispatch_job()
634 struct dm_kcopyd_client *kc = job->kc; in segment_complete() local
689 push(&kc->complete_jobs, job); in segment_complete()
690 wake(kc); in segment_complete()
701 atomic_inc(&master_job->kc->nr_jobs); in split_job()
710 int dm_kcopyd_copy(struct dm_kcopyd_client *kc, struct dm_io_region *from, in dm_kcopyd_copy() argument
721 job = mempool_alloc(kc->job_pool, GFP_NOIO); in dm_kcopyd_copy()
726 job->kc = kc; in dm_kcopyd_copy()
770 int dm_kcopyd_zero(struct dm_kcopyd_client *kc, in dm_kcopyd_zero() argument
774 return dm_kcopyd_copy(kc, NULL, num_dests, dests, flags, fn, context); in dm_kcopyd_zero()
778 void *dm_kcopyd_prepare_callback(struct dm_kcopyd_client *kc, in dm_kcopyd_prepare_callback() argument
783 job = mempool_alloc(kc->job_pool, GFP_NOIO); in dm_kcopyd_prepare_callback()
786 job->kc = kc; in dm_kcopyd_prepare_callback()
791 atomic_inc(&kc->nr_jobs); in dm_kcopyd_prepare_callback()
800 struct dm_kcopyd_client *kc = job->kc; in dm_kcopyd_do_callback() local
805 push(&kc->callback_jobs, job); in dm_kcopyd_do_callback()
806 wake(kc); in dm_kcopyd_do_callback()
828 struct dm_kcopyd_client *kc; in dm_kcopyd_client_create() local
830 kc = kzalloc(sizeof(*kc), GFP_KERNEL); in dm_kcopyd_client_create()
831 if (!kc) in dm_kcopyd_client_create()
834 spin_lock_init(&kc->job_lock); in dm_kcopyd_client_create()
835 INIT_LIST_HEAD(&kc->callback_jobs); in dm_kcopyd_client_create()
836 INIT_LIST_HEAD(&kc->complete_jobs); in dm_kcopyd_client_create()
837 INIT_LIST_HEAD(&kc->io_jobs); in dm_kcopyd_client_create()
838 INIT_LIST_HEAD(&kc->pages_jobs); in dm_kcopyd_client_create()
839 kc->throttle = throttle; in dm_kcopyd_client_create()
841 kc->job_pool = mempool_create_slab_pool(MIN_JOBS, _job_cache); in dm_kcopyd_client_create()
842 if (!kc->job_pool) in dm_kcopyd_client_create()
845 INIT_WORK(&kc->kcopyd_work, do_work); in dm_kcopyd_client_create()
846 kc->kcopyd_wq = alloc_workqueue("kcopyd", WQ_MEM_RECLAIM, 0); in dm_kcopyd_client_create()
847 if (!kc->kcopyd_wq) in dm_kcopyd_client_create()
850 kc->pages = NULL; in dm_kcopyd_client_create()
851 kc->nr_reserved_pages = kc->nr_free_pages = 0; in dm_kcopyd_client_create()
852 r = client_reserve_pages(kc, RESERVE_PAGES); in dm_kcopyd_client_create()
856 kc->io_client = dm_io_client_create(); in dm_kcopyd_client_create()
857 if (IS_ERR(kc->io_client)) { in dm_kcopyd_client_create()
858 r = PTR_ERR(kc->io_client); in dm_kcopyd_client_create()
862 init_waitqueue_head(&kc->destroyq); in dm_kcopyd_client_create()
863 atomic_set(&kc->nr_jobs, 0); in dm_kcopyd_client_create()
865 return kc; in dm_kcopyd_client_create()
868 client_free_pages(kc); in dm_kcopyd_client_create()
870 destroy_workqueue(kc->kcopyd_wq); in dm_kcopyd_client_create()
872 mempool_destroy(kc->job_pool); in dm_kcopyd_client_create()
874 kfree(kc); in dm_kcopyd_client_create()
880 void dm_kcopyd_client_destroy(struct dm_kcopyd_client *kc) in dm_kcopyd_client_destroy() argument
883 wait_event(kc->destroyq, !atomic_read(&kc->nr_jobs)); in dm_kcopyd_client_destroy()
885 BUG_ON(!list_empty(&kc->callback_jobs)); in dm_kcopyd_client_destroy()
886 BUG_ON(!list_empty(&kc->complete_jobs)); in dm_kcopyd_client_destroy()
887 BUG_ON(!list_empty(&kc->io_jobs)); in dm_kcopyd_client_destroy()
888 BUG_ON(!list_empty(&kc->pages_jobs)); in dm_kcopyd_client_destroy()
889 destroy_workqueue(kc->kcopyd_wq); in dm_kcopyd_client_destroy()
890 dm_io_client_destroy(kc->io_client); in dm_kcopyd_client_destroy()
891 client_free_pages(kc); in dm_kcopyd_client_destroy()
892 mempool_destroy(kc->job_pool); in dm_kcopyd_client_destroy()
893 kfree(kc); in dm_kcopyd_client_destroy()