49 	struct workqueue_struct *wq;  member
87 static inline int is_wq_single_threaded(struct workqueue_struct *wq)  in is_wq_single_threaded()  argument
89 	return wq->singlethread;  in is_wq_single_threaded()
92 static const struct cpumask *wq_cpu_map(struct workqueue_struct *wq)  in wq_cpu_map()  argument
94 	return is_wq_single_threaded(wq)  in wq_cpu_map()
99 struct cpu_workqueue_struct *wq_per_cpu(struct workqueue_struct *wq, int cpu)  in wq_per_cpu()  argument
101 	if (unlikely(is_wq_single_threaded(wq)))  in wq_per_cpu()
103 	return per_cpu_ptr(wq->cpu_wq, cpu);  in wq_per_cpu()
161 int queue_work(struct workqueue_struct *wq, struct work_struct *work)  in queue_work()  argument
165 	ret = queue_work_on(get_cpu(), wq, work);  in queue_work()
184 queue_work_on(int cpu, struct workqueue_struct *wq, struct work_struct *work)  in queue_work_on()  argument
190 		__queue_work(wq_per_cpu(wq, cpu), work);  in queue_work_on()
201 	struct workqueue_struct *wq = cwq->wq;  in delayed_work_timer_fn()  local
203 	__queue_work(wq_per_cpu(wq, smp_processor_id()), &dwork->work);  in delayed_work_timer_fn()
214 int queue_delayed_work(struct workqueue_struct *wq,  in queue_delayed_work()  argument
218 		return queue_work(wq, &dwork->work);  in queue_delayed_work()
220 	return queue_delayed_work_on(-1, wq, dwork, delay);  in queue_delayed_work()
233 int queue_delayed_work_on(int cpu, struct workqueue_struct *wq,  in queue_delayed_work_on()  argument
247 		set_wq_data(work, wq_per_cpu(wq, raw_smp_processor_id()));  in queue_delayed_work_on()
294 		lock_map_acquire(&cwq->wq->lockdep_map);  in run_workqueue()
298 		lock_map_release(&cwq->wq->lockdep_map);  in run_workqueue()
323 	if (cwq->wq->freezeable)  in worker_thread()
411 void flush_workqueue(struct workqueue_struct *wq)  in flush_workqueue()  argument
413 	const struct cpumask *cpu_map = wq_cpu_map(wq);  in flush_workqueue()
417 	lock_map_acquire(&wq->lockdep_map);  in flush_workqueue()
418 	lock_map_release(&wq->lockdep_map);  in flush_workqueue()
420 		flush_cpu_workqueue(per_cpu_ptr(wq->cpu_wq, cpu));  in flush_workqueue()
445 	lock_map_acquire(&cwq->wq->lockdep_map);  in flush_work()
446 	lock_map_release(&cwq->wq->lockdep_map);  in flush_work()
534 	struct workqueue_struct *wq;  in wait_on_work()  local
547 	wq = cwq->wq;  in wait_on_work()
548 	cpu_map = wq_cpu_map(wq);  in wait_on_work()
551 		wait_on_cpu_work(per_cpu_ptr(wq->cpu_wq, cpu), work);  in wait_on_work()
756 init_cpu_workqueue(struct workqueue_struct *wq, int cpu)  in init_cpu_workqueue()  argument
758 	struct cpu_workqueue_struct *cwq = per_cpu_ptr(wq->cpu_wq, cpu);  in init_cpu_workqueue()
760 	cwq->wq = wq;  in init_cpu_workqueue()
771 	struct workqueue_struct *wq = cwq->wq;  in create_workqueue_thread()  local
772 	const char *fmt = is_wq_single_threaded(wq) ? "%s" : "%s/%d";  in create_workqueue_thread()
775 	p = kthread_create(worker_thread, cwq, fmt, wq->name, cpu);  in create_workqueue_thread()
786 	if (cwq->wq->rt)  in create_workqueue_thread()
811 	struct workqueue_struct *wq;  in __create_workqueue_key()  local
815 	wq = kzalloc(sizeof(*wq), GFP_KERNEL);  in __create_workqueue_key()
816 	if (!wq)  in __create_workqueue_key()
819 	wq->cpu_wq = alloc_percpu(struct cpu_workqueue_struct);  in __create_workqueue_key()
820 	if (!wq->cpu_wq) {  in __create_workqueue_key()
821 		kfree(wq);  in __create_workqueue_key()
825 	wq->name = name;  in __create_workqueue_key()
826 	lockdep_init_map(&wq->lockdep_map, lock_name, key, 0);  in __create_workqueue_key()
827 	wq->singlethread = singlethread;  in __create_workqueue_key()
828 	wq->freezeable = freezeable;  in __create_workqueue_key()
829 	wq->rt = rt;  in __create_workqueue_key()
830 	INIT_LIST_HEAD(&wq->list);  in __create_workqueue_key()
833 		cwq = init_cpu_workqueue(wq, singlethread_cpu);  in __create_workqueue_key()
845 		list_add(&wq->list, &workqueues);  in __create_workqueue_key()
854 			cwq = init_cpu_workqueue(wq, cpu);  in __create_workqueue_key()
864 		destroy_workqueue(wq);  in __create_workqueue_key()
865 		wq = NULL;  in __create_workqueue_key()
867 	return wq;  in __create_workqueue_key()
880 	lock_map_acquire(&cwq->wq->lockdep_map);  in cleanup_workqueue_thread()
881 	lock_map_release(&cwq->wq->lockdep_map);  in cleanup_workqueue_thread()
904 void destroy_workqueue(struct workqueue_struct *wq)  in destroy_workqueue()  argument
906 	const struct cpumask *cpu_map = wq_cpu_map(wq);  in destroy_workqueue()
911 	list_del(&wq->list);  in destroy_workqueue()
915 		cleanup_workqueue_thread(per_cpu_ptr(wq->cpu_wq, cpu));  in destroy_workqueue()
918 	free_percpu(wq->cpu_wq);  in destroy_workqueue()
919 	kfree(wq);  in destroy_workqueue()
929 	struct workqueue_struct *wq;  in workqueue_cpu_callback()  local
939 	list_for_each_entry(wq, &workqueues, list) {  in workqueue_cpu_callback()
940 		cwq = per_cpu_ptr(wq->cpu_wq, cpu);  in workqueue_cpu_callback()
947 				wq->name, cpu);  in workqueue_cpu_callback()