• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Kernel thread helper functions.
3  *   Copyright (C) 2004 IBM Corporation, Rusty Russell.
4  *   Copyright (C) 2009 Red Hat, Inc.
5  *
6  * Creation is done via kthreadd, so that we get a clean environment
7  * even if we're invoked from userspace (think modprobe, hotplug cpu,
8  * etc.).
9  */
10 #include <uapi/linux/sched/types.h>
11 #include <linux/mm.h>
12 #include <linux/mmu_context.h>
13 #include <linux/sched.h>
14 #include <linux/sched/mm.h>
15 #include <linux/sched/task.h>
16 #include <linux/kthread.h>
17 #include <linux/completion.h>
18 #include <linux/err.h>
19 #include <linux/cgroup.h>
20 #include <linux/cpuset.h>
21 #include <linux/unistd.h>
22 #include <linux/file.h>
23 #include <linux/export.h>
24 #include <linux/mutex.h>
25 #include <linux/slab.h>
26 #include <linux/freezer.h>
27 #include <linux/ptrace.h>
28 #include <linux/uaccess.h>
29 #include <linux/numa.h>
30 #include <linux/sched/isolation.h>
31 #include <trace/events/sched.h>
32 
33 
34 static DEFINE_SPINLOCK(kthread_create_lock);
35 static LIST_HEAD(kthread_create_list);
36 struct task_struct *kthreadd_task;
37 
38 struct kthread_create_info
39 {
40 	/* Information passed to kthread() from kthreadd. */
41 	int (*threadfn)(void *data);
42 	void *data;
43 	int node;
44 
45 	/* Result passed back to kthread_create() from kthreadd. */
46 	struct task_struct *result;
47 	struct completion *done;
48 
49 	struct list_head list;
50 };
51 
52 struct kthread {
53 	unsigned long flags;
54 	unsigned int cpu;
55 	int (*threadfn)(void *);
56 	void *data;
57 	mm_segment_t oldfs;
58 	struct completion parked;
59 	struct completion exited;
60 #ifdef CONFIG_BLK_CGROUP
61 	struct cgroup_subsys_state *blkcg_css;
62 #endif
63 };
64 
65 enum KTHREAD_BITS {
66 	KTHREAD_IS_PER_CPU = 0,
67 	KTHREAD_SHOULD_STOP,
68 	KTHREAD_SHOULD_PARK,
69 };
70 
set_kthread_struct(void * kthread)71 static inline void set_kthread_struct(void *kthread)
72 {
73 	/*
74 	 * We abuse ->set_child_tid to avoid the new member and because it
75 	 * can't be wrongly copied by copy_process(). We also rely on fact
76 	 * that the caller can't exec, so PF_KTHREAD can't be cleared.
77 	 */
78 	current->set_child_tid = (__force void __user *)kthread;
79 }
80 
to_kthread(struct task_struct * k)81 static inline struct kthread *to_kthread(struct task_struct *k)
82 {
83 	WARN_ON(!(k->flags & PF_KTHREAD));
84 	return (__force void *)k->set_child_tid;
85 }
86 
87 /*
88  * Variant of to_kthread() that doesn't assume @p is a kthread.
89  *
90  * Per construction; when:
91  *
92  *   (p->flags & PF_KTHREAD) && p->set_child_tid
93  *
94  * the task is both a kthread and struct kthread is persistent. However
95  * PF_KTHREAD on it's own is not, kernel_thread() can exec() (See umh.c and
96  * begin_new_exec()).
97  */
__to_kthread(struct task_struct * p)98 static inline struct kthread *__to_kthread(struct task_struct *p)
99 {
100 	void *kthread = (__force void *)p->set_child_tid;
101 	if (kthread && !(p->flags & PF_KTHREAD))
102 		kthread = NULL;
103 	return kthread;
104 }
105 
free_kthread_struct(struct task_struct * k)106 void free_kthread_struct(struct task_struct *k)
107 {
108 	struct kthread *kthread;
109 
110 	/*
111 	 * Can be NULL if this kthread was created by kernel_thread()
112 	 * or if kmalloc() in kthread() failed.
113 	 */
114 	kthread = to_kthread(k);
115 #ifdef CONFIG_BLK_CGROUP
116 	WARN_ON_ONCE(kthread && kthread->blkcg_css);
117 #endif
118 	kfree(kthread);
119 }
120 
121 /**
122  * kthread_should_stop - should this kthread return now?
123  *
124  * When someone calls kthread_stop() on your kthread, it will be woken
125  * and this will return true.  You should then return, and your return
126  * value will be passed through to kthread_stop().
127  */
kthread_should_stop(void)128 bool kthread_should_stop(void)
129 {
130 	return test_bit(KTHREAD_SHOULD_STOP, &to_kthread(current)->flags);
131 }
132 EXPORT_SYMBOL(kthread_should_stop);
133 
__kthread_should_park(struct task_struct * k)134 bool __kthread_should_park(struct task_struct *k)
135 {
136 	return test_bit(KTHREAD_SHOULD_PARK, &to_kthread(k)->flags);
137 }
138 EXPORT_SYMBOL_GPL(__kthread_should_park);
139 
140 /**
141  * kthread_should_park - should this kthread park now?
142  *
143  * When someone calls kthread_park() on your kthread, it will be woken
144  * and this will return true.  You should then do the necessary
145  * cleanup and call kthread_parkme()
146  *
147  * Similar to kthread_should_stop(), but this keeps the thread alive
148  * and in a park position. kthread_unpark() "restarts" the thread and
149  * calls the thread function again.
150  */
kthread_should_park(void)151 bool kthread_should_park(void)
152 {
153 	return __kthread_should_park(current);
154 }
155 EXPORT_SYMBOL_GPL(kthread_should_park);
156 
157 /**
158  * kthread_freezable_should_stop - should this freezable kthread return now?
159  * @was_frozen: optional out parameter, indicates whether %current was frozen
160  *
161  * kthread_should_stop() for freezable kthreads, which will enter
162  * refrigerator if necessary.  This function is safe from kthread_stop() /
163  * freezer deadlock and freezable kthreads should use this function instead
164  * of calling try_to_freeze() directly.
165  */
kthread_freezable_should_stop(bool * was_frozen)166 bool kthread_freezable_should_stop(bool *was_frozen)
167 {
168 	bool frozen = false;
169 
170 	might_sleep();
171 
172 	if (unlikely(freezing(current)))
173 		frozen = __refrigerator(true);
174 
175 	if (was_frozen)
176 		*was_frozen = frozen;
177 
178 	return kthread_should_stop();
179 }
180 EXPORT_SYMBOL_GPL(kthread_freezable_should_stop);
181 
182 /**
183  * kthread_func - return the function specified on kthread creation
184  * @task: kthread task in question
185  *
186  * Returns NULL if the task is not a kthread.
187  */
kthread_func(struct task_struct * task)188 void *kthread_func(struct task_struct *task)
189 {
190 	struct kthread *kthread = __to_kthread(task);
191 	if (kthread)
192 		return kthread->threadfn;
193 	return NULL;
194 }
195 EXPORT_SYMBOL_GPL(kthread_func);
196 
197 /**
198  * kthread_data - return data value specified on kthread creation
199  * @task: kthread task in question
200  *
201  * Return the data value specified when kthread @task was created.
202  * The caller is responsible for ensuring the validity of @task when
203  * calling this function.
204  */
kthread_data(struct task_struct * task)205 void *kthread_data(struct task_struct *task)
206 {
207 	return to_kthread(task)->data;
208 }
209 EXPORT_SYMBOL_GPL(kthread_data);
210 
211 /**
212  * kthread_probe_data - speculative version of kthread_data()
213  * @task: possible kthread task in question
214  *
215  * @task could be a kthread task.  Return the data value specified when it
216  * was created if accessible.  If @task isn't a kthread task or its data is
217  * inaccessible for any reason, %NULL is returned.  This function requires
218  * that @task itself is safe to dereference.
219  */
kthread_probe_data(struct task_struct * task)220 void *kthread_probe_data(struct task_struct *task)
221 {
222 	struct kthread *kthread = __to_kthread(task);
223 	void *data = NULL;
224 
225 	if (kthread)
226 		copy_from_kernel_nofault(&data, &kthread->data, sizeof(data));
227 	return data;
228 }
229 
__kthread_parkme(struct kthread * self)230 static void __kthread_parkme(struct kthread *self)
231 {
232 	for (;;) {
233 		/*
234 		 * TASK_PARKED is a special state; we must serialize against
235 		 * possible pending wakeups to avoid store-store collisions on
236 		 * task->state.
237 		 *
238 		 * Such a collision might possibly result in the task state
239 		 * changin from TASK_PARKED and us failing the
240 		 * wait_task_inactive() in kthread_park().
241 		 */
242 		set_special_state(TASK_PARKED);
243 		if (!test_bit(KTHREAD_SHOULD_PARK, &self->flags))
244 			break;
245 
246 		/*
247 		 * Thread is going to call schedule(), do not preempt it,
248 		 * or the caller of kthread_park() may spend more time in
249 		 * wait_task_inactive().
250 		 */
251 		preempt_disable();
252 		complete(&self->parked);
253 		schedule_preempt_disabled();
254 		preempt_enable();
255 	}
256 	__set_current_state(TASK_RUNNING);
257 }
258 
kthread_parkme(void)259 void kthread_parkme(void)
260 {
261 	__kthread_parkme(to_kthread(current));
262 }
263 EXPORT_SYMBOL_GPL(kthread_parkme);
264 
kthread(void * _create)265 static int kthread(void *_create)
266 {
267 	/* Copy data: it's on kthread's stack */
268 	struct kthread_create_info *create = _create;
269 	int (*threadfn)(void *data) = create->threadfn;
270 	void *data = create->data;
271 	struct completion *done;
272 	struct kthread *self;
273 	int ret;
274 
275 	self = kzalloc(sizeof(*self), GFP_KERNEL);
276 	set_kthread_struct(self);
277 
278 	/* If user was SIGKILLed, I release the structure. */
279 	done = xchg(&create->done, NULL);
280 	if (!done) {
281 		kfree(create);
282 		do_exit(-EINTR);
283 	}
284 
285 	if (!self) {
286 		create->result = ERR_PTR(-ENOMEM);
287 		complete(done);
288 		do_exit(-ENOMEM);
289 	}
290 
291 	self->threadfn = threadfn;
292 	self->data = data;
293 	init_completion(&self->exited);
294 	init_completion(&self->parked);
295 	current->vfork_done = &self->exited;
296 
297 	/* OK, tell user we're spawned, wait for stop or wakeup */
298 	__set_current_state(TASK_UNINTERRUPTIBLE);
299 	create->result = current;
300 	/*
301 	 * Thread is going to call schedule(), do not preempt it,
302 	 * or the creator may spend more time in wait_task_inactive().
303 	 */
304 	preempt_disable();
305 	complete(done);
306 	schedule_preempt_disabled();
307 	preempt_enable();
308 
309 	ret = -EINTR;
310 	if (!test_bit(KTHREAD_SHOULD_STOP, &self->flags)) {
311 		cgroup_kthread_ready();
312 		__kthread_parkme(self);
313 		ret = threadfn(data);
314 	}
315 	do_exit(ret);
316 }
317 
318 /* called from do_fork() to get node information for about to be created task */
tsk_fork_get_node(struct task_struct * tsk)319 int tsk_fork_get_node(struct task_struct *tsk)
320 {
321 #ifdef CONFIG_NUMA
322 	if (tsk == kthreadd_task)
323 		return tsk->pref_node_fork;
324 #endif
325 	return NUMA_NO_NODE;
326 }
327 
create_kthread(struct kthread_create_info * create)328 static void create_kthread(struct kthread_create_info *create)
329 {
330 	int pid;
331 
332 #ifdef CONFIG_NUMA
333 	current->pref_node_fork = create->node;
334 #endif
335 	/* We want our own signal handler (we take no signals by default). */
336 	pid = kernel_thread(kthread, create, CLONE_FS | CLONE_FILES | SIGCHLD);
337 	if (pid < 0) {
338 		/* If user was SIGKILLed, I release the structure. */
339 		struct completion *done = xchg(&create->done, NULL);
340 
341 		if (!done) {
342 			kfree(create);
343 			return;
344 		}
345 		create->result = ERR_PTR(pid);
346 		complete(done);
347 	}
348 }
349 
350 static __printf(4, 0)
__kthread_create_on_node(int (* threadfn)(void * data),void * data,int node,const char namefmt[],va_list args)351 struct task_struct *__kthread_create_on_node(int (*threadfn)(void *data),
352 						    void *data, int node,
353 						    const char namefmt[],
354 						    va_list args)
355 {
356 	DECLARE_COMPLETION_ONSTACK(done);
357 	struct task_struct *task;
358 	struct kthread_create_info *create = kmalloc(sizeof(*create),
359 						     GFP_KERNEL);
360 
361 	if (!create)
362 		return ERR_PTR(-ENOMEM);
363 	create->threadfn = threadfn;
364 	create->data = data;
365 	create->node = node;
366 	create->done = &done;
367 
368 	spin_lock(&kthread_create_lock);
369 	list_add_tail(&create->list, &kthread_create_list);
370 	spin_unlock(&kthread_create_lock);
371 
372 	wake_up_process(kthreadd_task);
373 	/*
374 	 * Wait for completion in killable state, for I might be chosen by
375 	 * the OOM killer while kthreadd is trying to allocate memory for
376 	 * new kernel thread.
377 	 */
378 	if (unlikely(wait_for_completion_killable(&done))) {
379 		/*
380 		 * If I was SIGKILLed before kthreadd (or new kernel thread)
381 		 * calls complete(), leave the cleanup of this structure to
382 		 * that thread.
383 		 */
384 		if (xchg(&create->done, NULL))
385 			return ERR_PTR(-EINTR);
386 		/*
387 		 * kthreadd (or new kernel thread) will call complete()
388 		 * shortly.
389 		 */
390 		wait_for_completion(&done);
391 	}
392 	task = create->result;
393 	if (!IS_ERR(task)) {
394 		static const struct sched_param param = { .sched_priority = 0 };
395 		char name[TASK_COMM_LEN];
396 
397 		/*
398 		 * task is already visible to other tasks, so updating
399 		 * COMM must be protected.
400 		 */
401 		vsnprintf(name, sizeof(name), namefmt, args);
402 		set_task_comm(task, name);
403 		/*
404 		 * root may have changed our (kthreadd's) priority or CPU mask.
405 		 * The kernel thread should not inherit these properties.
406 		 */
407 		sched_setscheduler_nocheck(task, SCHED_NORMAL, &param);
408 		set_cpus_allowed_ptr(task,
409 				     housekeeping_cpumask(HK_FLAG_KTHREAD));
410 	}
411 	kfree(create);
412 	return task;
413 }
414 
415 /**
416  * kthread_create_on_node - create a kthread.
417  * @threadfn: the function to run until signal_pending(current).
418  * @data: data ptr for @threadfn.
419  * @node: task and thread structures for the thread are allocated on this node
420  * @namefmt: printf-style name for the thread.
421  *
422  * Description: This helper function creates and names a kernel
423  * thread.  The thread will be stopped: use wake_up_process() to start
424  * it.  See also kthread_run().  The new thread has SCHED_NORMAL policy and
425  * is affine to all CPUs.
426  *
427  * If thread is going to be bound on a particular cpu, give its node
428  * in @node, to get NUMA affinity for kthread stack, or else give NUMA_NO_NODE.
429  * When woken, the thread will run @threadfn() with @data as its
430  * argument. @threadfn() can either call do_exit() directly if it is a
431  * standalone thread for which no one will call kthread_stop(), or
432  * return when 'kthread_should_stop()' is true (which means
433  * kthread_stop() has been called).  The return value should be zero
434  * or a negative error number; it will be passed to kthread_stop().
435  *
436  * Returns a task_struct or ERR_PTR(-ENOMEM) or ERR_PTR(-EINTR).
437  */
kthread_create_on_node(int (* threadfn)(void * data),void * data,int node,const char namefmt[],...)438 struct task_struct *kthread_create_on_node(int (*threadfn)(void *data),
439 					   void *data, int node,
440 					   const char namefmt[],
441 					   ...)
442 {
443 	struct task_struct *task;
444 	va_list args;
445 
446 	va_start(args, namefmt);
447 	task = __kthread_create_on_node(threadfn, data, node, namefmt, args);
448 	va_end(args);
449 
450 	return task;
451 }
452 EXPORT_SYMBOL(kthread_create_on_node);
453 
__kthread_bind_mask(struct task_struct * p,const struct cpumask * mask,long state)454 static void __kthread_bind_mask(struct task_struct *p, const struct cpumask *mask, long state)
455 {
456 	unsigned long flags;
457 
458 	if (!wait_task_inactive(p, state)) {
459 		WARN_ON(1);
460 		return;
461 	}
462 
463 	/* It's safe because the task is inactive. */
464 	raw_spin_lock_irqsave(&p->pi_lock, flags);
465 	do_set_cpus_allowed(p, mask);
466 	p->flags |= PF_NO_SETAFFINITY;
467 	raw_spin_unlock_irqrestore(&p->pi_lock, flags);
468 }
469 
__kthread_bind(struct task_struct * p,unsigned int cpu,long state)470 static void __kthread_bind(struct task_struct *p, unsigned int cpu, long state)
471 {
472 	__kthread_bind_mask(p, cpumask_of(cpu), state);
473 }
474 
kthread_bind_mask(struct task_struct * p,const struct cpumask * mask)475 void kthread_bind_mask(struct task_struct *p, const struct cpumask *mask)
476 {
477 	__kthread_bind_mask(p, mask, TASK_UNINTERRUPTIBLE);
478 }
479 
480 /**
481  * kthread_bind - bind a just-created kthread to a cpu.
482  * @p: thread created by kthread_create().
483  * @cpu: cpu (might not be online, must be possible) for @k to run on.
484  *
485  * Description: This function is equivalent to set_cpus_allowed(),
486  * except that @cpu doesn't need to be online, and the thread must be
487  * stopped (i.e., just returned from kthread_create()).
488  */
kthread_bind(struct task_struct * p,unsigned int cpu)489 void kthread_bind(struct task_struct *p, unsigned int cpu)
490 {
491 	__kthread_bind(p, cpu, TASK_UNINTERRUPTIBLE);
492 }
493 EXPORT_SYMBOL(kthread_bind);
494 
495 /**
496  * kthread_create_on_cpu - Create a cpu bound kthread
497  * @threadfn: the function to run until signal_pending(current).
498  * @data: data ptr for @threadfn.
499  * @cpu: The cpu on which the thread should be bound,
500  * @namefmt: printf-style name for the thread. Format is restricted
501  *	     to "name.*%u". Code fills in cpu number.
502  *
503  * Description: This helper function creates and names a kernel thread
504  */
kthread_create_on_cpu(int (* threadfn)(void * data),void * data,unsigned int cpu,const char * namefmt)505 struct task_struct *kthread_create_on_cpu(int (*threadfn)(void *data),
506 					  void *data, unsigned int cpu,
507 					  const char *namefmt)
508 {
509 	struct task_struct *p;
510 
511 	p = kthread_create_on_node(threadfn, data, cpu_to_node(cpu), namefmt,
512 				   cpu);
513 	if (IS_ERR(p))
514 		return p;
515 	kthread_bind(p, cpu);
516 	/* CPU hotplug need to bind once again when unparking the thread. */
517 	to_kthread(p)->cpu = cpu;
518 	return p;
519 }
520 
kthread_set_per_cpu(struct task_struct * k,int cpu)521 void kthread_set_per_cpu(struct task_struct *k, int cpu)
522 {
523 	struct kthread *kthread = to_kthread(k);
524 	if (!kthread)
525 		return;
526 
527 	WARN_ON_ONCE(!(k->flags & PF_NO_SETAFFINITY));
528 
529 	if (cpu < 0) {
530 		clear_bit(KTHREAD_IS_PER_CPU, &kthread->flags);
531 		return;
532 	}
533 
534 	kthread->cpu = cpu;
535 	set_bit(KTHREAD_IS_PER_CPU, &kthread->flags);
536 }
537 
kthread_is_per_cpu(struct task_struct * p)538 bool kthread_is_per_cpu(struct task_struct *p)
539 {
540 	struct kthread *kthread = __to_kthread(p);
541 	if (!kthread)
542 		return false;
543 
544 	return test_bit(KTHREAD_IS_PER_CPU, &kthread->flags);
545 }
546 
547 /**
548  * kthread_unpark - unpark a thread created by kthread_create().
549  * @k:		thread created by kthread_create().
550  *
551  * Sets kthread_should_park() for @k to return false, wakes it, and
552  * waits for it to return. If the thread is marked percpu then its
553  * bound to the cpu again.
554  */
kthread_unpark(struct task_struct * k)555 void kthread_unpark(struct task_struct *k)
556 {
557 	struct kthread *kthread = to_kthread(k);
558 
559 	/*
560 	 * Newly created kthread was parked when the CPU was offline.
561 	 * The binding was lost and we need to set it again.
562 	 */
563 	if (test_bit(KTHREAD_IS_PER_CPU, &kthread->flags))
564 		__kthread_bind(k, kthread->cpu, TASK_PARKED);
565 
566 	clear_bit(KTHREAD_SHOULD_PARK, &kthread->flags);
567 	/*
568 	 * __kthread_parkme() will either see !SHOULD_PARK or get the wakeup.
569 	 */
570 	wake_up_state(k, TASK_PARKED);
571 }
572 EXPORT_SYMBOL_GPL(kthread_unpark);
573 
574 /**
575  * kthread_park - park a thread created by kthread_create().
576  * @k: thread created by kthread_create().
577  *
578  * Sets kthread_should_park() for @k to return true, wakes it, and
579  * waits for it to return. This can also be called after kthread_create()
580  * instead of calling wake_up_process(): the thread will park without
581  * calling threadfn().
582  *
583  * Returns 0 if the thread is parked, -ENOSYS if the thread exited.
584  * If called by the kthread itself just the park bit is set.
585  */
kthread_park(struct task_struct * k)586 int kthread_park(struct task_struct *k)
587 {
588 	struct kthread *kthread = to_kthread(k);
589 
590 	if (WARN_ON(k->flags & PF_EXITING))
591 		return -ENOSYS;
592 
593 	if (WARN_ON_ONCE(test_bit(KTHREAD_SHOULD_PARK, &kthread->flags)))
594 		return -EBUSY;
595 
596 	set_bit(KTHREAD_SHOULD_PARK, &kthread->flags);
597 	if (k != current) {
598 		wake_up_process(k);
599 		/*
600 		 * Wait for __kthread_parkme() to complete(), this means we
601 		 * _will_ have TASK_PARKED and are about to call schedule().
602 		 */
603 		wait_for_completion(&kthread->parked);
604 		/*
605 		 * Now wait for that schedule() to complete and the task to
606 		 * get scheduled out.
607 		 */
608 		WARN_ON_ONCE(!wait_task_inactive(k, TASK_PARKED));
609 	}
610 
611 	return 0;
612 }
613 EXPORT_SYMBOL_GPL(kthread_park);
614 
615 /**
616  * kthread_stop - stop a thread created by kthread_create().
617  * @k: thread created by kthread_create().
618  *
619  * Sets kthread_should_stop() for @k to return true, wakes it, and
620  * waits for it to exit. This can also be called after kthread_create()
621  * instead of calling wake_up_process(): the thread will exit without
622  * calling threadfn().
623  *
624  * If threadfn() may call do_exit() itself, the caller must ensure
625  * task_struct can't go away.
626  *
627  * Returns the result of threadfn(), or %-EINTR if wake_up_process()
628  * was never called.
629  */
kthread_stop(struct task_struct * k)630 int kthread_stop(struct task_struct *k)
631 {
632 	struct kthread *kthread;
633 	int ret;
634 
635 	trace_sched_kthread_stop(k);
636 
637 	get_task_struct(k);
638 	kthread = to_kthread(k);
639 	set_bit(KTHREAD_SHOULD_STOP, &kthread->flags);
640 	kthread_unpark(k);
641 	wake_up_process(k);
642 	wait_for_completion(&kthread->exited);
643 	ret = k->exit_code;
644 	put_task_struct(k);
645 
646 	trace_sched_kthread_stop_ret(ret);
647 	return ret;
648 }
649 EXPORT_SYMBOL(kthread_stop);
650 
kthreadd(void * unused)651 int kthreadd(void *unused)
652 {
653 	struct task_struct *tsk = current;
654 
655 	/* Setup a clean context for our children to inherit. */
656 	set_task_comm(tsk, "kthreadd");
657 	ignore_signals(tsk);
658 	set_cpus_allowed_ptr(tsk, housekeeping_cpumask(HK_FLAG_KTHREAD));
659 	set_mems_allowed(node_states[N_MEMORY]);
660 
661 	current->flags |= PF_NOFREEZE;
662 	cgroup_init_kthreadd();
663 
664 	for (;;) {
665 		set_current_state(TASK_INTERRUPTIBLE);
666 		if (list_empty(&kthread_create_list))
667 			schedule();
668 		__set_current_state(TASK_RUNNING);
669 
670 		spin_lock(&kthread_create_lock);
671 		while (!list_empty(&kthread_create_list)) {
672 			struct kthread_create_info *create;
673 
674 			create = list_entry(kthread_create_list.next,
675 					    struct kthread_create_info, list);
676 			list_del_init(&create->list);
677 			spin_unlock(&kthread_create_lock);
678 
679 			create_kthread(create);
680 
681 			spin_lock(&kthread_create_lock);
682 		}
683 		spin_unlock(&kthread_create_lock);
684 	}
685 
686 	return 0;
687 }
688 
__kthread_init_worker(struct kthread_worker * worker,const char * name,struct lock_class_key * key)689 void __kthread_init_worker(struct kthread_worker *worker,
690 				const char *name,
691 				struct lock_class_key *key)
692 {
693 	memset(worker, 0, sizeof(struct kthread_worker));
694 	raw_spin_lock_init(&worker->lock);
695 	lockdep_set_class_and_name(&worker->lock, key, name);
696 	INIT_LIST_HEAD(&worker->work_list);
697 	INIT_LIST_HEAD(&worker->delayed_work_list);
698 }
699 EXPORT_SYMBOL_GPL(__kthread_init_worker);
700 
701 /**
702  * kthread_worker_fn - kthread function to process kthread_worker
703  * @worker_ptr: pointer to initialized kthread_worker
704  *
705  * This function implements the main cycle of kthread worker. It processes
706  * work_list until it is stopped with kthread_stop(). It sleeps when the queue
707  * is empty.
708  *
709  * The works are not allowed to keep any locks, disable preemption or interrupts
710  * when they finish. There is defined a safe point for freezing when one work
711  * finishes and before a new one is started.
712  *
713  * Also the works must not be handled by more than one worker at the same time,
714  * see also kthread_queue_work().
715  */
kthread_worker_fn(void * worker_ptr)716 int kthread_worker_fn(void *worker_ptr)
717 {
718 	struct kthread_worker *worker = worker_ptr;
719 	struct kthread_work *work;
720 
721 	/*
722 	 * FIXME: Update the check and remove the assignment when all kthread
723 	 * worker users are created using kthread_create_worker*() functions.
724 	 */
725 	WARN_ON(worker->task && worker->task != current);
726 	worker->task = current;
727 
728 	if (worker->flags & KTW_FREEZABLE)
729 		set_freezable();
730 
731 repeat:
732 	set_current_state(TASK_INTERRUPTIBLE);	/* mb paired w/ kthread_stop */
733 
734 	if (kthread_should_stop()) {
735 		__set_current_state(TASK_RUNNING);
736 		raw_spin_lock_irq(&worker->lock);
737 		worker->task = NULL;
738 		raw_spin_unlock_irq(&worker->lock);
739 		return 0;
740 	}
741 
742 	work = NULL;
743 	raw_spin_lock_irq(&worker->lock);
744 	if (!list_empty(&worker->work_list)) {
745 		work = list_first_entry(&worker->work_list,
746 					struct kthread_work, node);
747 		list_del_init(&work->node);
748 	}
749 	worker->current_work = work;
750 	raw_spin_unlock_irq(&worker->lock);
751 
752 	if (work) {
753 		__set_current_state(TASK_RUNNING);
754 		work->func(work);
755 	} else if (!freezing(current))
756 		schedule();
757 
758 	try_to_freeze();
759 	cond_resched();
760 	goto repeat;
761 }
762 EXPORT_SYMBOL_GPL(kthread_worker_fn);
763 
764 static __printf(3, 0) struct kthread_worker *
__kthread_create_worker(int cpu,unsigned int flags,const char namefmt[],va_list args)765 __kthread_create_worker(int cpu, unsigned int flags,
766 			const char namefmt[], va_list args)
767 {
768 	struct kthread_worker *worker;
769 	struct task_struct *task;
770 	int node = NUMA_NO_NODE;
771 
772 	worker = kzalloc(sizeof(*worker), GFP_KERNEL);
773 	if (!worker)
774 		return ERR_PTR(-ENOMEM);
775 
776 	kthread_init_worker(worker);
777 
778 	if (cpu >= 0)
779 		node = cpu_to_node(cpu);
780 
781 	task = __kthread_create_on_node(kthread_worker_fn, worker,
782 						node, namefmt, args);
783 	if (IS_ERR(task))
784 		goto fail_task;
785 
786 	if (cpu >= 0)
787 		kthread_bind(task, cpu);
788 
789 	worker->flags = flags;
790 	worker->task = task;
791 	wake_up_process(task);
792 	return worker;
793 
794 fail_task:
795 	kfree(worker);
796 	return ERR_CAST(task);
797 }
798 
799 /**
800  * kthread_create_worker - create a kthread worker
801  * @flags: flags modifying the default behavior of the worker
802  * @namefmt: printf-style name for the kthread worker (task).
803  *
804  * Returns a pointer to the allocated worker on success, ERR_PTR(-ENOMEM)
805  * when the needed structures could not get allocated, and ERR_PTR(-EINTR)
806  * when the worker was SIGKILLed.
807  */
808 struct kthread_worker *
kthread_create_worker(unsigned int flags,const char namefmt[],...)809 kthread_create_worker(unsigned int flags, const char namefmt[], ...)
810 {
811 	struct kthread_worker *worker;
812 	va_list args;
813 
814 	va_start(args, namefmt);
815 	worker = __kthread_create_worker(-1, flags, namefmt, args);
816 	va_end(args);
817 
818 	return worker;
819 }
820 EXPORT_SYMBOL(kthread_create_worker);
821 
822 /**
823  * kthread_create_worker_on_cpu - create a kthread worker and bind it
824  *	to a given CPU and the associated NUMA node.
825  * @cpu: CPU number
826  * @flags: flags modifying the default behavior of the worker
827  * @namefmt: printf-style name for the kthread worker (task).
828  *
829  * Use a valid CPU number if you want to bind the kthread worker
830  * to the given CPU and the associated NUMA node.
831  *
832  * A good practice is to add the cpu number also into the worker name.
833  * For example, use kthread_create_worker_on_cpu(cpu, "helper/%d", cpu).
834  *
835  * Returns a pointer to the allocated worker on success, ERR_PTR(-ENOMEM)
836  * when the needed structures could not get allocated, and ERR_PTR(-EINTR)
837  * when the worker was SIGKILLed.
838  */
839 struct kthread_worker *
kthread_create_worker_on_cpu(int cpu,unsigned int flags,const char namefmt[],...)840 kthread_create_worker_on_cpu(int cpu, unsigned int flags,
841 			     const char namefmt[], ...)
842 {
843 	struct kthread_worker *worker;
844 	va_list args;
845 
846 	va_start(args, namefmt);
847 	worker = __kthread_create_worker(cpu, flags, namefmt, args);
848 	va_end(args);
849 
850 	return worker;
851 }
852 EXPORT_SYMBOL(kthread_create_worker_on_cpu);
853 
854 /*
855  * Returns true when the work could not be queued at the moment.
856  * It happens when it is already pending in a worker list
857  * or when it is being cancelled.
858  */
queuing_blocked(struct kthread_worker * worker,struct kthread_work * work)859 static inline bool queuing_blocked(struct kthread_worker *worker,
860 				   struct kthread_work *work)
861 {
862 	lockdep_assert_held(&worker->lock);
863 
864 	return !list_empty(&work->node) || work->canceling;
865 }
866 
kthread_insert_work_sanity_check(struct kthread_worker * worker,struct kthread_work * work)867 static void kthread_insert_work_sanity_check(struct kthread_worker *worker,
868 					     struct kthread_work *work)
869 {
870 	lockdep_assert_held(&worker->lock);
871 	WARN_ON_ONCE(!list_empty(&work->node));
872 	/* Do not use a work with >1 worker, see kthread_queue_work() */
873 	WARN_ON_ONCE(work->worker && work->worker != worker);
874 }
875 
876 /* insert @work before @pos in @worker */
kthread_insert_work(struct kthread_worker * worker,struct kthread_work * work,struct list_head * pos)877 static void kthread_insert_work(struct kthread_worker *worker,
878 				struct kthread_work *work,
879 				struct list_head *pos)
880 {
881 	kthread_insert_work_sanity_check(worker, work);
882 
883 	list_add_tail(&work->node, pos);
884 	work->worker = worker;
885 	if (!worker->current_work && likely(worker->task))
886 		wake_up_process(worker->task);
887 }
888 
889 /**
890  * kthread_queue_work - queue a kthread_work
891  * @worker: target kthread_worker
892  * @work: kthread_work to queue
893  *
894  * Queue @work to work processor @task for async execution.  @task
895  * must have been created with kthread_worker_create().  Returns %true
896  * if @work was successfully queued, %false if it was already pending.
897  *
898  * Reinitialize the work if it needs to be used by another worker.
899  * For example, when the worker was stopped and started again.
900  */
kthread_queue_work(struct kthread_worker * worker,struct kthread_work * work)901 bool kthread_queue_work(struct kthread_worker *worker,
902 			struct kthread_work *work)
903 {
904 	bool ret = false;
905 	unsigned long flags;
906 
907 	raw_spin_lock_irqsave(&worker->lock, flags);
908 	if (!queuing_blocked(worker, work)) {
909 		kthread_insert_work(worker, work, &worker->work_list);
910 		ret = true;
911 	}
912 	raw_spin_unlock_irqrestore(&worker->lock, flags);
913 	return ret;
914 }
915 EXPORT_SYMBOL_GPL(kthread_queue_work);
916 
917 /**
918  * kthread_delayed_work_timer_fn - callback that queues the associated kthread
919  *	delayed work when the timer expires.
920  * @t: pointer to the expired timer
921  *
922  * The format of the function is defined by struct timer_list.
923  * It should have been called from irqsafe timer with irq already off.
924  */
kthread_delayed_work_timer_fn(struct timer_list * t)925 void kthread_delayed_work_timer_fn(struct timer_list *t)
926 {
927 	struct kthread_delayed_work *dwork = from_timer(dwork, t, timer);
928 	struct kthread_work *work = &dwork->work;
929 	struct kthread_worker *worker = work->worker;
930 	unsigned long flags;
931 
932 	/*
933 	 * This might happen when a pending work is reinitialized.
934 	 * It means that it is used a wrong way.
935 	 */
936 	if (WARN_ON_ONCE(!worker))
937 		return;
938 
939 	raw_spin_lock_irqsave(&worker->lock, flags);
940 	/* Work must not be used with >1 worker, see kthread_queue_work(). */
941 	WARN_ON_ONCE(work->worker != worker);
942 
943 	/* Move the work from worker->delayed_work_list. */
944 	WARN_ON_ONCE(list_empty(&work->node));
945 	list_del_init(&work->node);
946 	if (!work->canceling)
947 		kthread_insert_work(worker, work, &worker->work_list);
948 
949 	raw_spin_unlock_irqrestore(&worker->lock, flags);
950 }
951 EXPORT_SYMBOL(kthread_delayed_work_timer_fn);
952 
__kthread_queue_delayed_work(struct kthread_worker * worker,struct kthread_delayed_work * dwork,unsigned long delay)953 static void __kthread_queue_delayed_work(struct kthread_worker *worker,
954 					 struct kthread_delayed_work *dwork,
955 					 unsigned long delay)
956 {
957 	struct timer_list *timer = &dwork->timer;
958 	struct kthread_work *work = &dwork->work;
959 
960 	WARN_ON_FUNCTION_MISMATCH(timer->function,
961 				  kthread_delayed_work_timer_fn);
962 
963 	/*
964 	 * If @delay is 0, queue @dwork->work immediately.  This is for
965 	 * both optimization and correctness.  The earliest @timer can
966 	 * expire is on the closest next tick and delayed_work users depend
967 	 * on that there's no such delay when @delay is 0.
968 	 */
969 	if (!delay) {
970 		kthread_insert_work(worker, work, &worker->work_list);
971 		return;
972 	}
973 
974 	/* Be paranoid and try to detect possible races already now. */
975 	kthread_insert_work_sanity_check(worker, work);
976 
977 	list_add(&work->node, &worker->delayed_work_list);
978 	work->worker = worker;
979 	timer->expires = jiffies + delay;
980 	add_timer(timer);
981 }
982 
983 /**
984  * kthread_queue_delayed_work - queue the associated kthread work
985  *	after a delay.
986  * @worker: target kthread_worker
987  * @dwork: kthread_delayed_work to queue
988  * @delay: number of jiffies to wait before queuing
989  *
990  * If the work has not been pending it starts a timer that will queue
991  * the work after the given @delay. If @delay is zero, it queues the
992  * work immediately.
993  *
994  * Return: %false if the @work has already been pending. It means that
995  * either the timer was running or the work was queued. It returns %true
996  * otherwise.
997  */
kthread_queue_delayed_work(struct kthread_worker * worker,struct kthread_delayed_work * dwork,unsigned long delay)998 bool kthread_queue_delayed_work(struct kthread_worker *worker,
999 				struct kthread_delayed_work *dwork,
1000 				unsigned long delay)
1001 {
1002 	struct kthread_work *work = &dwork->work;
1003 	unsigned long flags;
1004 	bool ret = false;
1005 
1006 	raw_spin_lock_irqsave(&worker->lock, flags);
1007 
1008 	if (!queuing_blocked(worker, work)) {
1009 		__kthread_queue_delayed_work(worker, dwork, delay);
1010 		ret = true;
1011 	}
1012 
1013 	raw_spin_unlock_irqrestore(&worker->lock, flags);
1014 	return ret;
1015 }
1016 EXPORT_SYMBOL_GPL(kthread_queue_delayed_work);
1017 
1018 struct kthread_flush_work {
1019 	struct kthread_work	work;
1020 	struct completion	done;
1021 };
1022 
kthread_flush_work_fn(struct kthread_work * work)1023 static void kthread_flush_work_fn(struct kthread_work *work)
1024 {
1025 	struct kthread_flush_work *fwork =
1026 		container_of(work, struct kthread_flush_work, work);
1027 	complete(&fwork->done);
1028 }
1029 
1030 /**
1031  * kthread_flush_work - flush a kthread_work
1032  * @work: work to flush
1033  *
1034  * If @work is queued or executing, wait for it to finish execution.
1035  */
kthread_flush_work(struct kthread_work * work)1036 void kthread_flush_work(struct kthread_work *work)
1037 {
1038 	struct kthread_flush_work fwork = {
1039 		KTHREAD_WORK_INIT(fwork.work, kthread_flush_work_fn),
1040 		COMPLETION_INITIALIZER_ONSTACK(fwork.done),
1041 	};
1042 	struct kthread_worker *worker;
1043 	bool noop = false;
1044 
1045 	worker = work->worker;
1046 	if (!worker)
1047 		return;
1048 
1049 	raw_spin_lock_irq(&worker->lock);
1050 	/* Work must not be used with >1 worker, see kthread_queue_work(). */
1051 	WARN_ON_ONCE(work->worker != worker);
1052 
1053 	if (!list_empty(&work->node))
1054 		kthread_insert_work(worker, &fwork.work, work->node.next);
1055 	else if (worker->current_work == work)
1056 		kthread_insert_work(worker, &fwork.work,
1057 				    worker->work_list.next);
1058 	else
1059 		noop = true;
1060 
1061 	raw_spin_unlock_irq(&worker->lock);
1062 
1063 	if (!noop)
1064 		wait_for_completion(&fwork.done);
1065 }
1066 EXPORT_SYMBOL_GPL(kthread_flush_work);
1067 
1068 /*
1069  * Make sure that the timer is neither set nor running and could
1070  * not manipulate the work list_head any longer.
1071  *
1072  * The function is called under worker->lock. The lock is temporary
1073  * released but the timer can't be set again in the meantime.
1074  */
kthread_cancel_delayed_work_timer(struct kthread_work * work,unsigned long * flags)1075 static void kthread_cancel_delayed_work_timer(struct kthread_work *work,
1076 					      unsigned long *flags)
1077 {
1078 	struct kthread_delayed_work *dwork =
1079 		container_of(work, struct kthread_delayed_work, work);
1080 	struct kthread_worker *worker = work->worker;
1081 
1082 	/*
1083 	 * del_timer_sync() must be called to make sure that the timer
1084 	 * callback is not running. The lock must be temporary released
1085 	 * to avoid a deadlock with the callback. In the meantime,
1086 	 * any queuing is blocked by setting the canceling counter.
1087 	 */
1088 	work->canceling++;
1089 	raw_spin_unlock_irqrestore(&worker->lock, *flags);
1090 	del_timer_sync(&dwork->timer);
1091 	raw_spin_lock_irqsave(&worker->lock, *flags);
1092 	work->canceling--;
1093 }
1094 
1095 /*
1096  * This function removes the work from the worker queue.
1097  *
1098  * It is called under worker->lock. The caller must make sure that
1099  * the timer used by delayed work is not running, e.g. by calling
1100  * kthread_cancel_delayed_work_timer().
1101  *
1102  * The work might still be in use when this function finishes. See the
1103  * current_work proceed by the worker.
1104  *
1105  * Return: %true if @work was pending and successfully canceled,
1106  *	%false if @work was not pending
1107  */
__kthread_cancel_work(struct kthread_work * work)1108 static bool __kthread_cancel_work(struct kthread_work *work)
1109 {
1110 	/*
1111 	 * Try to remove the work from a worker list. It might either
1112 	 * be from worker->work_list or from worker->delayed_work_list.
1113 	 */
1114 	if (!list_empty(&work->node)) {
1115 		list_del_init(&work->node);
1116 		return true;
1117 	}
1118 
1119 	return false;
1120 }
1121 
1122 /**
1123  * kthread_mod_delayed_work - modify delay of or queue a kthread delayed work
1124  * @worker: kthread worker to use
1125  * @dwork: kthread delayed work to queue
1126  * @delay: number of jiffies to wait before queuing
1127  *
1128  * If @dwork is idle, equivalent to kthread_queue_delayed_work(). Otherwise,
1129  * modify @dwork's timer so that it expires after @delay. If @delay is zero,
1130  * @work is guaranteed to be queued immediately.
1131  *
1132  * Return: %false if @dwork was idle and queued, %true otherwise.
1133  *
1134  * A special case is when the work is being canceled in parallel.
1135  * It might be caused either by the real kthread_cancel_delayed_work_sync()
1136  * or yet another kthread_mod_delayed_work() call. We let the other command
1137  * win and return %true here. The return value can be used for reference
1138  * counting and the number of queued works stays the same. Anyway, the caller
1139  * is supposed to synchronize these operations a reasonable way.
1140  *
1141  * This function is safe to call from any context including IRQ handler.
1142  * See __kthread_cancel_work() and kthread_delayed_work_timer_fn()
1143  * for details.
1144  */
kthread_mod_delayed_work(struct kthread_worker * worker,struct kthread_delayed_work * dwork,unsigned long delay)1145 bool kthread_mod_delayed_work(struct kthread_worker *worker,
1146 			      struct kthread_delayed_work *dwork,
1147 			      unsigned long delay)
1148 {
1149 	struct kthread_work *work = &dwork->work;
1150 	unsigned long flags;
1151 	int ret;
1152 
1153 	raw_spin_lock_irqsave(&worker->lock, flags);
1154 
1155 	/* Do not bother with canceling when never queued. */
1156 	if (!work->worker) {
1157 		ret = false;
1158 		goto fast_queue;
1159 	}
1160 
1161 	/* Work must not be used with >1 worker, see kthread_queue_work() */
1162 	WARN_ON_ONCE(work->worker != worker);
1163 
1164 	/*
1165 	 * Temporary cancel the work but do not fight with another command
1166 	 * that is canceling the work as well.
1167 	 *
1168 	 * It is a bit tricky because of possible races with another
1169 	 * mod_delayed_work() and cancel_delayed_work() callers.
1170 	 *
1171 	 * The timer must be canceled first because worker->lock is released
1172 	 * when doing so. But the work can be removed from the queue (list)
1173 	 * only when it can be queued again so that the return value can
1174 	 * be used for reference counting.
1175 	 */
1176 	kthread_cancel_delayed_work_timer(work, &flags);
1177 	if (work->canceling) {
1178 		/* The number of works in the queue does not change. */
1179 		ret = true;
1180 		goto out;
1181 	}
1182 	ret = __kthread_cancel_work(work);
1183 
1184 fast_queue:
1185 	__kthread_queue_delayed_work(worker, dwork, delay);
1186 out:
1187 	raw_spin_unlock_irqrestore(&worker->lock, flags);
1188 	return ret;
1189 }
1190 EXPORT_SYMBOL_GPL(kthread_mod_delayed_work);
1191 
__kthread_cancel_work_sync(struct kthread_work * work,bool is_dwork)1192 static bool __kthread_cancel_work_sync(struct kthread_work *work, bool is_dwork)
1193 {
1194 	struct kthread_worker *worker = work->worker;
1195 	unsigned long flags;
1196 	int ret = false;
1197 
1198 	if (!worker)
1199 		goto out;
1200 
1201 	raw_spin_lock_irqsave(&worker->lock, flags);
1202 	/* Work must not be used with >1 worker, see kthread_queue_work(). */
1203 	WARN_ON_ONCE(work->worker != worker);
1204 
1205 	if (is_dwork)
1206 		kthread_cancel_delayed_work_timer(work, &flags);
1207 
1208 	ret = __kthread_cancel_work(work);
1209 
1210 	if (worker->current_work != work)
1211 		goto out_fast;
1212 
1213 	/*
1214 	 * The work is in progress and we need to wait with the lock released.
1215 	 * In the meantime, block any queuing by setting the canceling counter.
1216 	 */
1217 	work->canceling++;
1218 	raw_spin_unlock_irqrestore(&worker->lock, flags);
1219 	kthread_flush_work(work);
1220 	raw_spin_lock_irqsave(&worker->lock, flags);
1221 	work->canceling--;
1222 
1223 out_fast:
1224 	raw_spin_unlock_irqrestore(&worker->lock, flags);
1225 out:
1226 	return ret;
1227 }
1228 
1229 /**
1230  * kthread_cancel_work_sync - cancel a kthread work and wait for it to finish
1231  * @work: the kthread work to cancel
1232  *
1233  * Cancel @work and wait for its execution to finish.  This function
1234  * can be used even if the work re-queues itself. On return from this
1235  * function, @work is guaranteed to be not pending or executing on any CPU.
1236  *
1237  * kthread_cancel_work_sync(&delayed_work->work) must not be used for
1238  * delayed_work's. Use kthread_cancel_delayed_work_sync() instead.
1239  *
1240  * The caller must ensure that the worker on which @work was last
1241  * queued can't be destroyed before this function returns.
1242  *
1243  * Return: %true if @work was pending, %false otherwise.
1244  */
kthread_cancel_work_sync(struct kthread_work * work)1245 bool kthread_cancel_work_sync(struct kthread_work *work)
1246 {
1247 	return __kthread_cancel_work_sync(work, false);
1248 }
1249 EXPORT_SYMBOL_GPL(kthread_cancel_work_sync);
1250 
1251 /**
1252  * kthread_cancel_delayed_work_sync - cancel a kthread delayed work and
1253  *	wait for it to finish.
1254  * @dwork: the kthread delayed work to cancel
1255  *
1256  * This is kthread_cancel_work_sync() for delayed works.
1257  *
1258  * Return: %true if @dwork was pending, %false otherwise.
1259  */
kthread_cancel_delayed_work_sync(struct kthread_delayed_work * dwork)1260 bool kthread_cancel_delayed_work_sync(struct kthread_delayed_work *dwork)
1261 {
1262 	return __kthread_cancel_work_sync(&dwork->work, true);
1263 }
1264 EXPORT_SYMBOL_GPL(kthread_cancel_delayed_work_sync);
1265 
1266 /**
1267  * kthread_flush_worker - flush all current works on a kthread_worker
1268  * @worker: worker to flush
1269  *
1270  * Wait until all currently executing or pending works on @worker are
1271  * finished.
1272  */
kthread_flush_worker(struct kthread_worker * worker)1273 void kthread_flush_worker(struct kthread_worker *worker)
1274 {
1275 	struct kthread_flush_work fwork = {
1276 		KTHREAD_WORK_INIT(fwork.work, kthread_flush_work_fn),
1277 		COMPLETION_INITIALIZER_ONSTACK(fwork.done),
1278 	};
1279 
1280 	kthread_queue_work(worker, &fwork.work);
1281 	wait_for_completion(&fwork.done);
1282 }
1283 EXPORT_SYMBOL_GPL(kthread_flush_worker);
1284 
1285 /**
1286  * kthread_destroy_worker - destroy a kthread worker
1287  * @worker: worker to be destroyed
1288  *
1289  * Flush and destroy @worker.  The simple flush is enough because the kthread
1290  * worker API is used only in trivial scenarios.  There are no multi-step state
1291  * machines needed.
1292  */
kthread_destroy_worker(struct kthread_worker * worker)1293 void kthread_destroy_worker(struct kthread_worker *worker)
1294 {
1295 	struct task_struct *task;
1296 
1297 	task = worker->task;
1298 	if (WARN_ON(!task))
1299 		return;
1300 
1301 	kthread_flush_worker(worker);
1302 	kthread_stop(task);
1303 	WARN_ON(!list_empty(&worker->work_list));
1304 	kfree(worker);
1305 }
1306 EXPORT_SYMBOL(kthread_destroy_worker);
1307 
1308 /**
1309  * kthread_use_mm - make the calling kthread operate on an address space
1310  * @mm: address space to operate on
1311  */
kthread_use_mm(struct mm_struct * mm)1312 void kthread_use_mm(struct mm_struct *mm)
1313 {
1314 	struct mm_struct *active_mm;
1315 	struct task_struct *tsk = current;
1316 
1317 	WARN_ON_ONCE(!(tsk->flags & PF_KTHREAD));
1318 	WARN_ON_ONCE(tsk->mm);
1319 
1320 	task_lock(tsk);
1321 	/* Hold off tlb flush IPIs while switching mm's */
1322 	local_irq_disable();
1323 	active_mm = tsk->active_mm;
1324 	if (active_mm != mm) {
1325 		mmgrab(mm);
1326 		tsk->active_mm = mm;
1327 	}
1328 	tsk->mm = mm;
1329 	switch_mm_irqs_off(active_mm, mm, tsk);
1330 	local_irq_enable();
1331 	task_unlock(tsk);
1332 #ifdef finish_arch_post_lock_switch
1333 	finish_arch_post_lock_switch();
1334 #endif
1335 
1336 	if (active_mm != mm)
1337 		mmdrop(active_mm);
1338 
1339 	to_kthread(tsk)->oldfs = force_uaccess_begin();
1340 }
1341 EXPORT_SYMBOL_GPL(kthread_use_mm);
1342 
1343 /**
1344  * kthread_unuse_mm - reverse the effect of kthread_use_mm()
1345  * @mm: address space to operate on
1346  */
kthread_unuse_mm(struct mm_struct * mm)1347 void kthread_unuse_mm(struct mm_struct *mm)
1348 {
1349 	struct task_struct *tsk = current;
1350 
1351 	WARN_ON_ONCE(!(tsk->flags & PF_KTHREAD));
1352 	WARN_ON_ONCE(!tsk->mm);
1353 
1354 	force_uaccess_end(to_kthread(tsk)->oldfs);
1355 
1356 	task_lock(tsk);
1357 	sync_mm_rss(mm);
1358 	local_irq_disable();
1359 	tsk->mm = NULL;
1360 	/* active_mm is still 'mm' */
1361 	enter_lazy_tlb(mm, tsk);
1362 	local_irq_enable();
1363 	task_unlock(tsk);
1364 }
1365 EXPORT_SYMBOL_GPL(kthread_unuse_mm);
1366 
1367 #ifdef CONFIG_BLK_CGROUP
1368 /**
1369  * kthread_associate_blkcg - associate blkcg to current kthread
1370  * @css: the cgroup info
1371  *
1372  * Current thread must be a kthread. The thread is running jobs on behalf of
1373  * other threads. In some cases, we expect the jobs attach cgroup info of
1374  * original threads instead of that of current thread. This function stores
1375  * original thread's cgroup info in current kthread context for later
1376  * retrieval.
1377  */
kthread_associate_blkcg(struct cgroup_subsys_state * css)1378 void kthread_associate_blkcg(struct cgroup_subsys_state *css)
1379 {
1380 	struct kthread *kthread;
1381 
1382 	if (!(current->flags & PF_KTHREAD))
1383 		return;
1384 	kthread = to_kthread(current);
1385 	if (!kthread)
1386 		return;
1387 
1388 	if (kthread->blkcg_css) {
1389 		css_put(kthread->blkcg_css);
1390 		kthread->blkcg_css = NULL;
1391 	}
1392 	if (css) {
1393 		css_get(css);
1394 		kthread->blkcg_css = css;
1395 	}
1396 }
1397 EXPORT_SYMBOL(kthread_associate_blkcg);
1398 
1399 /**
1400  * kthread_blkcg - get associated blkcg css of current kthread
1401  *
1402  * Current thread must be a kthread.
1403  */
kthread_blkcg(void)1404 struct cgroup_subsys_state *kthread_blkcg(void)
1405 {
1406 	struct kthread *kthread;
1407 
1408 	if (current->flags & PF_KTHREAD) {
1409 		kthread = to_kthread(current);
1410 		if (kthread)
1411 			return kthread->blkcg_css;
1412 	}
1413 	return NULL;
1414 }
1415 EXPORT_SYMBOL(kthread_blkcg);
1416 #endif
1417