• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Common SMP CPU bringup/teardown functions
3  */
4 #include <linux/cpu.h>
5 #include <linux/err.h>
6 #include <linux/smp.h>
7 #include <linux/delay.h>
8 #include <linux/init.h>
9 #include <linux/list.h>
10 #include <linux/slab.h>
11 #include <linux/sched.h>
12 #include <linux/export.h>
13 #include <linux/percpu.h>
14 #include <linux/kthread.h>
15 #include <linux/smpboot.h>
16 
17 #include "smpboot.h"
18 
19 #ifdef CONFIG_SMP
20 
21 #ifdef CONFIG_GENERIC_SMP_IDLE_THREAD
22 /*
23  * For the hotplug case we keep the task structs around and reuse
24  * them.
25  */
26 static DEFINE_PER_CPU(struct task_struct *, idle_threads);
27 
idle_thread_get(unsigned int cpu)28 struct task_struct *idle_thread_get(unsigned int cpu)
29 {
30 	struct task_struct *tsk = per_cpu(idle_threads, cpu);
31 
32 	if (!tsk)
33 		return ERR_PTR(-ENOMEM);
34 	init_idle(tsk, cpu);
35 	return tsk;
36 }
37 
idle_thread_set_boot_cpu(void)38 void __init idle_thread_set_boot_cpu(void)
39 {
40 	per_cpu(idle_threads, smp_processor_id()) = current;
41 }
42 
43 /**
44  * idle_init - Initialize the idle thread for a cpu
45  * @cpu:	The cpu for which the idle thread should be initialized
46  *
47  * Creates the thread if it does not exist.
48  */
idle_init(unsigned int cpu)49 static inline void idle_init(unsigned int cpu)
50 {
51 	struct task_struct *tsk = per_cpu(idle_threads, cpu);
52 
53 	if (!tsk) {
54 		tsk = fork_idle(cpu);
55 		if (IS_ERR(tsk))
56 			pr_err("SMP: fork_idle() failed for CPU %u\n", cpu);
57 		else
58 			per_cpu(idle_threads, cpu) = tsk;
59 	}
60 }
61 
62 /**
63  * idle_threads_init - Initialize idle threads for all cpus
64  */
idle_threads_init(void)65 void __init idle_threads_init(void)
66 {
67 	unsigned int cpu, boot_cpu;
68 
69 	boot_cpu = smp_processor_id();
70 
71 	for_each_possible_cpu(cpu) {
72 		if (cpu != boot_cpu)
73 			idle_init(cpu);
74 	}
75 }
76 #endif
77 
78 #endif /* #ifdef CONFIG_SMP */
79 
80 static LIST_HEAD(hotplug_threads);
81 static DEFINE_MUTEX(smpboot_threads_lock);
82 
83 struct smpboot_thread_data {
84 	unsigned int			cpu;
85 	unsigned int			status;
86 	struct smp_hotplug_thread	*ht;
87 };
88 
89 enum {
90 	HP_THREAD_NONE = 0,
91 	HP_THREAD_ACTIVE,
92 	HP_THREAD_PARKED,
93 };
94 
95 /**
96  * smpboot_thread_fn - percpu hotplug thread loop function
97  * @data:	thread data pointer
98  *
99  * Checks for thread stop and park conditions. Calls the necessary
100  * setup, cleanup, park and unpark functions for the registered
101  * thread.
102  *
103  * Returns 1 when the thread should exit, 0 otherwise.
104  */
smpboot_thread_fn(void * data)105 static int smpboot_thread_fn(void *data)
106 {
107 	struct smpboot_thread_data *td = data;
108 	struct smp_hotplug_thread *ht = td->ht;
109 
110 	while (1) {
111 		set_current_state(TASK_INTERRUPTIBLE);
112 		preempt_disable();
113 		if (kthread_should_stop()) {
114 			__set_current_state(TASK_RUNNING);
115 			preempt_enable();
116 			/* cleanup must mirror setup */
117 			if (ht->cleanup && td->status != HP_THREAD_NONE)
118 				ht->cleanup(td->cpu, cpu_online(td->cpu));
119 			kfree(td);
120 			return 0;
121 		}
122 
123 		if (kthread_should_park()) {
124 			__set_current_state(TASK_RUNNING);
125 			preempt_enable();
126 			if (ht->park && td->status == HP_THREAD_ACTIVE) {
127 				BUG_ON(td->cpu != smp_processor_id());
128 				ht->park(td->cpu);
129 				td->status = HP_THREAD_PARKED;
130 			}
131 			kthread_parkme();
132 			/* We might have been woken for stop */
133 			continue;
134 		}
135 
136 		BUG_ON(td->cpu != smp_processor_id());
137 
138 		/* Check for state change setup */
139 		switch (td->status) {
140 		case HP_THREAD_NONE:
141 			__set_current_state(TASK_RUNNING);
142 			preempt_enable();
143 			if (ht->setup)
144 				ht->setup(td->cpu);
145 			td->status = HP_THREAD_ACTIVE;
146 			continue;
147 
148 		case HP_THREAD_PARKED:
149 			__set_current_state(TASK_RUNNING);
150 			preempt_enable();
151 			if (ht->unpark)
152 				ht->unpark(td->cpu);
153 			td->status = HP_THREAD_ACTIVE;
154 			continue;
155 		}
156 
157 		if (!ht->thread_should_run(td->cpu)) {
158 			preempt_enable_no_resched();
159 			schedule();
160 		} else {
161 			__set_current_state(TASK_RUNNING);
162 			preempt_enable();
163 			ht->thread_fn(td->cpu);
164 		}
165 	}
166 }
167 
168 static int
__smpboot_create_thread(struct smp_hotplug_thread * ht,unsigned int cpu)169 __smpboot_create_thread(struct smp_hotplug_thread *ht, unsigned int cpu)
170 {
171 	struct task_struct *tsk = *per_cpu_ptr(ht->store, cpu);
172 	struct smpboot_thread_data *td;
173 
174 	if (tsk)
175 		return 0;
176 
177 	td = kzalloc_node(sizeof(*td), GFP_KERNEL, cpu_to_node(cpu));
178 	if (!td)
179 		return -ENOMEM;
180 	td->cpu = cpu;
181 	td->ht = ht;
182 
183 	tsk = kthread_create_on_cpu(smpboot_thread_fn, td, cpu,
184 				    ht->thread_comm);
185 	if (IS_ERR(tsk)) {
186 		kfree(td);
187 		return PTR_ERR(tsk);
188 	}
189 	get_task_struct(tsk);
190 	*per_cpu_ptr(ht->store, cpu) = tsk;
191 	if (ht->create) {
192 		/*
193 		 * Make sure that the task has actually scheduled out
194 		 * into park position, before calling the create
195 		 * callback. At least the migration thread callback
196 		 * requires that the task is off the runqueue.
197 		 */
198 		if (!wait_task_inactive(tsk, TASK_PARKED))
199 			WARN_ON(1);
200 		else
201 			ht->create(cpu);
202 	}
203 	return 0;
204 }
205 
smpboot_create_threads(unsigned int cpu)206 int smpboot_create_threads(unsigned int cpu)
207 {
208 	struct smp_hotplug_thread *cur;
209 	int ret = 0;
210 
211 	mutex_lock(&smpboot_threads_lock);
212 	list_for_each_entry(cur, &hotplug_threads, list) {
213 		ret = __smpboot_create_thread(cur, cpu);
214 		if (ret)
215 			break;
216 	}
217 	mutex_unlock(&smpboot_threads_lock);
218 	return ret;
219 }
220 
smpboot_unpark_thread(struct smp_hotplug_thread * ht,unsigned int cpu)221 static void smpboot_unpark_thread(struct smp_hotplug_thread *ht, unsigned int cpu)
222 {
223 	struct task_struct *tsk = *per_cpu_ptr(ht->store, cpu);
224 
225 	if (!ht->selfparking)
226 		kthread_unpark(tsk);
227 }
228 
smpboot_unpark_threads(unsigned int cpu)229 void smpboot_unpark_threads(unsigned int cpu)
230 {
231 	struct smp_hotplug_thread *cur;
232 
233 	mutex_lock(&smpboot_threads_lock);
234 	list_for_each_entry(cur, &hotplug_threads, list)
235 		if (cpumask_test_cpu(cpu, cur->cpumask))
236 			smpboot_unpark_thread(cur, cpu);
237 	mutex_unlock(&smpboot_threads_lock);
238 }
239 
smpboot_park_thread(struct smp_hotplug_thread * ht,unsigned int cpu)240 static void smpboot_park_thread(struct smp_hotplug_thread *ht, unsigned int cpu)
241 {
242 	struct task_struct *tsk = *per_cpu_ptr(ht->store, cpu);
243 
244 	if (tsk && !ht->selfparking)
245 		kthread_park(tsk);
246 }
247 
smpboot_park_threads(unsigned int cpu)248 void smpboot_park_threads(unsigned int cpu)
249 {
250 	struct smp_hotplug_thread *cur;
251 
252 	mutex_lock(&smpboot_threads_lock);
253 	list_for_each_entry_reverse(cur, &hotplug_threads, list)
254 		smpboot_park_thread(cur, cpu);
255 	mutex_unlock(&smpboot_threads_lock);
256 }
257 
smpboot_destroy_threads(struct smp_hotplug_thread * ht)258 static void smpboot_destroy_threads(struct smp_hotplug_thread *ht)
259 {
260 	unsigned int cpu;
261 
262 	/* We need to destroy also the parked threads of offline cpus */
263 	for_each_possible_cpu(cpu) {
264 		struct task_struct *tsk = *per_cpu_ptr(ht->store, cpu);
265 
266 		if (tsk) {
267 			kthread_stop(tsk);
268 			put_task_struct(tsk);
269 			*per_cpu_ptr(ht->store, cpu) = NULL;
270 		}
271 	}
272 }
273 
274 /**
275  * smpboot_register_percpu_thread_cpumask - Register a per_cpu thread related
276  * 					    to hotplug
277  * @plug_thread:	Hotplug thread descriptor
278  * @cpumask:		The cpumask where threads run
279  *
280  * Creates and starts the threads on all online cpus.
281  */
smpboot_register_percpu_thread_cpumask(struct smp_hotplug_thread * plug_thread,const struct cpumask * cpumask)282 int smpboot_register_percpu_thread_cpumask(struct smp_hotplug_thread *plug_thread,
283 					   const struct cpumask *cpumask)
284 {
285 	unsigned int cpu;
286 	int ret = 0;
287 
288 	if (!alloc_cpumask_var(&plug_thread->cpumask, GFP_KERNEL))
289 		return -ENOMEM;
290 	cpumask_copy(plug_thread->cpumask, cpumask);
291 
292 	get_online_cpus();
293 	mutex_lock(&smpboot_threads_lock);
294 	for_each_online_cpu(cpu) {
295 		ret = __smpboot_create_thread(plug_thread, cpu);
296 		if (ret) {
297 			smpboot_destroy_threads(plug_thread);
298 			free_cpumask_var(plug_thread->cpumask);
299 			goto out;
300 		}
301 		if (cpumask_test_cpu(cpu, cpumask))
302 			smpboot_unpark_thread(plug_thread, cpu);
303 	}
304 	list_add(&plug_thread->list, &hotplug_threads);
305 out:
306 	mutex_unlock(&smpboot_threads_lock);
307 	put_online_cpus();
308 	return ret;
309 }
310 EXPORT_SYMBOL_GPL(smpboot_register_percpu_thread_cpumask);
311 
312 /**
313  * smpboot_unregister_percpu_thread - Unregister a per_cpu thread related to hotplug
314  * @plug_thread:	Hotplug thread descriptor
315  *
316  * Stops all threads on all possible cpus.
317  */
smpboot_unregister_percpu_thread(struct smp_hotplug_thread * plug_thread)318 void smpboot_unregister_percpu_thread(struct smp_hotplug_thread *plug_thread)
319 {
320 	get_online_cpus();
321 	mutex_lock(&smpboot_threads_lock);
322 	list_del(&plug_thread->list);
323 	smpboot_destroy_threads(plug_thread);
324 	mutex_unlock(&smpboot_threads_lock);
325 	put_online_cpus();
326 	free_cpumask_var(plug_thread->cpumask);
327 }
328 EXPORT_SYMBOL_GPL(smpboot_unregister_percpu_thread);
329 
330 /**
331  * smpboot_update_cpumask_percpu_thread - Adjust which per_cpu hotplug threads stay parked
332  * @plug_thread:	Hotplug thread descriptor
333  * @new:		Revised mask to use
334  *
335  * The cpumask field in the smp_hotplug_thread must not be updated directly
336  * by the client, but only by calling this function.
337  * This function can only be called on a registered smp_hotplug_thread.
338  */
smpboot_update_cpumask_percpu_thread(struct smp_hotplug_thread * plug_thread,const struct cpumask * new)339 int smpboot_update_cpumask_percpu_thread(struct smp_hotplug_thread *plug_thread,
340 					 const struct cpumask *new)
341 {
342 	struct cpumask *old = plug_thread->cpumask;
343 	cpumask_var_t tmp;
344 	unsigned int cpu;
345 
346 	if (!alloc_cpumask_var(&tmp, GFP_KERNEL))
347 		return -ENOMEM;
348 
349 	get_online_cpus();
350 	mutex_lock(&smpboot_threads_lock);
351 
352 	/* Park threads that were exclusively enabled on the old mask. */
353 	cpumask_andnot(tmp, old, new);
354 	for_each_cpu_and(cpu, tmp, cpu_online_mask)
355 		smpboot_park_thread(plug_thread, cpu);
356 
357 	/* Unpark threads that are exclusively enabled on the new mask. */
358 	cpumask_andnot(tmp, new, old);
359 	for_each_cpu_and(cpu, tmp, cpu_online_mask)
360 		smpboot_unpark_thread(plug_thread, cpu);
361 
362 	cpumask_copy(old, new);
363 
364 	mutex_unlock(&smpboot_threads_lock);
365 	put_online_cpus();
366 
367 	free_cpumask_var(tmp);
368 
369 	return 0;
370 }
371 EXPORT_SYMBOL_GPL(smpboot_update_cpumask_percpu_thread);
372 
373 static DEFINE_PER_CPU(atomic_t, cpu_hotplug_state) = ATOMIC_INIT(CPU_POST_DEAD);
374 
375 /*
376  * Called to poll specified CPU's state, for example, when waiting for
377  * a CPU to come online.
378  */
cpu_report_state(int cpu)379 int cpu_report_state(int cpu)
380 {
381 	return atomic_read(&per_cpu(cpu_hotplug_state, cpu));
382 }
383 
384 /*
385  * If CPU has died properly, set its state to CPU_UP_PREPARE and
386  * return success.  Otherwise, return -EBUSY if the CPU died after
387  * cpu_wait_death() timed out.  And yet otherwise again, return -EAGAIN
388  * if cpu_wait_death() timed out and the CPU still hasn't gotten around
389  * to dying.  In the latter two cases, the CPU might not be set up
390  * properly, but it is up to the arch-specific code to decide.
391  * Finally, -EIO indicates an unanticipated problem.
392  *
393  * Note that it is permissible to omit this call entirely, as is
394  * done in architectures that do no CPU-hotplug error checking.
395  */
cpu_check_up_prepare(int cpu)396 int cpu_check_up_prepare(int cpu)
397 {
398 	if (!IS_ENABLED(CONFIG_HOTPLUG_CPU)) {
399 		atomic_set(&per_cpu(cpu_hotplug_state, cpu), CPU_UP_PREPARE);
400 		return 0;
401 	}
402 
403 	switch (atomic_read(&per_cpu(cpu_hotplug_state, cpu))) {
404 
405 	case CPU_POST_DEAD:
406 
407 		/* The CPU died properly, so just start it up again. */
408 		atomic_set(&per_cpu(cpu_hotplug_state, cpu), CPU_UP_PREPARE);
409 		return 0;
410 
411 	case CPU_DEAD_FROZEN:
412 
413 		/*
414 		 * Timeout during CPU death, so let caller know.
415 		 * The outgoing CPU completed its processing, but after
416 		 * cpu_wait_death() timed out and reported the error. The
417 		 * caller is free to proceed, in which case the state
418 		 * will be reset properly by cpu_set_state_online().
419 		 * Proceeding despite this -EBUSY return makes sense
420 		 * for systems where the outgoing CPUs take themselves
421 		 * offline, with no post-death manipulation required from
422 		 * a surviving CPU.
423 		 */
424 		return -EBUSY;
425 
426 	case CPU_BROKEN:
427 
428 		/*
429 		 * The most likely reason we got here is that there was
430 		 * a timeout during CPU death, and the outgoing CPU never
431 		 * did complete its processing.  This could happen on
432 		 * a virtualized system if the outgoing VCPU gets preempted
433 		 * for more than five seconds, and the user attempts to
434 		 * immediately online that same CPU.  Trying again later
435 		 * might return -EBUSY above, hence -EAGAIN.
436 		 */
437 		return -EAGAIN;
438 
439 	default:
440 
441 		/* Should not happen.  Famous last words. */
442 		return -EIO;
443 	}
444 }
445 
446 /*
447  * Mark the specified CPU online.
448  *
449  * Note that it is permissible to omit this call entirely, as is
450  * done in architectures that do no CPU-hotplug error checking.
451  */
cpu_set_state_online(int cpu)452 void cpu_set_state_online(int cpu)
453 {
454 	(void)atomic_xchg(&per_cpu(cpu_hotplug_state, cpu), CPU_ONLINE);
455 }
456 
457 #ifdef CONFIG_HOTPLUG_CPU
458 
459 /*
460  * Wait for the specified CPU to exit the idle loop and die.
461  */
cpu_wait_death(unsigned int cpu,int seconds)462 bool cpu_wait_death(unsigned int cpu, int seconds)
463 {
464 	int jf_left = seconds * HZ;
465 	int oldstate;
466 	bool ret = true;
467 	int sleep_jf = 1;
468 
469 	might_sleep();
470 
471 	/* The outgoing CPU will normally get done quite quickly. */
472 	if (atomic_read(&per_cpu(cpu_hotplug_state, cpu)) == CPU_DEAD)
473 		goto update_state;
474 	udelay(5);
475 
476 	/* But if the outgoing CPU dawdles, wait increasingly long times. */
477 	while (atomic_read(&per_cpu(cpu_hotplug_state, cpu)) != CPU_DEAD) {
478 		schedule_timeout_uninterruptible(sleep_jf);
479 		jf_left -= sleep_jf;
480 		if (jf_left <= 0)
481 			break;
482 		sleep_jf = DIV_ROUND_UP(sleep_jf * 11, 10);
483 	}
484 update_state:
485 	oldstate = atomic_read(&per_cpu(cpu_hotplug_state, cpu));
486 	if (oldstate == CPU_DEAD) {
487 		/* Outgoing CPU died normally, update state. */
488 		smp_mb(); /* atomic_read() before update. */
489 		atomic_set(&per_cpu(cpu_hotplug_state, cpu), CPU_POST_DEAD);
490 	} else {
491 		/* Outgoing CPU still hasn't died, set state accordingly. */
492 		if (atomic_cmpxchg(&per_cpu(cpu_hotplug_state, cpu),
493 				   oldstate, CPU_BROKEN) != oldstate)
494 			goto update_state;
495 		ret = false;
496 	}
497 	return ret;
498 }
499 
500 /*
501  * Called by the outgoing CPU to report its successful death.  Return
502  * false if this report follows the surviving CPU's timing out.
503  *
504  * A separate "CPU_DEAD_FROZEN" is used when the surviving CPU
505  * timed out.  This approach allows architectures to omit calls to
506  * cpu_check_up_prepare() and cpu_set_state_online() without defeating
507  * the next cpu_wait_death()'s polling loop.
508  */
cpu_report_death(void)509 bool cpu_report_death(void)
510 {
511 	int oldstate;
512 	int newstate;
513 	int cpu = smp_processor_id();
514 
515 	do {
516 		oldstate = atomic_read(&per_cpu(cpu_hotplug_state, cpu));
517 		if (oldstate != CPU_BROKEN)
518 			newstate = CPU_DEAD;
519 		else
520 			newstate = CPU_DEAD_FROZEN;
521 	} while (atomic_cmpxchg(&per_cpu(cpu_hotplug_state, cpu),
522 				oldstate, newstate) != oldstate);
523 	return newstate == CPU_DEAD;
524 }
525 
526 #endif /* #ifdef CONFIG_HOTPLUG_CPU */
527