1 // SPDX-License-Identifier: GPL-2.0-or-later
3  * coupled.c - helper functions to enter the same idle state on multiple cpus
27  * will corrupt the gic state unless the other cpu runs a work
28  * around).  Each cpu has a power state that it can enter without
32  * sometimes the whole SoC).  Entering a coupled power state must
36  * WFI state until all cpus are ready to enter a coupled state, at
37  * which point the coupled state function will be called on all
40  * Once all cpus are ready to enter idle, they are woken by an smp
42  * cpus will find work to do, and choose not to enter idle.  A
44  * power state enter function at the same time.  During this pass,
47  * cpu exits idle, the other cpus will decrement their counter and
50  * requested_state stores the deepest coupled idle state each cpu
56  * and only read after all the cpus are ready for the coupled idle
57  * state are are no longer updating it.
62  * the waiting loop, in the ready loop, or in the coupled idle state.
64  * or in the coupled idle state.
73  *    Set struct cpuidle_device.safe_state to a state that is not a
74  *    coupled state.  This is usually WFI.
77  *    state that affects multiple cpus.
79  *    Provide a struct cpuidle_state.enter function for each state
88  * struct cpuidle_coupled - data for set of cpus that share a coupled idle state
95  * @prevent: flag to prevent coupled idle while a cpu is hotplugging
109 #define WAITING_MASK (MAX_WAITING_CPUS - 1)
112 #define CPUIDLE_COUPLED_NOT_IDLE	(-1)
132  * cpuidle_coupled_parallel_barrier - synchronize all online coupled cpus
144  * Must only be called from within a coupled idle state handler
145  * (state.enter when state.flags has CPUIDLE_FLAG_COUPLED set).
151 	int n = dev->coupled->online_count;  in cpuidle_coupled_parallel_barrier()
169  * cpuidle_state_is_coupled - check if a state is part of a coupled set
171  * @state: index of the target state in drv->states
173  * Returns true if the target state is coupled with cpus besides this one
175 bool cpuidle_state_is_coupled(struct cpuidle_driver *drv, int state)  in cpuidle_state_is_coupled()  argument
177 	return drv->states[state].flags & CPUIDLE_FLAG_COUPLED;  in cpuidle_state_is_coupled()
181  * cpuidle_coupled_state_verify - check if the coupled states are correctly set.
184  * Returns 0 for valid state values, a negative error code otherwise:
185  *  * -EINVAL if any coupled state(safe_state_index) is wrongly set.
191 	for (i = drv->state_count - 1; i >= 0; i--) {  in cpuidle_coupled_state_verify()
193 		    (drv->safe_state_index == i ||  in cpuidle_coupled_state_verify()
194 		     drv->safe_state_index < 0 ||  in cpuidle_coupled_state_verify()
195 		     drv->safe_state_index >= drv->state_count))  in cpuidle_coupled_state_verify()
196 			return -EINVAL;  in cpuidle_coupled_state_verify()
203  * cpuidle_coupled_set_ready - mark a cpu as ready
208 	atomic_add(MAX_WAITING_CPUS, &coupled->ready_waiting_counts);  in cpuidle_coupled_set_ready()
212  * cpuidle_coupled_set_not_ready - mark a cpu as not ready
217  * decrements the waiting counter and then re-increments it just before another
219  * down from the number of online cpus without going through the coupled idle
220  * state.
222  * Returns 0 if the counter was decremented successfully, -EINVAL if the ready
231 	all = coupled->online_count | (coupled->online_count << WAITING_BITS);  in cpuidle_coupled_set_not_ready()
232 	ret = atomic_add_unless(&coupled->ready_waiting_counts,  in cpuidle_coupled_set_not_ready()
233 		-MAX_WAITING_CPUS, all);  in cpuidle_coupled_set_not_ready()
235 	return ret ? 0 : -EINVAL;  in cpuidle_coupled_set_not_ready()
239  * cpuidle_coupled_no_cpus_ready - check if no cpus in a coupled set are ready
246 	int r = atomic_read(&coupled->ready_waiting_counts) >> WAITING_BITS;  in cpuidle_coupled_no_cpus_ready()
251  * cpuidle_coupled_cpus_ready - check if all cpus in a coupled set are ready
254  * Returns true if all cpus coupled to this target state are in the ready loop
258 	int r = atomic_read(&coupled->ready_waiting_counts) >> WAITING_BITS;  in cpuidle_coupled_cpus_ready()
259 	return r == coupled->online_count;  in cpuidle_coupled_cpus_ready()
263  * cpuidle_coupled_cpus_waiting - check if all cpus in a coupled set are waiting
266  * Returns true if all cpus coupled to this target state are in the wait loop
270 	int w = atomic_read(&coupled->ready_waiting_counts) & WAITING_MASK;  in cpuidle_coupled_cpus_waiting()
271 	return w == coupled->online_count;  in cpuidle_coupled_cpus_waiting()
275  * cpuidle_coupled_no_cpus_waiting - check if no cpus in coupled set are waiting
282 	int w = atomic_read(&coupled->ready_waiting_counts) & WAITING_MASK;  in cpuidle_coupled_no_cpus_waiting()
287  * cpuidle_coupled_get_state - determine the deepest idle state
291  * Returns the deepest idle state that all coupled cpus can enter
297 	int state = INT_MAX;  in cpuidle_coupled_get_state()  local
306 	for_each_cpu(i, &coupled->coupled_cpus)  in cpuidle_coupled_get_state()
307 		if (cpu_online(i) && coupled->requested_state[i] < state)  in cpuidle_coupled_get_state()
308 			state = coupled->requested_state[i];  in cpuidle_coupled_get_state()
310 	return state;  in cpuidle_coupled_get_state()
321  * cpuidle_coupled_poke - wake up a cpu that may be waiting
324  * Ensures that the target cpu exits it's waiting idle state (if it is in it)
325  * and will see updates to waiting_count before it re-enters it's waiting idle
326  * state.
329  * either has or will soon have a pending IPI that will wake it out of idle,
330  * or it is currently processing the IPI and is not in idle.
341  * cpuidle_coupled_poke_others - wake up all other cpus that may be waiting
352 	for_each_cpu(cpu, &coupled->coupled_cpus)  in cpuidle_coupled_poke_others()
358  * cpuidle_coupled_set_waiting - mark this cpu as in the wait loop
361  * @next_state: the index in drv->states of the requested state for this cpu
363  * Updates the requested idle state for the specified cpuidle device.
369 	coupled->requested_state[cpu] = next_state;  in cpuidle_coupled_set_waiting()
375 	return atomic_inc_return(&coupled->ready_waiting_counts) & WAITING_MASK;  in cpuidle_coupled_set_waiting()
379  * cpuidle_coupled_set_not_waiting - mark this cpu as leaving the wait loop
383  * Removes the requested idle state for the specified cpuidle device.
394 	atomic_dec(&coupled->ready_waiting_counts);  in cpuidle_coupled_set_not_waiting()
396 	coupled->requested_state[cpu] = CPUIDLE_COUPLED_NOT_IDLE;  in cpuidle_coupled_set_not_waiting()
400  * cpuidle_coupled_set_done - mark this cpu as leaving the ready loop
406  * this cpu as waiting just before it exits idle.
411 	atomic_sub(MAX_WAITING_CPUS, &coupled->ready_waiting_counts);  in cpuidle_coupled_set_done()
415  * cpuidle_coupled_clear_pokes - spin until the poke interrupt is processed
423  * the interrupt didn't schedule work that should take the cpu out of idle.
445 	cpumask_and(&cpus, cpu_online_mask, &coupled->coupled_cpus);  in cpuidle_coupled_any_pokes_pending()
452  * cpuidle_enter_state_coupled - attempt to enter a state with coupled cpus
455  * @next_state: index of the requested state in drv->states
457  * Coordinate with coupled cpus to enter the target state.  This is a two
461  * go to an intermediate state (the cpuidle_device's safe state), and wait for
462  * all the other cpus to call this function.  Once all coupled cpus are idle,
467  * interrupts while preparing for idle, and it will always return with
473 	int entered_state = -1;  in cpuidle_enter_state_coupled()
474 	struct cpuidle_coupled *coupled = dev->coupled;  in cpuidle_enter_state_coupled()
478 		return -EINVAL;  in cpuidle_enter_state_coupled()
480 	while (coupled->prevent) {  in cpuidle_enter_state_coupled()
481 		cpuidle_coupled_clear_pokes(dev->cpu);  in cpuidle_enter_state_coupled()
487 			drv->safe_state_index);  in cpuidle_enter_state_coupled()
495 	cpumask_clear_cpu(dev->cpu, &cpuidle_coupled_poked);  in cpuidle_enter_state_coupled()
497 	w = cpuidle_coupled_set_waiting(dev->cpu, coupled, next_state);  in cpuidle_enter_state_coupled()
499 	 * If this is the last cpu to enter the waiting state, poke  in cpuidle_enter_state_coupled()
500 	 * all the other cpus out of their waiting state so they can  in cpuidle_enter_state_coupled()
501 	 * enter a deeper state.  This can race with one of the cpus  in cpuidle_enter_state_coupled()
502 	 * exiting the waiting state due to an interrupt and  in cpuidle_enter_state_coupled()
505 	if (w == coupled->online_count) {  in cpuidle_enter_state_coupled()
506 		cpumask_set_cpu(dev->cpu, &cpuidle_coupled_poked);  in cpuidle_enter_state_coupled()
507 		cpuidle_coupled_poke_others(dev->cpu, coupled);  in cpuidle_enter_state_coupled()
512 	 * Wait for all coupled cpus to be idle, using the deepest state  in cpuidle_enter_state_coupled()
520 			!cpumask_test_cpu(dev->cpu, &cpuidle_coupled_poked)) {  in cpuidle_enter_state_coupled()
521 		if (cpuidle_coupled_clear_pokes(dev->cpu))  in cpuidle_enter_state_coupled()
525 			cpuidle_coupled_set_not_waiting(dev->cpu, coupled);  in cpuidle_enter_state_coupled()
529 		if (coupled->prevent) {  in cpuidle_enter_state_coupled()
530 			cpuidle_coupled_set_not_waiting(dev->cpu, coupled);  in cpuidle_enter_state_coupled()
535 			drv->safe_state_index);  in cpuidle_enter_state_coupled()
539 	cpuidle_coupled_clear_pokes(dev->cpu);  in cpuidle_enter_state_coupled()
541 		cpuidle_coupled_set_not_waiting(dev->cpu, coupled);  in cpuidle_enter_state_coupled()
552 	 * All coupled cpus are probably idle.  There is a small chance that  in cpuidle_enter_state_coupled()
555 	 * cpu has incremented the ready counter, it cannot abort idle and must  in cpuidle_enter_state_coupled()
557 	 * another cpu leaves idle and decrements the waiting counter.  in cpuidle_enter_state_coupled()
562 		/* Check if any other cpus bailed out of idle. */  in cpuidle_enter_state_coupled()
576 	 * There is a small chance that a cpu left and reentered idle after this  in cpuidle_enter_state_coupled()
577 	 * cpu saw that all cpus were waiting.  The cpu that reentered idle will  in cpuidle_enter_state_coupled()
580 	 * controller when entering the deep idle state.  It's not possible to  in cpuidle_enter_state_coupled()
583 	 * coupled idle state of all cpus and retry.  in cpuidle_enter_state_coupled()
586 		cpuidle_coupled_set_done(dev->cpu, coupled);  in cpuidle_enter_state_coupled()
588 		cpuidle_coupled_parallel_barrier(dev, &coupled->abort_barrier);  in cpuidle_enter_state_coupled()
592 	/* all cpus have acked the coupled state */  in cpuidle_enter_state_coupled()
597 	cpuidle_coupled_set_done(dev->cpu, coupled);  in cpuidle_enter_state_coupled()
603 	 * that brings it out of idle will process that interrupt before  in cpuidle_enter_state_coupled()
604 	 * exiting the idle enter function and decrementing ready_count.  All  in cpuidle_enter_state_coupled()
607 	 * all other cpus will loop back into the safe idle state instead of  in cpuidle_enter_state_coupled()
617 	 * Wait until all coupled cpus have exited idle.  There is no risk that  in cpuidle_enter_state_coupled()
618 	 * a cpu exits and re-enters the ready state because this cpu has  in cpuidle_enter_state_coupled()
630 	cpumask_and(&cpus, cpu_online_mask, &coupled->coupled_cpus);  in cpuidle_coupled_update_online_cpus()
631 	coupled->online_count = cpumask_weight(&cpus);  in cpuidle_coupled_update_online_cpus()
635  * cpuidle_coupled_register_device - register a coupled cpuidle device
638  * Called from cpuidle_register_device to handle coupled idle init.  Finds the
649 	if (cpumask_empty(&dev->coupled_cpus))  in cpuidle_coupled_register_device()
652 	for_each_cpu(cpu, &dev->coupled_cpus) {  in cpuidle_coupled_register_device()
654 		if (other_dev && other_dev->coupled) {  in cpuidle_coupled_register_device()
655 			coupled = other_dev->coupled;  in cpuidle_coupled_register_device()
663 		return -ENOMEM;  in cpuidle_coupled_register_device()
665 	coupled->coupled_cpus = dev->coupled_cpus;  in cpuidle_coupled_register_device()
668 	dev->coupled = coupled;  in cpuidle_coupled_register_device()
669 	if (WARN_ON(!cpumask_equal(&dev->coupled_cpus, &coupled->coupled_cpus)))  in cpuidle_coupled_register_device()
670 		coupled->prevent++;  in cpuidle_coupled_register_device()
674 	coupled->refcnt++;  in cpuidle_coupled_register_device()
676 	csd = &per_cpu(cpuidle_coupled_poke_cb, dev->cpu);  in cpuidle_coupled_register_device()
677 	csd->func = cpuidle_coupled_handle_poke;  in cpuidle_coupled_register_device()
678 	csd->info = (void *)(unsigned long)dev->cpu;  in cpuidle_coupled_register_device()
684  * cpuidle_coupled_unregister_device - unregister a coupled cpuidle device
687  * Called from cpuidle_unregister_device to tear down coupled idle.  Removes the
688  * cpu from the coupled idle set, and frees the cpuidle_coupled_info struct if
693 	struct cpuidle_coupled *coupled = dev->coupled;  in cpuidle_coupled_unregister_device()
695 	if (cpumask_empty(&dev->coupled_cpus))  in cpuidle_coupled_unregister_device()
698 	if (--coupled->refcnt)  in cpuidle_coupled_unregister_device()
700 	dev->coupled = NULL;  in cpuidle_coupled_unregister_device()
704  * cpuidle_coupled_prevent_idle - prevent cpus from entering a coupled state
705  * @coupled: the struct coupled that contains the cpu that is changing state
708  * cpu_online_mask doesn't change while cpus are coordinating coupled idle.
715 	coupled->prevent++;  in cpuidle_coupled_prevent_idle()
723  * cpuidle_coupled_allow_idle - allows cpus to enter a coupled state
724  * @coupled: the struct coupled that contains the cpu that is changing state
727  * cpu_online_mask doesn't change while cpus are coordinating coupled idle.
738 	coupled->prevent--;  in cpuidle_coupled_allow_idle()
751 	if (dev && dev->coupled) {  in coupled_cpu_online()
752 		cpuidle_coupled_update_online_cpus(dev->coupled);  in coupled_cpu_online()
753 		cpuidle_coupled_allow_idle(dev->coupled);  in coupled_cpu_online()
767 	if (dev && dev->coupled)  in coupled_cpu_up_prepare()
768 		cpuidle_coupled_prevent_idle(dev->coupled);  in coupled_cpu_up_prepare()