2  * coupled.c - helper functions to enter the same idle state on multiple cpus
33  * cpus cannot be independently powered down, either due to
40  * shared between the cpus (L2 cache, interrupt controller, and
42  * be tightly controlled on both cpus.
45  * WFI state until all cpus are ready to enter a coupled state, at
47  * cpus at approximately the same time.
49  * Once all cpus are ready to enter idle, they are woken by an smp
51  * cpus will find work to do, and choose not to enter idle.  A
52  * final pass is needed to guarantee that all cpus will call the
55  * ready counter matches the number of online coupled cpus.  If any
56  * cpu exits idle, the other cpus will decrement their counter and
65  * and only read after all the cpus are ready for the coupled idle
69  * of cpus in the coupled set that are currently or soon will be
70  * online.  waiting_count tracks the number of cpus that are in
72  * ready_count tracks the number of cpus that are in the ready loop
78  *    coupled cpus, usually the same as cpu_possible_mask if all cpus
86  *    state that affects multiple cpus.
89  *    that affects multiple cpus.  This function is guaranteed to be
90  *    called on all cpus at approximately the same time.  The driver
91  *    should ensure that the cpus all abort together if any cpu tries
97  * struct cpuidle_coupled - data for set of cpus that share a coupled idle state
98  * @coupled_cpus: mask of cpus that are part of the coupled set
99  * @requested_state: array of requested states for cpus in the coupled set
100  * @ready_waiting_counts: combined count of cpus  in ready or waiting loops
101  * @online_count: count of cpus that are online
127  * in use.  This prevents a deadlock where two cpus are waiting for each others
140  * cpuidle_coupled_parallel_barrier - synchronize all online coupled cpus
145  * cpus in the same coupled group have called this function.  Once any caller
181  * Returns true if the target state is coupled with cpus besides this one
224  * is equal to the number of online cpus.  Prevents a race where one cpu
227  * down from the number of online cpus without going through the coupled idle
231  * counter was equal to the number of online cpus.
247  * cpuidle_coupled_no_cpus_ready - check if no cpus in a coupled set are ready
250  * Returns true if all of the cpus in a coupled set are out of the ready loop.
259  * cpuidle_coupled_cpus_ready - check if all cpus in a coupled set are ready
262  * Returns true if all cpus coupled to this target state are in the ready loop
271  * cpuidle_coupled_cpus_waiting - check if all cpus in a coupled set are waiting
274  * Returns true if all cpus coupled to this target state are in the wait loop
283  * cpuidle_coupled_no_cpus_waiting - check if no cpus in coupled set are waiting
286  * Returns true if all of the cpus in a coupled set are out of the waiting loop.
299  * Returns the deepest idle state that all coupled cpus can enter
349  * cpuidle_coupled_poke_others - wake up all other cpus that may be waiting
353  * Calls cpuidle_coupled_poke on all other online cpus.
372  * Returns the number of waiting cpus.
399 	 * cpus will increment ready_count and then spin until they  in cpuidle_coupled_set_not_waiting()
450 	cpumask_t cpus;  in cpuidle_coupled_any_pokes_pending()  local
453 	cpumask_and(&cpus, cpu_online_mask, &coupled->coupled_cpus);  in cpuidle_coupled_any_pokes_pending()
454 	ret = cpumask_and(&cpus, &cpuidle_coupled_poke_pending, &cpus);  in cpuidle_coupled_any_pokes_pending()
460  * cpuidle_enter_state_coupled - attempt to enter a state with coupled cpus
465  * Coordinate with coupled cpus to enter the target state.  This is a two
466  * stage process.  In the first stage, the cpus are operating independently,
468  * To save as much power as possible, the first cpus to call this function will
470  * all the other cpus to call this function.  Once all coupled cpus are idle,
471  * the second stage will start.  Each coupled cpu will spin until all cpus have
508 	 * all the other cpus out of their waiting state so they can  in cpuidle_enter_state_coupled()
509 	 * enter a deeper state.  This can race with one of the cpus  in cpuidle_enter_state_coupled()
520 	 * Wait for all coupled cpus to be idle, using the deepest state  in cpuidle_enter_state_coupled()
523 	 * two cpus could arrive at the waiting loop at the same time,  in cpuidle_enter_state_coupled()
560 	 * All coupled cpus are probably idle.  There is a small chance that  in cpuidle_enter_state_coupled()
561 	 * one of the other cpus just became active.  Increment the ready count,  in cpuidle_enter_state_coupled()
562 	 * and spin until all coupled cpus have incremented the counter. Once a  in cpuidle_enter_state_coupled()
564 	 * spin until either all cpus have incremented the ready counter, or  in cpuidle_enter_state_coupled()
570 		/* Check if any other cpus bailed out of idle. */  in cpuidle_enter_state_coupled()
579 	 * Make sure read of all cpus ready is done before reading pending pokes  in cpuidle_enter_state_coupled()
585 	 * cpu saw that all cpus were waiting.  The cpu that reentered idle will  in cpuidle_enter_state_coupled()
591 	 * coupled idle state of all cpus and retry.  in cpuidle_enter_state_coupled()
595 		/* Wait for all cpus to see the pending pokes */  in cpuidle_enter_state_coupled()
600 	/* all cpus have acked the coupled state */  in cpuidle_enter_state_coupled()
613 	 * other cpus will need to spin waiting for the cpu that is processing  in cpuidle_enter_state_coupled()
615 	 * all other cpus will loop back into the safe idle state instead of  in cpuidle_enter_state_coupled()
625 	 * Wait until all coupled cpus have exited idle.  There is no risk that  in cpuidle_enter_state_coupled()
637 	cpumask_t cpus;  in cpuidle_coupled_update_online_cpus()  local
638 	cpumask_and(&cpus, cpu_online_mask, &coupled->coupled_cpus);  in cpuidle_coupled_update_online_cpus()
639 	coupled->online_count = cpumask_weight(&cpus);  in cpuidle_coupled_update_online_cpus()
647  * cpuidle_coupled struct for this set of coupled cpus, or creates one if none
712  * cpuidle_coupled_prevent_idle - prevent cpus from entering a coupled state
715  * Disables coupled cpuidle on a coupled set of cpus.  Used to ensure that
716  * cpu_online_mask doesn't change while cpus are coordinating coupled idle.
722 	/* Force all cpus out of the waiting loop. */  in cpuidle_coupled_prevent_idle()
731  * cpuidle_coupled_allow_idle - allows cpus to enter a coupled state
734  * Enables coupled cpuidle on a coupled set of cpus.  Used to ensure that
735  * cpu_online_mask doesn't change while cpus are coordinating coupled idle.
747 	/* Force cpus out of the prevent loop. */  in cpuidle_coupled_allow_idle()