159 	int n = dev->coupled->online_count;  in cpuidle_coupled_parallel_barrier()
214 static inline void cpuidle_coupled_set_ready(struct cpuidle_coupled *coupled)  in cpuidle_coupled_set_ready()  argument
216 	atomic_add(MAX_WAITING_CPUS, &coupled->ready_waiting_counts);  in cpuidle_coupled_set_ready()
234 inline int cpuidle_coupled_set_not_ready(struct cpuidle_coupled *coupled)  in cpuidle_coupled_set_not_ready()  argument
239 	all = coupled->online_count | (coupled->online_count << WAITING_BITS);  in cpuidle_coupled_set_not_ready()
240 	ret = atomic_add_unless(&coupled->ready_waiting_counts,  in cpuidle_coupled_set_not_ready()
252 static inline int cpuidle_coupled_no_cpus_ready(struct cpuidle_coupled *coupled)  in cpuidle_coupled_no_cpus_ready()  argument
254 	int r = atomic_read(&coupled->ready_waiting_counts) >> WAITING_BITS;  in cpuidle_coupled_no_cpus_ready()
264 static inline bool cpuidle_coupled_cpus_ready(struct cpuidle_coupled *coupled)  in cpuidle_coupled_cpus_ready()  argument
266 	int r = atomic_read(&coupled->ready_waiting_counts) >> WAITING_BITS;  in cpuidle_coupled_cpus_ready()
267 	return r == coupled->online_count;  in cpuidle_coupled_cpus_ready()
276 static inline bool cpuidle_coupled_cpus_waiting(struct cpuidle_coupled *coupled)  in cpuidle_coupled_cpus_waiting()  argument
278 	int w = atomic_read(&coupled->ready_waiting_counts) & WAITING_MASK;  in cpuidle_coupled_cpus_waiting()
279 	return w == coupled->online_count;  in cpuidle_coupled_cpus_waiting()
288 static inline int cpuidle_coupled_no_cpus_waiting(struct cpuidle_coupled *coupled)  in cpuidle_coupled_no_cpus_waiting()  argument
290 	int w = atomic_read(&coupled->ready_waiting_counts) & WAITING_MASK;  in cpuidle_coupled_no_cpus_waiting()
302 		struct cpuidle_coupled *coupled)  in cpuidle_coupled_get_state()  argument
314 	for_each_cpu(i, &coupled->coupled_cpus)  in cpuidle_coupled_get_state()
315 		if (cpu_online(i) && coupled->requested_state[i] < state)  in cpuidle_coupled_get_state()
316 			state = coupled->requested_state[i];  in cpuidle_coupled_get_state()
356 		struct cpuidle_coupled *coupled)  in cpuidle_coupled_poke_others()  argument
360 	for_each_cpu(cpu, &coupled->coupled_cpus)  in cpuidle_coupled_poke_others()
375 		struct cpuidle_coupled *coupled, int next_state)  in cpuidle_coupled_set_waiting()  argument
377 	coupled->requested_state[cpu] = next_state;  in cpuidle_coupled_set_waiting()
383 	return atomic_inc_return(&coupled->ready_waiting_counts) & WAITING_MASK;  in cpuidle_coupled_set_waiting()
394 		struct cpuidle_coupled *coupled)  in cpuidle_coupled_set_not_waiting()  argument
402 	atomic_dec(&coupled->ready_waiting_counts);  in cpuidle_coupled_set_not_waiting()
404 	coupled->requested_state[cpu] = CPUIDLE_COUPLED_NOT_IDLE;  in cpuidle_coupled_set_not_waiting()
416 static void cpuidle_coupled_set_done(int cpu, struct cpuidle_coupled *coupled)  in cpuidle_coupled_set_done()  argument
418 	cpuidle_coupled_set_not_waiting(cpu, coupled);  in cpuidle_coupled_set_done()
419 	atomic_sub(MAX_WAITING_CPUS, &coupled->ready_waiting_counts);  in cpuidle_coupled_set_done()
448 static bool cpuidle_coupled_any_pokes_pending(struct cpuidle_coupled *coupled)  in cpuidle_coupled_any_pokes_pending()  argument
453 	cpumask_and(&cpus, cpu_online_mask, &coupled->coupled_cpus);  in cpuidle_coupled_any_pokes_pending()
482 	struct cpuidle_coupled *coupled = dev->coupled;  in cpuidle_enter_state_coupled()  local
485 	if (!coupled)  in cpuidle_enter_state_coupled()
488 	while (coupled->prevent) {  in cpuidle_enter_state_coupled()
505 	w = cpuidle_coupled_set_waiting(dev->cpu, coupled, next_state);  in cpuidle_enter_state_coupled()
513 	if (w == coupled->online_count) {  in cpuidle_enter_state_coupled()
515 		cpuidle_coupled_poke_others(dev->cpu, coupled);  in cpuidle_enter_state_coupled()
527 	while (!cpuidle_coupled_cpus_waiting(coupled) ||  in cpuidle_enter_state_coupled()
533 			cpuidle_coupled_set_not_waiting(dev->cpu, coupled);  in cpuidle_enter_state_coupled()
537 		if (coupled->prevent) {  in cpuidle_enter_state_coupled()
538 			cpuidle_coupled_set_not_waiting(dev->cpu, coupled);  in cpuidle_enter_state_coupled()
549 		cpuidle_coupled_set_not_waiting(dev->cpu, coupled);  in cpuidle_enter_state_coupled()
568 	cpuidle_coupled_set_ready(coupled);  in cpuidle_enter_state_coupled()
569 	while (!cpuidle_coupled_cpus_ready(coupled)) {  in cpuidle_enter_state_coupled()
571 		if (!cpuidle_coupled_cpus_waiting(coupled))  in cpuidle_enter_state_coupled()
572 			if (!cpuidle_coupled_set_not_ready(coupled))  in cpuidle_enter_state_coupled()
593 	if (cpuidle_coupled_any_pokes_pending(coupled)) {  in cpuidle_enter_state_coupled()
594 		cpuidle_coupled_set_done(dev->cpu, coupled);  in cpuidle_enter_state_coupled()
596 		cpuidle_coupled_parallel_barrier(dev, &coupled->abort_barrier);  in cpuidle_enter_state_coupled()
601 	next_state = cpuidle_coupled_get_state(dev, coupled);  in cpuidle_enter_state_coupled()
605 	cpuidle_coupled_set_done(dev->cpu, coupled);  in cpuidle_enter_state_coupled()
629 	while (!cpuidle_coupled_no_cpus_ready(coupled))  in cpuidle_enter_state_coupled()
635 static void cpuidle_coupled_update_online_cpus(struct cpuidle_coupled *coupled)  in cpuidle_coupled_update_online_cpus()  argument
638 	cpumask_and(&cpus, cpu_online_mask, &coupled->coupled_cpus);  in cpuidle_coupled_update_online_cpus()
639 	coupled->online_count = cpumask_weight(&cpus);  in cpuidle_coupled_update_online_cpus()
655 	struct cpuidle_coupled *coupled;  in cpuidle_coupled_register_device()  local
662 		if (other_dev && other_dev->coupled) {  in cpuidle_coupled_register_device()
663 			coupled = other_dev->coupled;  in cpuidle_coupled_register_device()
669 	coupled = kzalloc(sizeof(struct cpuidle_coupled), GFP_KERNEL);  in cpuidle_coupled_register_device()
670 	if (!coupled)  in cpuidle_coupled_register_device()
673 	coupled->coupled_cpus = dev->coupled_cpus;  in cpuidle_coupled_register_device()
676 	dev->coupled = coupled;  in cpuidle_coupled_register_device()
677 	if (WARN_ON(!cpumask_equal(&dev->coupled_cpus, &coupled->coupled_cpus)))  in cpuidle_coupled_register_device()
678 		coupled->prevent++;  in cpuidle_coupled_register_device()
680 	cpuidle_coupled_update_online_cpus(coupled);  in cpuidle_coupled_register_device()
682 	coupled->refcnt++;  in cpuidle_coupled_register_device()
701 	struct cpuidle_coupled *coupled = dev->coupled;  in cpuidle_coupled_unregister_device()  local
706 	if (--coupled->refcnt)  in cpuidle_coupled_unregister_device()
707 		kfree(coupled);  in cpuidle_coupled_unregister_device()
708 	dev->coupled = NULL;  in cpuidle_coupled_unregister_device()
718 static void cpuidle_coupled_prevent_idle(struct cpuidle_coupled *coupled)  in cpuidle_coupled_prevent_idle()  argument
723 	coupled->prevent++;  in cpuidle_coupled_prevent_idle()
724 	cpuidle_coupled_poke_others(cpu, coupled);  in cpuidle_coupled_prevent_idle()
726 	while (!cpuidle_coupled_no_cpus_waiting(coupled))  in cpuidle_coupled_prevent_idle()
737 static void cpuidle_coupled_allow_idle(struct cpuidle_coupled *coupled)  in cpuidle_coupled_allow_idle()  argument
746 	coupled->prevent--;  in cpuidle_coupled_allow_idle()
748 	cpuidle_coupled_poke_others(cpu, coupled);  in cpuidle_coupled_allow_idle()
759 	if (dev && dev->coupled) {  in coupled_cpu_online()
760 		cpuidle_coupled_update_online_cpus(dev->coupled);  in coupled_cpu_online()
761 		cpuidle_coupled_allow_idle(dev->coupled);  in coupled_cpu_online()
775 	if (dev && dev->coupled)  in coupled_cpu_up_prepare()
776 		cpuidle_coupled_prevent_idle(dev->coupled);  in coupled_cpu_up_prepare()