21 #define GENPD_DEV_CALLBACK(genpd, type, callback, dev)		\  argument
26 	__routine = genpd->dev_ops.callback; 			\
33 #define GENPD_DEV_TIMED_CALLBACK(genpd, type, callback, dev, field, name)	\  argument
36 	type __retval = GENPD_DEV_CALLBACK(genpd, type, callback, dev);		\
43 		genpd->max_off_time_changed = true;				\
54 	struct generic_pm_domain *genpd = NULL, *gpd;  in pm_genpd_lookup_name()  local
62 			genpd = gpd;  in pm_genpd_lookup_name()
67 	return genpd;  in pm_genpd_lookup_name()
78 static int genpd_stop_dev(struct generic_pm_domain *genpd, struct device *dev)  in genpd_stop_dev()  argument
80 	return GENPD_DEV_TIMED_CALLBACK(genpd, int, stop, dev,  in genpd_stop_dev()
84 static int genpd_start_dev(struct generic_pm_domain *genpd, struct device *dev)  in genpd_start_dev()  argument
86 	return GENPD_DEV_TIMED_CALLBACK(genpd, int, start, dev,  in genpd_start_dev()
90 static bool genpd_sd_counter_dec(struct generic_pm_domain *genpd)  in genpd_sd_counter_dec()  argument
94 	if (!WARN_ON(atomic_read(&genpd->sd_count) == 0))  in genpd_sd_counter_dec()
95 		ret = !!atomic_dec_and_test(&genpd->sd_count);  in genpd_sd_counter_dec()
100 static void genpd_sd_counter_inc(struct generic_pm_domain *genpd)  in genpd_sd_counter_inc()  argument
102 	atomic_inc(&genpd->sd_count);  in genpd_sd_counter_inc()
106 static void genpd_acquire_lock(struct generic_pm_domain *genpd)  in genpd_acquire_lock()  argument
110 	mutex_lock(&genpd->lock);  in genpd_acquire_lock()
116 		prepare_to_wait(&genpd->status_wait_queue, &wait,  in genpd_acquire_lock()
118 		if (genpd->status == GPD_STATE_ACTIVE  in genpd_acquire_lock()
119 		    || genpd->status == GPD_STATE_POWER_OFF)  in genpd_acquire_lock()
121 		mutex_unlock(&genpd->lock);  in genpd_acquire_lock()
125 		mutex_lock(&genpd->lock);  in genpd_acquire_lock()
127 	finish_wait(&genpd->status_wait_queue, &wait);  in genpd_acquire_lock()
130 static void genpd_release_lock(struct generic_pm_domain *genpd)  in genpd_release_lock()  argument
132 	mutex_unlock(&genpd->lock);  in genpd_release_lock()
135 static void genpd_set_active(struct generic_pm_domain *genpd)  in genpd_set_active()  argument
137 	if (genpd->resume_count == 0)  in genpd_set_active()
138 		genpd->status = GPD_STATE_ACTIVE;  in genpd_set_active()
141 static void genpd_recalc_cpu_exit_latency(struct generic_pm_domain *genpd)  in genpd_recalc_cpu_exit_latency()  argument
145 	if (!genpd->cpuidle_data)  in genpd_recalc_cpu_exit_latency()
148 	usecs64 = genpd->power_on_latency_ns;  in genpd_recalc_cpu_exit_latency()
150 	usecs64 += genpd->cpuidle_data->saved_exit_latency;  in genpd_recalc_cpu_exit_latency()
151 	genpd->cpuidle_data->idle_state->exit_latency = usecs64;  in genpd_recalc_cpu_exit_latency()
161 static int __pm_genpd_poweron(struct generic_pm_domain *genpd)  in __pm_genpd_poweron()  argument
162 	__releases(&genpd->lock) __acquires(&genpd->lock)  in __pm_genpd_poweron()
170 		prepare_to_wait(&genpd->status_wait_queue, &wait,  in __pm_genpd_poweron()
172 		if (genpd->status != GPD_STATE_WAIT_MASTER)  in __pm_genpd_poweron()
174 		mutex_unlock(&genpd->lock);  in __pm_genpd_poweron()
178 		mutex_lock(&genpd->lock);  in __pm_genpd_poweron()
180 	finish_wait(&genpd->status_wait_queue, &wait);  in __pm_genpd_poweron()
182 	if (genpd->status == GPD_STATE_ACTIVE  in __pm_genpd_poweron()
183 	    || (genpd->prepared_count > 0 && genpd->suspend_power_off))  in __pm_genpd_poweron()
186 	if (genpd->status != GPD_STATE_POWER_OFF) {  in __pm_genpd_poweron()
187 		genpd_set_active(genpd);  in __pm_genpd_poweron()
191 	if (genpd->cpuidle_data) {  in __pm_genpd_poweron()
193 		genpd->cpuidle_data->idle_state->disabled = true;  in __pm_genpd_poweron()
203 	list_for_each_entry(link, &genpd->slave_links, slave_node) {  in __pm_genpd_poweron()
205 		genpd->status = GPD_STATE_WAIT_MASTER;  in __pm_genpd_poweron()
207 		mutex_unlock(&genpd->lock);  in __pm_genpd_poweron()
211 		mutex_lock(&genpd->lock);  in __pm_genpd_poweron()
217 		genpd->status = GPD_STATE_POWER_OFF;  in __pm_genpd_poweron()
218 		wake_up_all(&genpd->status_wait_queue);  in __pm_genpd_poweron()
225 	if (genpd->power_on) {  in __pm_genpd_poweron()
229 		ret = genpd->power_on(genpd);  in __pm_genpd_poweron()
234 		if (elapsed_ns > genpd->power_on_latency_ns) {  in __pm_genpd_poweron()
235 			genpd->power_on_latency_ns = elapsed_ns;  in __pm_genpd_poweron()
236 			genpd->max_off_time_changed = true;  in __pm_genpd_poweron()
237 			genpd_recalc_cpu_exit_latency(genpd);  in __pm_genpd_poweron()
238 			if (genpd->name)  in __pm_genpd_poweron()
240 					"new value %lld ns\n", genpd->name,  in __pm_genpd_poweron()
246 	genpd_set_active(genpd);  in __pm_genpd_poweron()
251 	list_for_each_entry_continue_reverse(link, &genpd->slave_links, slave_node)  in __pm_genpd_poweron()
261 int pm_genpd_poweron(struct generic_pm_domain *genpd)  in pm_genpd_poweron()  argument
265 	mutex_lock(&genpd->lock);  in pm_genpd_poweron()
266 	ret = __pm_genpd_poweron(genpd);  in pm_genpd_poweron()
267 	mutex_unlock(&genpd->lock);  in pm_genpd_poweron()
277 	struct generic_pm_domain *genpd;  in pm_genpd_name_poweron()  local
279 	genpd = pm_genpd_lookup_name(domain_name);  in pm_genpd_name_poweron()
280 	return genpd ? pm_genpd_poweron(genpd) : -EINVAL;  in pm_genpd_name_poweron()
285 static int genpd_start_dev_no_timing(struct generic_pm_domain *genpd,  in genpd_start_dev_no_timing()  argument
288 	return GENPD_DEV_CALLBACK(genpd, int, start, dev);  in genpd_start_dev_no_timing()
291 static int genpd_save_dev(struct generic_pm_domain *genpd, struct device *dev)  in genpd_save_dev()  argument
293 	return GENPD_DEV_TIMED_CALLBACK(genpd, int, save_state, dev,  in genpd_save_dev()
297 static int genpd_restore_dev(struct generic_pm_domain *genpd, struct device *dev)  in genpd_restore_dev()  argument
299 	return GENPD_DEV_TIMED_CALLBACK(genpd, int, restore_state, dev,  in genpd_restore_dev()
321 		struct generic_pm_domain *genpd;  in genpd_dev_pm_qos_notifier()  local
330 			genpd = dev_to_genpd(dev);  in genpd_dev_pm_qos_notifier()
332 			genpd = ERR_PTR(-ENODATA);  in genpd_dev_pm_qos_notifier()
337 		if (!IS_ERR(genpd)) {  in genpd_dev_pm_qos_notifier()
338 			mutex_lock(&genpd->lock);  in genpd_dev_pm_qos_notifier()
339 			genpd->max_off_time_changed = true;  in genpd_dev_pm_qos_notifier()
340 			mutex_unlock(&genpd->lock);  in genpd_dev_pm_qos_notifier()
357 				  struct generic_pm_domain *genpd)  in __pm_genpd_save_device()  argument
358 	__releases(&genpd->lock) __acquires(&genpd->lock)  in __pm_genpd_save_device()
377 	mutex_unlock(&genpd->lock);  in __pm_genpd_save_device()
379 	genpd_start_dev(genpd, dev);  in __pm_genpd_save_device()
380 	ret = genpd_save_dev(genpd, dev);  in __pm_genpd_save_device()
381 	genpd_stop_dev(genpd, dev);  in __pm_genpd_save_device()
383 	mutex_lock(&genpd->lock);  in __pm_genpd_save_device()
397 				      struct generic_pm_domain *genpd)  in __pm_genpd_restore_device()  argument
398 	__releases(&genpd->lock) __acquires(&genpd->lock)  in __pm_genpd_restore_device()
405 	mutex_unlock(&genpd->lock);  in __pm_genpd_restore_device()
407 	genpd_start_dev(genpd, dev);  in __pm_genpd_restore_device()
414 		genpd_restore_dev(genpd, dev);  in __pm_genpd_restore_device()
416 	mutex_lock(&genpd->lock);  in __pm_genpd_restore_device()
428 static bool genpd_abort_poweroff(struct generic_pm_domain *genpd)  in genpd_abort_poweroff()  argument
430 	return genpd->status == GPD_STATE_WAIT_MASTER  in genpd_abort_poweroff()
431 		|| genpd->status == GPD_STATE_ACTIVE || genpd->resume_count > 0;  in genpd_abort_poweroff()
441 static void genpd_queue_power_off_work(struct generic_pm_domain *genpd)  in genpd_queue_power_off_work()  argument
443 	queue_work(pm_wq, &genpd->power_off_work);  in genpd_queue_power_off_work()
454 static int pm_genpd_poweroff(struct generic_pm_domain *genpd)  in pm_genpd_poweroff()  argument
455 	__releases(&genpd->lock) __acquires(&genpd->lock)  in pm_genpd_poweroff()
470 	if (genpd->status == GPD_STATE_POWER_OFF  in pm_genpd_poweroff()
471 	    || genpd->status == GPD_STATE_WAIT_MASTER  in pm_genpd_poweroff()
472 	    || genpd->resume_count > 0 || genpd->prepared_count > 0)  in pm_genpd_poweroff()
475 	if (atomic_read(&genpd->sd_count) > 0)  in pm_genpd_poweroff()
479 	list_for_each_entry(pdd, &genpd->dev_list, list_node) {  in pm_genpd_poweroff()
493 	if (not_suspended > genpd->in_progress)  in pm_genpd_poweroff()
496 	if (genpd->poweroff_task) {  in pm_genpd_poweroff()
501 		genpd->status = GPD_STATE_REPEAT;  in pm_genpd_poweroff()
505 	if (genpd->gov && genpd->gov->power_down_ok) {  in pm_genpd_poweroff()
506 		if (!genpd->gov->power_down_ok(&genpd->domain))  in pm_genpd_poweroff()
510 	genpd->status = GPD_STATE_BUSY;  in pm_genpd_poweroff()
511 	genpd->poweroff_task = current;  in pm_genpd_poweroff()
513 	list_for_each_entry_reverse(pdd, &genpd->dev_list, list_node) {  in pm_genpd_poweroff()
514 		ret = atomic_read(&genpd->sd_count) == 0 ?  in pm_genpd_poweroff()
515 			__pm_genpd_save_device(pdd, genpd) : -EBUSY;  in pm_genpd_poweroff()
517 		if (genpd_abort_poweroff(genpd))  in pm_genpd_poweroff()
521 			genpd_set_active(genpd);  in pm_genpd_poweroff()
525 		if (genpd->status == GPD_STATE_REPEAT) {  in pm_genpd_poweroff()
526 			genpd->poweroff_task = NULL;  in pm_genpd_poweroff()
531 	if (genpd->cpuidle_data) {  in pm_genpd_poweroff()
539 		genpd->status = GPD_STATE_POWER_OFF;  in pm_genpd_poweroff()
541 		genpd->cpuidle_data->idle_state->disabled = false;  in pm_genpd_poweroff()
546 	if (genpd->power_off) {  in pm_genpd_poweroff()
550 		if (atomic_read(&genpd->sd_count) > 0) {  in pm_genpd_poweroff()
565 		ret = genpd->power_off(genpd);  in pm_genpd_poweroff()
567 			genpd_set_active(genpd);  in pm_genpd_poweroff()
572 		if (elapsed_ns > genpd->power_off_latency_ns) {  in pm_genpd_poweroff()
573 			genpd->power_off_latency_ns = elapsed_ns;  in pm_genpd_poweroff()
574 			genpd->max_off_time_changed = true;  in pm_genpd_poweroff()
575 			if (genpd->name)  in pm_genpd_poweroff()
577 					"new value %lld ns\n", genpd->name,  in pm_genpd_poweroff()
582 	genpd->status = GPD_STATE_POWER_OFF;  in pm_genpd_poweroff()
584 	list_for_each_entry(link, &genpd->slave_links, slave_node) {  in pm_genpd_poweroff()
590 	genpd->poweroff_task = NULL;  in pm_genpd_poweroff()
591 	wake_up_all(&genpd->status_wait_queue);  in pm_genpd_poweroff()
601 	struct generic_pm_domain *genpd;  in genpd_power_off_work_fn()  local
603 	genpd = container_of(work, struct generic_pm_domain, power_off_work);  in genpd_power_off_work_fn()
605 	genpd_acquire_lock(genpd);  in genpd_power_off_work_fn()
606 	pm_genpd_poweroff(genpd);  in genpd_power_off_work_fn()
607 	genpd_release_lock(genpd);  in genpd_power_off_work_fn()
620 	struct generic_pm_domain *genpd;  in pm_genpd_runtime_suspend()  local
627 	genpd = dev_to_genpd(dev);  in pm_genpd_runtime_suspend()
628 	if (IS_ERR(genpd))  in pm_genpd_runtime_suspend()
631 	stop_ok = genpd->gov ? genpd->gov->stop_ok : NULL;  in pm_genpd_runtime_suspend()
635 	ret = genpd_stop_dev(genpd, dev);  in pm_genpd_runtime_suspend()
646 	mutex_lock(&genpd->lock);  in pm_genpd_runtime_suspend()
657 	genpd->in_progress++;  in pm_genpd_runtime_suspend()
658 	pm_genpd_poweroff(genpd);  in pm_genpd_runtime_suspend()
659 	genpd->in_progress--;  in pm_genpd_runtime_suspend()
660 	mutex_unlock(&genpd->lock);  in pm_genpd_runtime_suspend()
675 	struct generic_pm_domain *genpd;  in pm_genpd_runtime_resume()  local
681 	genpd = dev_to_genpd(dev);  in pm_genpd_runtime_resume()
682 	if (IS_ERR(genpd))  in pm_genpd_runtime_resume()
687 		return genpd_start_dev_no_timing(genpd, dev);  in pm_genpd_runtime_resume()
689 	mutex_lock(&genpd->lock);  in pm_genpd_runtime_resume()
690 	ret = __pm_genpd_poweron(genpd);  in pm_genpd_runtime_resume()
692 		mutex_unlock(&genpd->lock);  in pm_genpd_runtime_resume()
695 	genpd->status = GPD_STATE_BUSY;  in pm_genpd_runtime_resume()
696 	genpd->resume_count++;  in pm_genpd_runtime_resume()
698 		prepare_to_wait(&genpd->status_wait_queue, &wait,  in pm_genpd_runtime_resume()
705 		if (!genpd->poweroff_task || genpd->poweroff_task == current)  in pm_genpd_runtime_resume()
707 		mutex_unlock(&genpd->lock);  in pm_genpd_runtime_resume()
711 		mutex_lock(&genpd->lock);  in pm_genpd_runtime_resume()
713 	finish_wait(&genpd->status_wait_queue, &wait);  in pm_genpd_runtime_resume()
714 	__pm_genpd_restore_device(dev->power.subsys_data->domain_data, genpd);  in pm_genpd_runtime_resume()
715 	genpd->resume_count--;  in pm_genpd_runtime_resume()
716 	genpd_set_active(genpd);  in pm_genpd_runtime_resume()
717 	wake_up_all(&genpd->status_wait_queue);  in pm_genpd_runtime_resume()
718 	mutex_unlock(&genpd->lock);  in pm_genpd_runtime_resume()
736 	struct generic_pm_domain *genpd;  in pm_genpd_poweroff_unused()  local
745 	list_for_each_entry(genpd, &gpd_list, gpd_list_node)  in pm_genpd_poweroff_unused()
746 		genpd_queue_power_off_work(genpd);  in pm_genpd_poweroff_unused()
767 genpd_queue_power_off_work(struct generic_pm_domain *genpd) {}  in genpd_queue_power_off_work()  argument
782 static bool pm_genpd_present(struct generic_pm_domain *genpd)  in pm_genpd_present()  argument
786 	if (IS_ERR_OR_NULL(genpd))  in pm_genpd_present()
790 		if (gpd == genpd)  in pm_genpd_present()
796 static bool genpd_dev_active_wakeup(struct generic_pm_domain *genpd,  in genpd_dev_active_wakeup()  argument
799 	return GENPD_DEV_CALLBACK(genpd, bool, active_wakeup, dev);  in genpd_dev_active_wakeup()
814 static void pm_genpd_sync_poweroff(struct generic_pm_domain *genpd)  in pm_genpd_sync_poweroff()  argument
818 	if (genpd->status == GPD_STATE_POWER_OFF)  in pm_genpd_sync_poweroff()
821 	if (genpd->suspended_count != genpd->device_count  in pm_genpd_sync_poweroff()
822 	    || atomic_read(&genpd->sd_count) > 0)  in pm_genpd_sync_poweroff()
825 	if (genpd->power_off)  in pm_genpd_sync_poweroff()
826 		genpd->power_off(genpd);  in pm_genpd_sync_poweroff()
828 	genpd->status = GPD_STATE_POWER_OFF;  in pm_genpd_sync_poweroff()
830 	list_for_each_entry(link, &genpd->slave_links, slave_node) {  in pm_genpd_sync_poweroff()
845 static void pm_genpd_sync_poweron(struct generic_pm_domain *genpd)  in pm_genpd_sync_poweron()  argument
849 	if (genpd->status != GPD_STATE_POWER_OFF)  in pm_genpd_sync_poweron()
852 	list_for_each_entry(link, &genpd->slave_links, slave_node) {  in pm_genpd_sync_poweron()
857 	if (genpd->power_on)  in pm_genpd_sync_poweron()
858 		genpd->power_on(genpd);  in pm_genpd_sync_poweron()
860 	genpd->status = GPD_STATE_ACTIVE;  in pm_genpd_sync_poweron()
879 static bool resume_needed(struct device *dev, struct generic_pm_domain *genpd)  in resume_needed()  argument
886 	active_wakeup = genpd_dev_active_wakeup(genpd, dev);  in resume_needed()
901 	struct generic_pm_domain *genpd;  in pm_genpd_prepare()  local
906 	genpd = dev_to_genpd(dev);  in pm_genpd_prepare()
907 	if (IS_ERR(genpd))  in pm_genpd_prepare()
924 	if (resume_needed(dev, genpd))  in pm_genpd_prepare()
927 	genpd_acquire_lock(genpd);  in pm_genpd_prepare()
929 	if (genpd->prepared_count++ == 0) {  in pm_genpd_prepare()
930 		genpd->suspended_count = 0;  in pm_genpd_prepare()
931 		genpd->suspend_power_off = genpd->status == GPD_STATE_POWER_OFF;  in pm_genpd_prepare()
934 	genpd_release_lock(genpd);  in pm_genpd_prepare()
936 	if (genpd->suspend_power_off) {  in pm_genpd_prepare()
952 		mutex_lock(&genpd->lock);  in pm_genpd_prepare()
954 		if (--genpd->prepared_count == 0)  in pm_genpd_prepare()
955 			genpd->suspend_power_off = false;  in pm_genpd_prepare()
957 		mutex_unlock(&genpd->lock);  in pm_genpd_prepare()
975 	struct generic_pm_domain *genpd;  in pm_genpd_suspend()  local
979 	genpd = dev_to_genpd(dev);  in pm_genpd_suspend()
980 	if (IS_ERR(genpd))  in pm_genpd_suspend()
983 	return genpd->suspend_power_off ? 0 : pm_generic_suspend(dev);  in pm_genpd_suspend()
996 	struct generic_pm_domain *genpd;  in pm_genpd_suspend_late()  local
1000 	genpd = dev_to_genpd(dev);  in pm_genpd_suspend_late()
1001 	if (IS_ERR(genpd))  in pm_genpd_suspend_late()
1004 	return genpd->suspend_power_off ? 0 : pm_generic_suspend_late(dev);  in pm_genpd_suspend_late()
1016 	struct generic_pm_domain *genpd;  in pm_genpd_suspend_noirq()  local
1020 	genpd = dev_to_genpd(dev);  in pm_genpd_suspend_noirq()
1021 	if (IS_ERR(genpd))  in pm_genpd_suspend_noirq()
1024 	if (genpd->suspend_power_off  in pm_genpd_suspend_noirq()
1025 	    || (dev->power.wakeup_path && genpd_dev_active_wakeup(genpd, dev)))  in pm_genpd_suspend_noirq()
1028 	genpd_stop_dev(genpd, dev);  in pm_genpd_suspend_noirq()
1035 	genpd->suspended_count++;  in pm_genpd_suspend_noirq()
1036 	pm_genpd_sync_poweroff(genpd);  in pm_genpd_suspend_noirq()
1049 	struct generic_pm_domain *genpd;  in pm_genpd_resume_noirq()  local
1053 	genpd = dev_to_genpd(dev);  in pm_genpd_resume_noirq()
1054 	if (IS_ERR(genpd))  in pm_genpd_resume_noirq()
1057 	if (genpd->suspend_power_off  in pm_genpd_resume_noirq()
1058 	    || (dev->power.wakeup_path && genpd_dev_active_wakeup(genpd, dev)))  in pm_genpd_resume_noirq()
1066 	pm_genpd_sync_poweron(genpd);  in pm_genpd_resume_noirq()
1067 	genpd->suspended_count--;  in pm_genpd_resume_noirq()
1069 	return genpd_start_dev(genpd, dev);  in pm_genpd_resume_noirq()
1083 	struct generic_pm_domain *genpd;  in pm_genpd_resume_early()  local
1087 	genpd = dev_to_genpd(dev);  in pm_genpd_resume_early()
1088 	if (IS_ERR(genpd))  in pm_genpd_resume_early()
1091 	return genpd->suspend_power_off ? 0 : pm_generic_resume_early(dev);  in pm_genpd_resume_early()
1104 	struct generic_pm_domain *genpd;  in pm_genpd_resume()  local
1108 	genpd = dev_to_genpd(dev);  in pm_genpd_resume()
1109 	if (IS_ERR(genpd))  in pm_genpd_resume()
1112 	return genpd->suspend_power_off ? 0 : pm_generic_resume(dev);  in pm_genpd_resume()
1125 	struct generic_pm_domain *genpd;  in pm_genpd_freeze()  local
1129 	genpd = dev_to_genpd(dev);  in pm_genpd_freeze()
1130 	if (IS_ERR(genpd))  in pm_genpd_freeze()
1133 	return genpd->suspend_power_off ? 0 : pm_generic_freeze(dev);  in pm_genpd_freeze()
1147 	struct generic_pm_domain *genpd;  in pm_genpd_freeze_late()  local
1151 	genpd = dev_to_genpd(dev);  in pm_genpd_freeze_late()
1152 	if (IS_ERR(genpd))  in pm_genpd_freeze_late()
1155 	return genpd->suspend_power_off ? 0 : pm_generic_freeze_late(dev);  in pm_genpd_freeze_late()
1169 	struct generic_pm_domain *genpd;  in pm_genpd_freeze_noirq()  local
1173 	genpd = dev_to_genpd(dev);  in pm_genpd_freeze_noirq()
1174 	if (IS_ERR(genpd))  in pm_genpd_freeze_noirq()
1177 	return genpd->suspend_power_off ? 0 : genpd_stop_dev(genpd, dev);  in pm_genpd_freeze_noirq()
1189 	struct generic_pm_domain *genpd;  in pm_genpd_thaw_noirq()  local
1193 	genpd = dev_to_genpd(dev);  in pm_genpd_thaw_noirq()
1194 	if (IS_ERR(genpd))  in pm_genpd_thaw_noirq()
1197 	return genpd->suspend_power_off ? 0 : genpd_start_dev(genpd, dev);  in pm_genpd_thaw_noirq()
1211 	struct generic_pm_domain *genpd;  in pm_genpd_thaw_early()  local
1215 	genpd = dev_to_genpd(dev);  in pm_genpd_thaw_early()
1216 	if (IS_ERR(genpd))  in pm_genpd_thaw_early()
1219 	return genpd->suspend_power_off ? 0 : pm_generic_thaw_early(dev);  in pm_genpd_thaw_early()
1232 	struct generic_pm_domain *genpd;  in pm_genpd_thaw()  local
1236 	genpd = dev_to_genpd(dev);  in pm_genpd_thaw()
1237 	if (IS_ERR(genpd))  in pm_genpd_thaw()
1240 	return genpd->suspend_power_off ? 0 : pm_generic_thaw(dev);  in pm_genpd_thaw()
1252 	struct generic_pm_domain *genpd;  in pm_genpd_restore_noirq()  local
1256 	genpd = dev_to_genpd(dev);  in pm_genpd_restore_noirq()
1257 	if (IS_ERR(genpd))  in pm_genpd_restore_noirq()
1268 	if (genpd->suspended_count++ == 0) {  in pm_genpd_restore_noirq()
1274 		genpd->status = GPD_STATE_POWER_OFF;  in pm_genpd_restore_noirq()
1275 		if (genpd->suspend_power_off) {  in pm_genpd_restore_noirq()
1280 			if (genpd->power_off)  in pm_genpd_restore_noirq()
1281 				genpd->power_off(genpd);  in pm_genpd_restore_noirq()
1287 	if (genpd->suspend_power_off)  in pm_genpd_restore_noirq()
1290 	pm_genpd_sync_poweron(genpd);  in pm_genpd_restore_noirq()
1292 	return genpd_start_dev(genpd, dev);  in pm_genpd_restore_noirq()
1306 	struct generic_pm_domain *genpd;  in pm_genpd_complete()  local
1311 	genpd = dev_to_genpd(dev);  in pm_genpd_complete()
1312 	if (IS_ERR(genpd))  in pm_genpd_complete()
1315 	mutex_lock(&genpd->lock);  in pm_genpd_complete()
1317 	run_complete = !genpd->suspend_power_off;  in pm_genpd_complete()
1318 	if (--genpd->prepared_count == 0)  in pm_genpd_complete()
1319 		genpd->suspend_power_off = false;  in pm_genpd_complete()
1321 	mutex_unlock(&genpd->lock);  in pm_genpd_complete()
1340 	struct generic_pm_domain *genpd;  in genpd_syscore_switch()  local
1342 	genpd = dev_to_genpd(dev);  in genpd_syscore_switch()
1343 	if (!pm_genpd_present(genpd))  in genpd_syscore_switch()
1347 		genpd->suspended_count++;  in genpd_syscore_switch()
1348 		pm_genpd_sync_poweroff(genpd);  in genpd_syscore_switch()
1350 		pm_genpd_sync_poweron(genpd);  in genpd_syscore_switch()
1351 		genpd->suspended_count--;  in genpd_syscore_switch()
1414 int __pm_genpd_add_device(struct generic_pm_domain *genpd, struct device *dev,  in __pm_genpd_add_device()  argument
1423 	if (IS_ERR_OR_NULL(genpd) || IS_ERR_OR_NULL(dev))  in __pm_genpd_add_device()
1430 	genpd_acquire_lock(genpd);  in __pm_genpd_add_device()
1432 	if (genpd->prepared_count > 0) {  in __pm_genpd_add_device()
1437 	list_for_each_entry(pdd, &genpd->dev_list, list_node)  in __pm_genpd_add_device()
1447 	genpd->device_count++;  in __pm_genpd_add_device()
1448 	genpd->max_off_time_changed = true;  in __pm_genpd_add_device()
1452 	dev->pm_domain = &genpd->domain;  in __pm_genpd_add_device()
1465 	if (genpd->attach_dev)  in __pm_genpd_add_device()
1466 		genpd->attach_dev(genpd, dev);  in __pm_genpd_add_device()
1470 	list_add_tail(&gpd_data->base.list_node, &genpd->dev_list);  in __pm_genpd_add_device()
1477 	genpd_release_lock(genpd);  in __pm_genpd_add_device()
1502 int pm_genpd_remove_device(struct generic_pm_domain *genpd,  in pm_genpd_remove_device()  argument
1512 	if (IS_ERR_OR_NULL(genpd) || IS_ERR_OR_NULL(dev)  in pm_genpd_remove_device()
1514 	    ||  pd_to_genpd(dev->pm_domain) != genpd)  in pm_genpd_remove_device()
1517 	genpd_acquire_lock(genpd);  in pm_genpd_remove_device()
1519 	if (genpd->prepared_count > 0) {  in pm_genpd_remove_device()
1524 	genpd->device_count--;  in pm_genpd_remove_device()
1525 	genpd->max_off_time_changed = true;  in pm_genpd_remove_device()
1527 	if (genpd->detach_dev)  in pm_genpd_remove_device()
1528 		genpd->detach_dev(genpd, dev);  in pm_genpd_remove_device()
1547 	genpd_release_lock(genpd);  in pm_genpd_remove_device()
1556 	genpd_release_lock(genpd);  in pm_genpd_remove_device()
1586 int pm_genpd_add_subdomain(struct generic_pm_domain *genpd,  in pm_genpd_add_subdomain()  argument
1592 	if (IS_ERR_OR_NULL(genpd) || IS_ERR_OR_NULL(subdomain)  in pm_genpd_add_subdomain()
1593 	    || genpd == subdomain)  in pm_genpd_add_subdomain()
1597 	genpd_acquire_lock(genpd);  in pm_genpd_add_subdomain()
1603 		genpd_release_lock(genpd);  in pm_genpd_add_subdomain()
1607 	if (genpd->status == GPD_STATE_POWER_OFF  in pm_genpd_add_subdomain()
1613 	list_for_each_entry(link, &genpd->master_links, master_node) {  in pm_genpd_add_subdomain()
1614 		if (link->slave == subdomain && link->master == genpd) {  in pm_genpd_add_subdomain()
1625 	link->master = genpd;  in pm_genpd_add_subdomain()
1626 	list_add_tail(&link->master_node, &genpd->master_links);  in pm_genpd_add_subdomain()
1630 		genpd_sd_counter_inc(genpd);  in pm_genpd_add_subdomain()
1634 	genpd_release_lock(genpd);  in pm_genpd_add_subdomain()
1673 int pm_genpd_remove_subdomain(struct generic_pm_domain *genpd,  in pm_genpd_remove_subdomain()  argument
1679 	if (IS_ERR_OR_NULL(genpd) || IS_ERR_OR_NULL(subdomain))  in pm_genpd_remove_subdomain()
1683 	genpd_acquire_lock(genpd);  in pm_genpd_remove_subdomain()
1685 	list_for_each_entry_safe(link, l, &genpd->master_links, master_node) {  in pm_genpd_remove_subdomain()
1694 			genpd_release_lock(genpd);  in pm_genpd_remove_subdomain()
1702 			genpd_sd_counter_dec(genpd);  in pm_genpd_remove_subdomain()
1710 	genpd_release_lock(genpd);  in pm_genpd_remove_subdomain()
1724 int pm_genpd_attach_cpuidle(struct generic_pm_domain *genpd, int state)  in pm_genpd_attach_cpuidle()  argument
1731 	if (IS_ERR_OR_NULL(genpd) || state < 0)  in pm_genpd_attach_cpuidle()
1734 	genpd_acquire_lock(genpd);  in pm_genpd_attach_cpuidle()
1736 	if (genpd->cpuidle_data) {  in pm_genpd_attach_cpuidle()
1761 	genpd->cpuidle_data = cpuidle_data;  in pm_genpd_attach_cpuidle()
1762 	genpd_recalc_cpu_exit_latency(genpd);  in pm_genpd_attach_cpuidle()
1765 	genpd_release_lock(genpd);  in pm_genpd_attach_cpuidle()
1793 int pm_genpd_detach_cpuidle(struct generic_pm_domain *genpd)  in pm_genpd_detach_cpuidle()  argument
1799 	if (IS_ERR_OR_NULL(genpd))  in pm_genpd_detach_cpuidle()
1802 	genpd_acquire_lock(genpd);  in pm_genpd_detach_cpuidle()
1804 	cpuidle_data = genpd->cpuidle_data;  in pm_genpd_detach_cpuidle()
1816 	genpd->cpuidle_data = NULL;  in pm_genpd_detach_cpuidle()
1820 	genpd_release_lock(genpd);  in pm_genpd_detach_cpuidle()
1887 void pm_genpd_init(struct generic_pm_domain *genpd,  in pm_genpd_init()  argument
1890 	if (IS_ERR_OR_NULL(genpd))  in pm_genpd_init()
1893 	INIT_LIST_HEAD(&genpd->master_links);  in pm_genpd_init()
1894 	INIT_LIST_HEAD(&genpd->slave_links);  in pm_genpd_init()
1895 	INIT_LIST_HEAD(&genpd->dev_list);  in pm_genpd_init()
1896 	mutex_init(&genpd->lock);  in pm_genpd_init()
1897 	genpd->gov = gov;  in pm_genpd_init()
1898 	INIT_WORK(&genpd->power_off_work, genpd_power_off_work_fn);  in pm_genpd_init()
1899 	genpd->in_progress = 0;  in pm_genpd_init()
1900 	atomic_set(&genpd->sd_count, 0);  in pm_genpd_init()
1901 	genpd->status = is_off ? GPD_STATE_POWER_OFF : GPD_STATE_ACTIVE;  in pm_genpd_init()
1902 	init_waitqueue_head(&genpd->status_wait_queue);  in pm_genpd_init()
1903 	genpd->poweroff_task = NULL;  in pm_genpd_init()
1904 	genpd->resume_count = 0;  in pm_genpd_init()
1905 	genpd->device_count = 0;  in pm_genpd_init()
1906 	genpd->max_off_time_ns = -1;  in pm_genpd_init()
1907 	genpd->max_off_time_changed = true;  in pm_genpd_init()
1908 	genpd->domain.ops.runtime_suspend = pm_genpd_runtime_suspend;  in pm_genpd_init()
1909 	genpd->domain.ops.runtime_resume = pm_genpd_runtime_resume;  in pm_genpd_init()
1910 	genpd->domain.ops.prepare = pm_genpd_prepare;  in pm_genpd_init()
1911 	genpd->domain.ops.suspend = pm_genpd_suspend;  in pm_genpd_init()
1912 	genpd->domain.ops.suspend_late = pm_genpd_suspend_late;  in pm_genpd_init()
1913 	genpd->domain.ops.suspend_noirq = pm_genpd_suspend_noirq;  in pm_genpd_init()
1914 	genpd->domain.ops.resume_noirq = pm_genpd_resume_noirq;  in pm_genpd_init()
1915 	genpd->domain.ops.resume_early = pm_genpd_resume_early;  in pm_genpd_init()
1916 	genpd->domain.ops.resume = pm_genpd_resume;  in pm_genpd_init()
1917 	genpd->domain.ops.freeze = pm_genpd_freeze;  in pm_genpd_init()
1918 	genpd->domain.ops.freeze_late = pm_genpd_freeze_late;  in pm_genpd_init()
1919 	genpd->domain.ops.freeze_noirq = pm_genpd_freeze_noirq;  in pm_genpd_init()
1920 	genpd->domain.ops.thaw_noirq = pm_genpd_thaw_noirq;  in pm_genpd_init()
1921 	genpd->domain.ops.thaw_early = pm_genpd_thaw_early;  in pm_genpd_init()
1922 	genpd->domain.ops.thaw = pm_genpd_thaw;  in pm_genpd_init()
1923 	genpd->domain.ops.poweroff = pm_genpd_suspend;  in pm_genpd_init()
1924 	genpd->domain.ops.poweroff_late = pm_genpd_suspend_late;  in pm_genpd_init()
1925 	genpd->domain.ops.poweroff_noirq = pm_genpd_suspend_noirq;  in pm_genpd_init()
1926 	genpd->domain.ops.restore_noirq = pm_genpd_restore_noirq;  in pm_genpd_init()
1927 	genpd->domain.ops.restore_early = pm_genpd_resume_early;  in pm_genpd_init()
1928 	genpd->domain.ops.restore = pm_genpd_resume;  in pm_genpd_init()
1929 	genpd->domain.ops.complete = pm_genpd_complete;  in pm_genpd_init()
1930 	genpd->dev_ops.save_state = pm_genpd_default_save_state;  in pm_genpd_init()
1931 	genpd->dev_ops.restore_state = pm_genpd_default_restore_state;  in pm_genpd_init()
1933 	list_add(&genpd->gpd_list_node, &gpd_list);  in pm_genpd_init()
2088 	struct generic_pm_domain *genpd = ERR_PTR(-ENOENT);  in of_genpd_get_from_provider()  local
2096 			genpd = provider->xlate(genpdspec, provider->data);  in of_genpd_get_from_provider()
2097 		if (!IS_ERR(genpd))  in of_genpd_get_from_provider()
2103 	return genpd;  in of_genpd_get_from_provider()