70 #define lock_policy_rwsem(mode, cpu)					\  argument
71 static int lock_policy_rwsem_##mode(int cpu)				\
73 	int policy_cpu = per_cpu(cpufreq_policy_cpu, cpu);		\
80 lock_policy_rwsem(read, cpu);
81 lock_policy_rwsem(write, cpu);
83 #define unlock_policy_rwsem(mode, cpu)					\  argument
84 static void unlock_policy_rwsem_##mode(int cpu)				\
86 	int policy_cpu = per_cpu(cpufreq_policy_cpu, cpu);		\
91 unlock_policy_rwsem(read, cpu);
92 unlock_policy_rwsem(write, cpu);
97 static unsigned int __cpufreq_get(unsigned int cpu);
146 static struct cpufreq_policy *__cpufreq_cpu_get(unsigned int cpu, bool sysfs)  in __cpufreq_cpu_get()  argument
151 	if (cpu >= nr_cpu_ids)  in __cpufreq_cpu_get()
165 	data = per_cpu(cpufreq_cpu_data, cpu);  in __cpufreq_cpu_get()
184 struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu)  in cpufreq_cpu_get()  argument
189 	return __cpufreq_cpu_get(cpu, false);  in cpufreq_cpu_get()
193 static struct cpufreq_policy *cpufreq_cpu_get_sysfs(unsigned int cpu)  in cpufreq_cpu_get_sysfs()  argument
195 	return __cpufreq_cpu_get(cpu, true);  in cpufreq_cpu_get_sysfs()
282 			if ((policy) && (policy->cpu == freqs->cpu) &&  in __cpufreq_notify_transition()
298 			(unsigned long)freqs->cpu);  in __cpufreq_notify_transition()
299 		trace_cpu_frequency(freqs->new, freqs->cpu);  in __cpufreq_notify_transition()
302 		if (likely(policy) && likely(policy->cpu == freqs->cpu))  in __cpufreq_notify_transition()
318 	for_each_cpu(freqs->cpu, policy->cpus)  in cpufreq_notify_transition()
425 	ret = cpufreq_get_policy(&new_policy, policy->cpu);		\
448 	unsigned int cur_freq = __cpufreq_get(policy->cpu);  in show_cpuinfo_cur_freq()
481 	ret = cpufreq_get_policy(&new_policy, policy->cpu);  in store_scaling_governor()
542 	unsigned int cpu;  in show_cpus()  local
544 	for_each_cpu(cpu, mask) {  in show_cpus()
547 		i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), "%u", cpu);  in show_cpus()
606 		ret = cpufreq_driver->bios_limit(policy->cpu, &limit);  in show_bios_limit()
654 	policy = cpufreq_cpu_get_sysfs(policy->cpu);  in show()
658 	if (lock_policy_rwsem_read(policy->cpu) < 0)  in show()
666 	unlock_policy_rwsem_read(policy->cpu);  in show()
679 	policy = cpufreq_cpu_get_sysfs(policy->cpu);  in store()
683 	if (lock_policy_rwsem_write(policy->cpu) < 0)  in store()
691 	unlock_policy_rwsem_write(policy->cpu);  in store()
717 static int cpufreq_add_dev_symlink(unsigned int cpu,  in cpufreq_add_dev_symlink()  argument
727 		if (j == cpu)  in cpufreq_add_dev_symlink()
731 		managed_policy = cpufreq_cpu_get(cpu);  in cpufreq_add_dev_symlink()
743 static int cpufreq_add_dev_interface(unsigned int cpu,  in cpufreq_add_dev_interface()  argument
786 		per_cpu(cpufreq_policy_cpu, j) = policy->cpu;  in cpufreq_add_dev_interface()
790 	ret = cpufreq_add_dev_symlink(cpu, policy);  in cpufreq_add_dev_interface()
817 static int cpufreq_add_policy_cpu(unsigned int cpu, unsigned int sibling,  in cpufreq_add_policy_cpu()  argument
834 	cpumask_set_cpu(cpu, policy->cpus);  in cpufreq_add_policy_cpu()
835 	per_cpu(cpufreq_policy_cpu, cpu) = policy->cpu;  in cpufreq_add_policy_cpu()
836 	per_cpu(cpufreq_cpu_data, cpu) = policy;  in cpufreq_add_policy_cpu()
867 	unsigned int j, cpu = dev->id;  in cpufreq_add_dev()  local
876 	if (cpu_is_offline(cpu))  in cpufreq_add_dev()
879 	pr_debug("adding CPU %u\n", cpu);  in cpufreq_add_dev()
884 	policy = cpufreq_cpu_get(cpu);  in cpufreq_add_dev()
895 		if (cp && cpumask_test_cpu(cpu, cp->related_cpus)) {  in cpufreq_add_dev()
897 			return cpufreq_add_policy_cpu(cpu, sibling, dev);  in cpufreq_add_dev()
919 	policy->cpu = cpu;  in cpufreq_add_dev()
921 	cpumask_copy(policy->cpus, cpumask_of(cpu));  in cpufreq_add_dev()
924 	per_cpu(cpufreq_policy_cpu, cpu) = cpu;  in cpufreq_add_dev()
954 	gov = __find_governor(per_cpu(cpufreq_cpu_governor, cpu));  in cpufreq_add_dev()
958 		       policy->governor->name, cpu);  in cpufreq_add_dev()
962 	ret = cpufreq_add_dev_interface(cpu, policy, dev);  in cpufreq_add_dev()
982 	per_cpu(cpufreq_policy_cpu, cpu) = -1;  in cpufreq_add_dev()
994 static void update_policy_cpu(struct cpufreq_policy *policy, unsigned int cpu)  in update_policy_cpu()  argument
998 	policy->last_cpu = policy->cpu;  in update_policy_cpu()
999 	policy->cpu = cpu;  in update_policy_cpu()
1002 		per_cpu(cpufreq_policy_cpu, j) = cpu;  in update_policy_cpu()
1020 	unsigned int cpu = dev->id, ret, cpus;  in __cpufreq_remove_dev()  local
1027 	pr_debug("%s: unregistering CPU %u\n", __func__, cpu);  in __cpufreq_remove_dev()
1031 	data = per_cpu(cpufreq_cpu_data, cpu);  in __cpufreq_remove_dev()
1032 	per_cpu(cpufreq_cpu_data, cpu) = NULL;  in __cpufreq_remove_dev()
1046 		strncpy(per_cpu(cpufreq_cpu_governor, cpu),  in __cpufreq_remove_dev()
1050 	WARN_ON(lock_policy_rwsem_write(cpu));  in __cpufreq_remove_dev()
1054 		cpumask_clear_cpu(cpu, data->cpus);  in __cpufreq_remove_dev()
1055 	unlock_policy_rwsem_write(cpu);  in __cpufreq_remove_dev()
1057 	if (cpu != data->cpu) {  in __cpufreq_remove_dev()
1067 			WARN_ON(lock_policy_rwsem_write(cpu));  in __cpufreq_remove_dev()
1068 			cpumask_set_cpu(cpu, data->cpus);  in __cpufreq_remove_dev()
1071 			per_cpu(cpufreq_cpu_data, cpu) = data;  in __cpufreq_remove_dev()
1074 			unlock_policy_rwsem_write(cpu);  in __cpufreq_remove_dev()
1081 		WARN_ON(lock_policy_rwsem_write(cpu));  in __cpufreq_remove_dev()
1083 		unlock_policy_rwsem_write(cpu);  in __cpufreq_remove_dev()
1085 				__func__, cpu_dev->id, cpu);  in __cpufreq_remove_dev()
1091 	pr_debug("%s: removing link, cpu: %d\n", __func__, cpu);  in __cpufreq_remove_dev()
1096 		lock_policy_rwsem_read(cpu);  in __cpufreq_remove_dev()
1099 		unlock_policy_rwsem_read(cpu);  in __cpufreq_remove_dev()
1121 	per_cpu(cpufreq_policy_cpu, cpu) = -1;  in __cpufreq_remove_dev()
1128 	unsigned int cpu = dev->id;  in cpufreq_remove_dev()  local
1131 	if (cpu_is_offline(cpu))  in cpufreq_remove_dev()
1143 	unsigned int cpu = policy->cpu;  in handle_update()  local
1144 	pr_debug("handle_update for cpu %u called\n", cpu);  in handle_update()
1145 	cpufreq_update_policy(cpu);  in handle_update()
1157 static void cpufreq_out_of_sync(unsigned int cpu, unsigned int old_freq,  in cpufreq_out_of_sync()  argument
1172 	policy = per_cpu(cpufreq_cpu_data, cpu);  in cpufreq_out_of_sync()
1187 unsigned int cpufreq_quick_get(unsigned int cpu)  in cpufreq_quick_get()  argument
1193 		return cpufreq_driver->get(cpu);  in cpufreq_quick_get()
1195 	policy = cpufreq_cpu_get(cpu);  in cpufreq_quick_get()
1211 unsigned int cpufreq_quick_get_max(unsigned int cpu)  in cpufreq_quick_get_max()  argument
1213 	struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);  in cpufreq_quick_get_max()
1226 static unsigned int __cpufreq_get(unsigned int cpu)  in __cpufreq_get()  argument
1228 	struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);  in __cpufreq_get()
1234 	ret_freq = cpufreq_driver->get(cpu);  in __cpufreq_get()
1241 			cpufreq_out_of_sync(cpu, policy->cur, ret_freq);  in __cpufreq_get()
1255 unsigned int cpufreq_get(unsigned int cpu)  in cpufreq_get()  argument
1258 	struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);  in cpufreq_get()
1263 	if (unlikely(lock_policy_rwsem_read(cpu)))  in cpufreq_get()
1266 	ret_freq = __cpufreq_get(cpu);  in cpufreq_get()
1268 	unlock_policy_rwsem_read(cpu);  in cpufreq_get()
1295 	int cpu = smp_processor_id();  in cpufreq_bp_suspend()  local
1298 	pr_debug("suspending cpu %u\n", cpu);  in cpufreq_bp_suspend()
1301 	cpu_policy = cpufreq_cpu_get(cpu);  in cpufreq_bp_suspend()
1309 					"step on CPU %u\n", cpu_policy->cpu);  in cpufreq_bp_suspend()
1333 	int cpu = smp_processor_id();  in cpufreq_bp_resume()  local
1336 	pr_debug("resuming cpu %u\n", cpu);  in cpufreq_bp_resume()
1339 	cpu_policy = cpufreq_cpu_get(cpu);  in cpufreq_bp_resume()
1347 					"step on CPU %u\n", cpu_policy->cpu);  in cpufreq_bp_resume()
1479 			policy->cpu, target_freq, relation, old_target_freq);  in __cpufreq_driver_target()
1497 	policy = cpufreq_cpu_get(policy->cpu);  in cpufreq_driver_target()
1501 	if (unlikely(lock_policy_rwsem_write(policy->cpu)))  in cpufreq_driver_target()
1506 	unlock_policy_rwsem_write(policy->cpu);  in cpufreq_driver_target()
1515 int __cpufreq_driver_getavg(struct cpufreq_policy *policy, unsigned int cpu)  in __cpufreq_driver_getavg()  argument
1525 	policy = cpufreq_cpu_get(policy->cpu);  in __cpufreq_driver_getavg()
1529 	ret = cpufreq_driver->getavg(policy, cpu);  in __cpufreq_driver_getavg()
1574 						policy->cpu, event);  in __cpufreq_governor()
1623 	int cpu;  in cpufreq_unregister_governor()  local
1633 	for_each_present_cpu(cpu) {  in cpufreq_unregister_governor()
1634 		if (cpu_online(cpu))  in cpufreq_unregister_governor()
1636 		if (!strcmp(per_cpu(cpufreq_cpu_governor, cpu), governor->name))  in cpufreq_unregister_governor()
1637 			strcpy(per_cpu(cpufreq_cpu_governor, cpu), "\0");  in cpufreq_unregister_governor()
1661 int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu)  in cpufreq_get_policy()  argument
1667 	cpu_policy = cpufreq_cpu_get(cpu);  in cpufreq_get_policy()
1688 	pr_debug("setting new policy for CPU %u: %u - %u kHz\n", policy->cpu,  in __cpufreq_set_policy()
1724 	trace_cpu_frequency_limits(policy->max, policy->min, policy->cpu);  in __cpufreq_set_policy()
1743 				unlock_policy_rwsem_write(policy->cpu);  in __cpufreq_set_policy()
1746 				lock_policy_rwsem_write(policy->cpu);  in __cpufreq_set_policy()
1755 					unlock_policy_rwsem_write(policy->cpu);  in __cpufreq_set_policy()
1758 					lock_policy_rwsem_write(policy->cpu);  in __cpufreq_set_policy()
1793 int cpufreq_update_policy(unsigned int cpu)  in cpufreq_update_policy()  argument
1795 	struct cpufreq_policy *data = cpufreq_cpu_get(cpu);  in cpufreq_update_policy()
1804 	if (unlikely(lock_policy_rwsem_write(cpu))) {  in cpufreq_update_policy()
1809 	pr_debug("updating policy for CPU %u\n", cpu);  in cpufreq_update_policy()
1819 		policy.cur = cpufreq_driver->get(cpu);  in cpufreq_update_policy()
1825 				cpufreq_out_of_sync(cpu, data->cur,  in cpufreq_update_policy()
1832 	unlock_policy_rwsem_write(cpu);  in cpufreq_update_policy()
1844 	unsigned int cpu = (unsigned long)hcpu;  in cpufreq_cpu_callback()  local
1847 	dev = get_cpu_device(cpu);  in cpufreq_cpu_callback()
1976 	int cpu;  in cpufreq_core_init()  local
1981 	for_each_possible_cpu(cpu) {  in cpufreq_core_init()
1982 		per_cpu(cpufreq_policy_cpu, cpu) = -1;  in cpufreq_core_init()
1983 		init_rwsem(&per_cpu(cpu_policy_rwsem, cpu));  in cpufreq_core_init()