Lines Matching refs:cpu
126 int (*single)(unsigned int cpu);
127 int (*multi)(unsigned int cpu,
131 int (*single)(unsigned int cpu);
132 int (*multi)(unsigned int cpu,
167 static int cpuhp_invoke_callback(unsigned int cpu, enum cpuhp_state state, in cpuhp_invoke_callback() argument
171 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); in cpuhp_invoke_callback()
173 int (*cbm)(unsigned int cpu, struct hlist_node *node); in cpuhp_invoke_callback()
174 int (*cb)(unsigned int cpu); in cpuhp_invoke_callback()
191 trace_cpuhp_enter(cpu, st->target, state, cb); in cpuhp_invoke_callback()
192 ret = cb(cpu); in cpuhp_invoke_callback()
193 trace_cpuhp_exit(cpu, st->state, state, ret); in cpuhp_invoke_callback()
201 trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node); in cpuhp_invoke_callback()
202 ret = cbm(cpu, node); in cpuhp_invoke_callback()
203 trace_cpuhp_exit(cpu, st->state, state, ret); in cpuhp_invoke_callback()
213 trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node); in cpuhp_invoke_callback()
214 ret = cbm(cpu, node); in cpuhp_invoke_callback()
215 trace_cpuhp_exit(cpu, st->state, state, ret); in cpuhp_invoke_callback()
238 trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node); in cpuhp_invoke_callback()
239 ret = cbm(cpu, node); in cpuhp_invoke_callback()
240 trace_cpuhp_exit(cpu, st->state, state, ret); in cpuhp_invoke_callback()
449 static inline bool cpu_smt_allowed(unsigned int cpu) in cpu_smt_allowed() argument
454 if (topology_is_primary_thread(cpu)) in cpu_smt_allowed()
463 return !cpumask_test_cpu(cpu, &cpus_booted_once_mask); in cpu_smt_allowed()
474 static inline bool cpu_smt_allowed(unsigned int cpu) { return true; } in cpu_smt_allowed() argument
478 cpuhp_set_state(int cpu, struct cpuhp_cpu_state *st, enum cpuhp_state target) in cpuhp_set_state() argument
489 if (cpu_dying(cpu) != !bringup) in cpuhp_set_state()
490 set_cpu_dying(cpu, !bringup); in cpuhp_set_state()
496 cpuhp_reset_state(int cpu, struct cpuhp_cpu_state *st, in cpuhp_reset_state() argument
524 if (cpu_dying(cpu) != !bringup) in cpuhp_reset_state()
525 set_cpu_dying(cpu, !bringup); in cpuhp_reset_state()
545 static int cpuhp_kick_ap(int cpu, struct cpuhp_cpu_state *st, in cpuhp_kick_ap() argument
551 prev_state = cpuhp_set_state(cpu, st, target); in cpuhp_kick_ap()
554 cpuhp_reset_state(cpu, st, prev_state); in cpuhp_kick_ap()
561 static int bringup_wait_for_ap(unsigned int cpu) in bringup_wait_for_ap() argument
563 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); in bringup_wait_for_ap()
567 if (WARN_ON_ONCE((!cpu_online(cpu)))) in bringup_wait_for_ap()
580 if (!cpu_smt_allowed(cpu)) in bringup_wait_for_ap()
586 return cpuhp_kick_ap(cpu, st, st->target); in bringup_wait_for_ap()
589 static int bringup_cpu(unsigned int cpu) in bringup_cpu() argument
591 struct task_struct *idle = idle_thread_get(cpu); in bringup_cpu()
608 ret = __cpu_up(cpu, idle); in bringup_cpu()
612 return bringup_wait_for_ap(cpu); in bringup_cpu()
615 static int finish_cpu(unsigned int cpu) in finish_cpu() argument
617 struct task_struct *idle = idle_thread_get(cpu); in finish_cpu()
667 unsigned int cpu, in __cpuhp_invoke_callback_range() argument
678 err = cpuhp_invoke_callback(cpu, state, bringup, NULL, NULL); in __cpuhp_invoke_callback_range()
684 cpu, bringup ? "UP" : "DOWN", in __cpuhp_invoke_callback_range()
698 unsigned int cpu, in cpuhp_invoke_callback_range() argument
702 return __cpuhp_invoke_callback_range(bringup, cpu, st, target, false); in cpuhp_invoke_callback_range()
706 unsigned int cpu, in cpuhp_invoke_callback_range_nofail() argument
710 __cpuhp_invoke_callback_range(bringup, cpu, st, target, true); in cpuhp_invoke_callback_range_nofail()
727 static int cpuhp_up_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st, in cpuhp_up_callbacks() argument
733 ret = cpuhp_invoke_callback_range(true, cpu, st, target); in cpuhp_up_callbacks()
736 ret, cpu, cpuhp_get_step(st->state)->name, in cpuhp_up_callbacks()
739 cpuhp_reset_state(cpu, st, prev_state); in cpuhp_up_callbacks()
741 WARN_ON(cpuhp_invoke_callback_range(false, cpu, st, in cpuhp_up_callbacks()
750 static void cpuhp_create(unsigned int cpu) in cpuhp_create() argument
752 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); in cpuhp_create()
758 static int cpuhp_should_run(unsigned int cpu) in cpuhp_should_run() argument
779 static void cpuhp_thread_fun(unsigned int cpu) in cpuhp_thread_fun() argument
815 st->result = cpuhp_invoke_callback(cpu, state, bringup, st->node, &st->last); in cpuhp_thread_fun()
823 st->result = cpuhp_invoke_callback(cpu, state, bringup, st->node, &st->last); in cpuhp_thread_fun()
846 cpuhp_invoke_ap_callback(int cpu, enum cpuhp_state state, bool bringup, in cpuhp_invoke_ap_callback() argument
849 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); in cpuhp_invoke_ap_callback()
852 if (!cpu_online(cpu)) in cpuhp_invoke_ap_callback()
866 return cpuhp_invoke_callback(cpu, state, bringup, node, NULL); in cpuhp_invoke_ap_callback()
896 static int cpuhp_kick_ap_work(unsigned int cpu) in cpuhp_kick_ap_work() argument
898 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); in cpuhp_kick_ap_work()
908 trace_cpuhp_enter(cpu, st->target, prev_state, cpuhp_kick_ap_work); in cpuhp_kick_ap_work()
909 ret = cpuhp_kick_ap(cpu, st, st->target); in cpuhp_kick_ap_work()
910 trace_cpuhp_exit(cpu, st->state, prev_state, ret); in cpuhp_kick_ap_work()
978 #define arch_clear_mm_cpumask_cpu(cpu, mm) cpumask_clear_cpu(cpu, mm_cpumask(mm)) argument
993 void clear_tasks_mm_cpumask(int cpu) in clear_tasks_mm_cpumask() argument
1004 WARN_ON(cpu_online(cpu)); in clear_tasks_mm_cpumask()
1016 arch_clear_mm_cpumask_cpu(cpu, t->mm); in clear_tasks_mm_cpumask()
1027 int err, cpu = smp_processor_id(); in take_cpu_down() local
1043 cpuhp_invoke_callback_range_nofail(false, cpu, st, target); in take_cpu_down()
1048 tick_offline_cpu(cpu); in take_cpu_down()
1050 stop_machine_park(cpu); in take_cpu_down()
1054 static int takedown_cpu(unsigned int cpu) in takedown_cpu() argument
1056 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); in takedown_cpu()
1071 err = stop_machine_cpuslocked(take_cpu_down, NULL, cpumask_of(cpu)); in takedown_cpu()
1079 BUG_ON(cpu_online(cpu)); in takedown_cpu()
1094 hotplug_cpu__broadcast_tick_pull(cpu); in takedown_cpu()
1096 __cpu_die(cpu); in takedown_cpu()
1098 tick_cleanup_dead_cpu(cpu); in takedown_cpu()
1099 rcutree_migrate_callbacks(cpu); in takedown_cpu()
1125 static int cpuhp_down_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st, in cpuhp_down_callbacks() argument
1131 ret = cpuhp_invoke_callback_range(false, cpu, st, target); in cpuhp_down_callbacks()
1134 ret, cpu, cpuhp_get_step(st->state)->name, in cpuhp_down_callbacks()
1137 cpuhp_reset_state(cpu, st, prev_state); in cpuhp_down_callbacks()
1140 WARN_ON(cpuhp_invoke_callback_range(true, cpu, st, in cpuhp_down_callbacks()
1148 static int __ref _cpu_down(unsigned int cpu, int tasks_frozen, in _cpu_down() argument
1151 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); in _cpu_down()
1157 if (!cpu_present(cpu)) in _cpu_down()
1164 prev_state = cpuhp_set_state(cpu, st, target); in _cpu_down()
1171 ret = cpuhp_kick_ap_work(cpu); in _cpu_down()
1192 ret = cpuhp_down_callbacks(cpu, st, target); in _cpu_down()
1195 cpuhp_reset_state(cpu, st, prev_state); in _cpu_down()
1198 WARN(1, "DEAD callback error for CPU%d", cpu); in _cpu_down()
1214 static int cpu_down_maps_locked(unsigned int cpu, enum cpuhp_state target) in cpu_down_maps_locked() argument
1218 return _cpu_down(cpu, 0, target); in cpu_down_maps_locked()
1221 static int cpu_down(unsigned int cpu, enum cpuhp_state target) in cpu_down() argument
1226 err = cpu_down_maps_locked(cpu, target); in cpu_down()
1246 int remove_cpu(unsigned int cpu) in remove_cpu() argument
1251 ret = device_offline(get_cpu_device(cpu)); in remove_cpu()
1260 unsigned int cpu; in smp_shutdown_nonboot_cpus() local
1273 for_each_online_cpu(cpu) { in smp_shutdown_nonboot_cpus()
1274 if (cpu == primary_cpu) in smp_shutdown_nonboot_cpus()
1277 error = cpu_down_maps_locked(cpu, CPUHP_OFFLINE); in smp_shutdown_nonboot_cpus()
1280 cpu, error); in smp_shutdown_nonboot_cpus()
1311 void notify_cpu_starting(unsigned int cpu) in notify_cpu_starting() argument
1313 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); in notify_cpu_starting()
1316 rcu_cpu_starting(cpu); /* Enables RCU usage on this CPU. */ in notify_cpu_starting()
1317 cpumask_set_cpu(cpu, &cpus_booted_once_mask); in notify_cpu_starting()
1322 cpuhp_invoke_callback_range_nofail(true, cpu, st, target); in notify_cpu_starting()
1368 static int _cpu_up(unsigned int cpu, int tasks_frozen, enum cpuhp_state target) in _cpu_up() argument
1370 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); in _cpu_up()
1376 if (!cpu_present(cpu)) { in _cpu_up()
1390 idle = idle_thread_get(cpu); in _cpu_up()
1399 cpuhp_set_state(cpu, st, target); in _cpu_up()
1405 ret = cpuhp_kick_ap_work(cpu); in _cpu_up()
1420 ret = cpuhp_up_callbacks(cpu, st, target); in _cpu_up()
1428 static int cpu_up(unsigned int cpu, enum cpuhp_state target) in cpu_up() argument
1433 if (!cpu_possible(cpu)) { in cpu_up()
1435 cpu); in cpu_up()
1454 err = try_online_node(cpu_to_node(cpu)); in cpu_up()
1464 if (!cpu_smt_allowed(cpu)) { in cpu_up()
1469 err = _cpu_up(cpu, 0, target); in cpu_up()
1498 int add_cpu(unsigned int cpu) in add_cpu() argument
1503 ret = device_online(get_cpu_device(cpu)); in add_cpu()
1537 unsigned int cpu; in bringup_nonboot_cpus() local
1539 for_each_present_cpu(cpu) { in bringup_nonboot_cpus()
1542 if (!cpu_online(cpu)) in bringup_nonboot_cpus()
1543 cpu_up(cpu, CPUHP_ONLINE); in bringup_nonboot_cpus()
1552 int cpu, error = 0; in freeze_secondary_cpus() local
1571 for_each_online_cpu(cpu) { in freeze_secondary_cpus()
1572 if (cpu == primary) in freeze_secondary_cpus()
1581 trace_suspend_resume(TPS("CPU_OFF"), cpu, true); in freeze_secondary_cpus()
1582 error = _cpu_down(cpu, 1, CPUHP_OFFLINE); in freeze_secondary_cpus()
1583 trace_suspend_resume(TPS("CPU_OFF"), cpu, false); in freeze_secondary_cpus()
1585 cpumask_set_cpu(cpu, frozen_cpus); in freeze_secondary_cpus()
1587 pr_err("Error taking CPU%d down: %d\n", cpu, error); in freeze_secondary_cpus()
1618 int cpu, error; in thaw_secondary_cpus() local
1630 for_each_cpu(cpu, frozen_cpus) { in thaw_secondary_cpus()
1631 trace_suspend_resume(TPS("CPU_ON"), cpu, true); in thaw_secondary_cpus()
1632 error = _cpu_up(cpu, 1, CPUHP_ONLINE); in thaw_secondary_cpus()
1633 trace_suspend_resume(TPS("CPU_ON"), cpu, false); in thaw_secondary_cpus()
1635 pr_info("CPU%d is up\n", cpu); in thaw_secondary_cpus()
1638 pr_warn("Error taking CPU%d up: %d\n", cpu, error); in thaw_secondary_cpus()
1708 static int random_and_perf_prepare_fusion(unsigned int cpu) in random_and_perf_prepare_fusion() argument
1710 int (*fn)(unsigned int cpu); in random_and_perf_prepare_fusion()
1713 fn(cpu); in random_and_perf_prepare_fusion()
1716 fn(cpu); in random_and_perf_prepare_fusion()
1719 static int random_and_workqueue_online_fusion(unsigned int cpu) in random_and_workqueue_online_fusion() argument
1721 int (*fn)(unsigned int cpu); in random_and_workqueue_online_fusion()
1724 fn(cpu); in random_and_workqueue_online_fusion()
1727 fn(cpu); in random_and_workqueue_online_fusion()
1942 int (*startup)(unsigned int cpu), in cpuhp_store_callbacks() argument
1943 int (*teardown)(unsigned int cpu), in cpuhp_store_callbacks() argument
1987 static int cpuhp_issue_call(int cpu, enum cpuhp_state state, bool bringup, in cpuhp_issue_call() argument
2005 ret = cpuhp_invoke_ap_callback(cpu, state, bringup, node); in cpuhp_issue_call()
2007 ret = cpuhp_invoke_callback(cpu, state, bringup, node, NULL); in cpuhp_issue_call()
2009 ret = cpuhp_invoke_callback(cpu, state, bringup, node, NULL); in cpuhp_issue_call()
2023 int cpu; in cpuhp_rollback_install() local
2026 for_each_present_cpu(cpu) { in cpuhp_rollback_install()
2027 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); in cpuhp_rollback_install()
2030 if (cpu >= failedcpu) in cpuhp_rollback_install()
2035 cpuhp_issue_call(cpu, state, false, node); in cpuhp_rollback_install()
2044 int cpu; in __cpuhp_state_add_instance_cpuslocked() local
2062 for_each_present_cpu(cpu) { in __cpuhp_state_add_instance_cpuslocked()
2063 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); in __cpuhp_state_add_instance_cpuslocked()
2069 ret = cpuhp_issue_call(cpu, state, true, node); in __cpuhp_state_add_instance_cpuslocked()
2072 cpuhp_rollback_install(cpu, state, node); in __cpuhp_state_add_instance_cpuslocked()
2116 int (*startup)(unsigned int cpu), in __cpuhp_setup_state_cpuslocked() argument
2117 int (*teardown)(unsigned int cpu), in __cpuhp_setup_state_cpuslocked() argument
2120 int cpu, ret = 0; in __cpuhp_setup_state_cpuslocked() local
2146 for_each_present_cpu(cpu) { in __cpuhp_setup_state_cpuslocked()
2147 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); in __cpuhp_setup_state_cpuslocked()
2153 ret = cpuhp_issue_call(cpu, state, true, NULL); in __cpuhp_setup_state_cpuslocked()
2156 cpuhp_rollback_install(cpu, state, NULL); in __cpuhp_setup_state_cpuslocked()
2175 int (*startup)(unsigned int cpu), in __cpuhp_setup_state() argument
2176 int (*teardown)(unsigned int cpu), in __cpuhp_setup_state() argument
2193 int cpu; in __cpuhp_state_remove_instance() local
2210 for_each_present_cpu(cpu) { in __cpuhp_state_remove_instance()
2211 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); in __cpuhp_state_remove_instance()
2215 cpuhp_issue_call(cpu, state, false, node); in __cpuhp_state_remove_instance()
2240 int cpu; in __cpuhp_remove_state_cpuslocked() local
2262 for_each_present_cpu(cpu) { in __cpuhp_remove_state_cpuslocked()
2263 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); in __cpuhp_remove_state_cpuslocked()
2267 cpuhp_issue_call(cpu, state, false, NULL); in __cpuhp_remove_state_cpuslocked()
2284 static void cpuhp_offline_cpu_device(unsigned int cpu) in cpuhp_offline_cpu_device() argument
2286 struct device *dev = get_cpu_device(cpu); in cpuhp_offline_cpu_device()
2293 static void cpuhp_online_cpu_device(unsigned int cpu) in cpuhp_online_cpu_device() argument
2295 struct device *dev = get_cpu_device(cpu); in cpuhp_online_cpu_device()
2304 int cpu, ret = 0; in cpuhp_smt_disable() local
2307 for_each_online_cpu(cpu) { in cpuhp_smt_disable()
2308 if (topology_is_primary_thread(cpu)) in cpuhp_smt_disable()
2310 ret = cpu_down_maps_locked(cpu, CPUHP_OFFLINE); in cpuhp_smt_disable()
2326 cpuhp_offline_cpu_device(cpu); in cpuhp_smt_disable()
2336 int cpu, ret = 0; in cpuhp_smt_enable() local
2340 for_each_present_cpu(cpu) { in cpuhp_smt_enable()
2342 if (cpu_online(cpu) || !node_online(cpu_to_node(cpu))) in cpuhp_smt_enable()
2344 ret = _cpu_up(cpu, 0, CPUHP_ONLINE); in cpuhp_smt_enable()
2348 cpuhp_online_cpu_device(cpu); in cpuhp_smt_enable()
2622 int cpu, ret; in cpuhp_sysfs_init() local
2633 for_each_possible_cpu(cpu) { in cpuhp_sysfs_init()
2634 struct device *dev = get_cpu_device(cpu); in cpuhp_sysfs_init()
2713 void set_cpu_online(unsigned int cpu, bool online) in set_cpu_online() argument
2726 if (!cpumask_test_and_set_cpu(cpu, &__cpu_online_mask)) in set_cpu_online()
2729 if (cpumask_test_and_clear_cpu(cpu, &__cpu_online_mask)) in set_cpu_online()
2739 int cpu = smp_processor_id(); in boot_cpu_init() local
2742 set_cpu_online(cpu, true); in boot_cpu_init()
2743 set_cpu_active(cpu, true); in boot_cpu_init()
2744 set_cpu_present(cpu, true); in boot_cpu_init()
2745 set_cpu_possible(cpu, true); in boot_cpu_init()
2748 __boot_cpu_id = cpu; in boot_cpu_init()