Lines Matching +full:multi +full:- +full:attr
35 #include <linux/percpu-rwsem.h>
46 * cpuhp_cpu_state - Per cpu hotplug state storage
56 * @done_up: Signal completion to the issuer of the task for cpu-up
57 * @done_down: Signal completion to the issuer of the task for cpu-down
88 STATIC_LOCKDEP_MAP_INIT("cpuhp_state-up", &cpuhp_state_up_map);
90 STATIC_LOCKDEP_MAP_INIT("cpuhp_state-down", &cpuhp_state_down_map);
110 * cpuhp_step - Hotplug state machine step
120 int (*multi)(unsigned int cpu, member
125 int (*multi)(unsigned int cpu, member
146 * @node: For multi-instance, do a single entry callback for install/remove
147 * @lastp: For multi-instance rollback, remember how far we got
161 if (st->fail == state) { in cpuhp_invoke_callback()
162 st->fail = CPUHP_INVALID; in cpuhp_invoke_callback()
164 if (!(bringup ? step->startup.single : step->teardown.single)) in cpuhp_invoke_callback()
167 return -EAGAIN; in cpuhp_invoke_callback()
170 if (!step->multi_instance) { in cpuhp_invoke_callback()
172 cb = bringup ? step->startup.single : step->teardown.single; in cpuhp_invoke_callback()
175 trace_cpuhp_enter(cpu, st->target, state, cb); in cpuhp_invoke_callback()
177 trace_cpuhp_exit(cpu, st->state, state, ret); in cpuhp_invoke_callback()
180 cbm = bringup ? step->startup.multi : step->teardown.multi; in cpuhp_invoke_callback()
187 trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node); in cpuhp_invoke_callback()
189 trace_cpuhp_exit(cpu, st->state, state, ret); in cpuhp_invoke_callback()
195 hlist_for_each(node, &step->list) { in cpuhp_invoke_callback()
199 trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node); in cpuhp_invoke_callback()
201 trace_cpuhp_exit(cpu, st->state, state, ret); in cpuhp_invoke_callback()
216 cbm = !bringup ? step->startup.multi : step->teardown.multi; in cpuhp_invoke_callback()
220 hlist_for_each(node, &step->list) { in cpuhp_invoke_callback()
221 if (!cnt--) in cpuhp_invoke_callback()
224 trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node); in cpuhp_invoke_callback()
226 trace_cpuhp_exit(cpu, st->state, state, ret); in cpuhp_invoke_callback()
247 struct completion *done = bringup ? &st->done_up : &st->done_down; in wait_for_ap_thread()
253 struct completion *done = bringup ? &st->done_up : &st->done_down; in complete_ap_thread()
285 * If set, cpu_up and cpu_down will return -EBUSY and do nothing.
365 cpu_hotplug_disabled--; in __cpu_hotplug_enable()
389 * Architectures that need SMT-specific errata handling during SMT hotplug
459 enum cpuhp_state prev_state = st->state; in cpuhp_set_state()
461 st->rollback = false; in cpuhp_set_state()
462 st->last = NULL; in cpuhp_set_state()
464 st->target = target; in cpuhp_set_state()
465 st->single = false; in cpuhp_set_state()
466 st->bringup = st->state < target; in cpuhp_set_state()
474 st->rollback = true; in cpuhp_reset_state()
477 * If we have st->last we need to undo partial multi_instance of this in cpuhp_reset_state()
480 if (!st->last) { in cpuhp_reset_state()
481 if (st->bringup) in cpuhp_reset_state()
482 st->state--; in cpuhp_reset_state()
484 st->state++; in cpuhp_reset_state()
487 st->target = prev_state; in cpuhp_reset_state()
488 st->bringup = !st->bringup; in cpuhp_reset_state()
494 if (!st->single && st->state == st->target) in __cpuhp_kick_ap()
497 st->result = 0; in __cpuhp_kick_ap()
503 st->should_run = true; in __cpuhp_kick_ap()
504 wake_up_process(st->thread); in __cpuhp_kick_ap()
505 wait_for_ap_thread(st, st->bringup); in __cpuhp_kick_ap()
515 if ((ret = st->result)) { in cpuhp_kick_ap()
530 return -ECANCELED; in bringup_wait_for_ap()
533 kthread_unpark(st->thread); in bringup_wait_for_ap()
543 return -ECANCELED; in bringup_wait_for_ap()
545 if (st->target <= CPUHP_AP_ONLINE_IDLE) in bringup_wait_for_ap()
548 return cpuhp_kick_ap(st, st->target); in bringup_wait_for_ap()
569 /* Arch-specific enabling code. */ in bringup_cpu()
580 struct mm_struct *mm = idle->active_mm; in finish_cpu()
587 idle->active_mm = &init_mm; in finish_cpu()
598 for (st->state--; st->state > st->target; st->state--) in undo_cpu_up()
599 cpuhp_invoke_callback(cpu, st->state, false, NULL, NULL); in undo_cpu_up()
613 return st->state <= CPUHP_BRINGUP_CPU; in can_rollback_cpu()
619 enum cpuhp_state prev_state = st->state; in cpuhp_up_callbacks()
622 while (st->state < target) { in cpuhp_up_callbacks()
623 st->state++; in cpuhp_up_callbacks()
624 ret = cpuhp_invoke_callback(cpu, st->state, true, NULL, NULL); in cpuhp_up_callbacks()
627 st->target = prev_state; in cpuhp_up_callbacks()
643 init_completion(&st->done_up); in cpuhp_create()
644 init_completion(&st->done_down); in cpuhp_create()
651 return st->should_run; in cpuhp_should_run()
662 * - single: runs st->cb_state
663 * - up: runs ++st->state, while st->state < st->target
664 * - down: runs st->state--, while st->state > st->target
671 bool bringup = st->bringup; in cpuhp_thread_fun()
674 if (WARN_ON_ONCE(!st->should_run)) in cpuhp_thread_fun()
678 * ACQUIRE for the cpuhp_should_run() load of ->should_run. Ensures in cpuhp_thread_fun()
679 * that if we see ->should_run we also see the rest of the state. in cpuhp_thread_fun()
691 if (st->single) { in cpuhp_thread_fun()
692 state = st->cb_state; in cpuhp_thread_fun()
693 st->should_run = false; in cpuhp_thread_fun()
696 st->state++; in cpuhp_thread_fun()
697 state = st->state; in cpuhp_thread_fun()
698 st->should_run = (st->state < st->target); in cpuhp_thread_fun()
699 WARN_ON_ONCE(st->state > st->target); in cpuhp_thread_fun()
701 state = st->state; in cpuhp_thread_fun()
702 st->state--; in cpuhp_thread_fun()
703 st->should_run = (st->state > st->target); in cpuhp_thread_fun()
704 WARN_ON_ONCE(st->state < st->target); in cpuhp_thread_fun()
712 st->result = cpuhp_invoke_callback(cpu, state, bringup, st->node, &st->last); in cpuhp_thread_fun()
718 WARN_ON_ONCE(st->result); in cpuhp_thread_fun()
720 st->result = cpuhp_invoke_callback(cpu, state, bringup, st->node, &st->last); in cpuhp_thread_fun()
723 if (st->result) { in cpuhp_thread_fun()
729 WARN_ON_ONCE(st->rollback); in cpuhp_thread_fun()
730 st->should_run = false; in cpuhp_thread_fun()
736 if (!st->should_run) in cpuhp_thread_fun()
761 if (!st->thread) in cpuhp_invoke_ap_callback()
764 st->rollback = false; in cpuhp_invoke_ap_callback()
765 st->last = NULL; in cpuhp_invoke_ap_callback()
767 st->node = node; in cpuhp_invoke_ap_callback()
768 st->bringup = bringup; in cpuhp_invoke_ap_callback()
769 st->cb_state = state; in cpuhp_invoke_ap_callback()
770 st->single = true; in cpuhp_invoke_ap_callback()
777 if ((ret = st->result) && st->last) { in cpuhp_invoke_ap_callback()
778 st->rollback = true; in cpuhp_invoke_ap_callback()
779 st->bringup = !bringup; in cpuhp_invoke_ap_callback()
788 st->node = st->last = NULL; in cpuhp_invoke_ap_callback()
795 enum cpuhp_state prev_state = st->state; in cpuhp_kick_ap_work()
804 trace_cpuhp_enter(cpu, st->target, prev_state, cpuhp_kick_ap_work); in cpuhp_kick_ap_work()
805 ret = cpuhp_kick_ap(st, st->target); in cpuhp_kick_ap_work()
806 trace_cpuhp_exit(cpu, st->state, prev_state, ret); in cpuhp_kick_ap_work()
878 * clear_tasks_mm_cpumask - Safely clear tasks' mm_cpumask for a CPU
883 * trivial, there are various non-obvious corner cases, which this function
896 * their mm mask. -- Peter Zijlstra in clear_tasks_mm_cpumask()
898 * full-fledged tasklist_lock. in clear_tasks_mm_cpumask()
912 arch_clear_mm_cpumask_cpu(cpu, t->mm); in clear_tasks_mm_cpumask()
922 enum cpuhp_state target = max((int)st->target, CPUHP_AP_OFFLINE); in take_cpu_down()
935 WARN_ON(st->state != CPUHP_TEARDOWN_CPU); in take_cpu_down()
936 st->state--; in take_cpu_down()
938 for (; st->state > target; st->state--) { in take_cpu_down()
939 ret = cpuhp_invoke_callback(cpu, st->state, false, NULL, NULL); in take_cpu_down()
961 kthread_park(per_cpu_ptr(&cpuhp_state, cpu)->thread); in takedown_cpu()
977 kthread_unpark(per_cpu_ptr(&cpuhp_state, cpu)->thread); in takedown_cpu()
990 BUG_ON(st->state != CPUHP_AP_IDLE_DEAD); in takedown_cpu()
1015 BUG_ON(st->state != CPUHP_AP_OFFLINE); in cpuhp_report_idle_dead()
1017 st->state = CPUHP_AP_IDLE_DEAD; in cpuhp_report_idle_dead()
1028 for (st->state++; st->state < st->target; st->state++) in undo_cpu_down()
1029 cpuhp_invoke_callback(cpu, st->state, true, NULL, NULL); in undo_cpu_down()
1035 enum cpuhp_state prev_state = st->state; in cpuhp_down_callbacks()
1038 for (; st->state > target; st->state--) { in cpuhp_down_callbacks()
1039 ret = cpuhp_invoke_callback(cpu, st->state, false, NULL, NULL); in cpuhp_down_callbacks()
1041 st->target = prev_state; in cpuhp_down_callbacks()
1042 if (st->state < prev_state) in cpuhp_down_callbacks()
1058 return -EBUSY; in _cpu_down()
1061 return -EINVAL; in _cpu_down()
1065 return -EBUSY; in _cpu_down()
1077 if (st->state > CPUHP_TEARDOWN_CPU) { in _cpu_down()
1078 st->target = max((int)target, CPUHP_TEARDOWN_CPU); in _cpu_down()
1091 if (st->state > CPUHP_TEARDOWN_CPU) in _cpu_down()
1094 st->target = target; in _cpu_down()
1101 if (ret && st->state == CPUHP_TEARDOWN_CPU && st->state < prev_state) { in _cpu_down()
1121 return -EBUSY; in cpu_down_maps_locked()
1136 * cpu_device_down - Bring down a cpu device
1145 return cpu_down(dev->id, CPUHP_OFFLINE); in cpu_device_down()
1181 pr_err("Failed to offline CPU%d - error=%d", in smp_shutdown_nonboot_cpus()
1207 * notify_cpu_starting(cpu) - Invoke the callbacks on the starting CPU
1216 enum cpuhp_state target = min((int)st->target, CPUHP_AP_ONLINE); in notify_cpu_starting()
1221 while (st->state < target) { in notify_cpu_starting()
1222 st->state++; in notify_cpu_starting()
1223 ret = cpuhp_invoke_callback(cpu, st->state, true, NULL, NULL); in notify_cpu_starting()
1250 st->state = CPUHP_AP_ONLINE_IDLE; in cpuhp_online_idle()
1264 ret = -EINVAL; in _cpu_up()
1272 if (st->state >= target) in _cpu_up()
1275 if (st->state == CPUHP_OFFLINE) { in _cpu_up()
1291 if (st->state > CPUHP_BRINGUP_CPU) { in _cpu_up()
1320 pr_err("can't online cpu %d because it is not configured as may-hotadd at boot time\n", in cpu_up()
1325 return -EINVAL; in cpu_up()
1335 err = -EBUSY; in cpu_up()
1339 err = -EPERM; in cpu_up()
1350 * cpu_device_up - Bring up a cpu device
1359 return cpu_up(dev->id, CPUHP_ONLINE); in cpu_device_up()
1375 * bringup_hibernate_cpu - Bring up the CPU that we hibernated on
1390 pr_err("Failed to bring hibernate-CPU up!\n"); in bringup_hibernate_cpu()
1417 if (primary == -1) { in freeze_secondary_cpus()
1427 * We take down all of the non-boot CPUs in one shot to avoid races in freeze_secondary_cpus()
1432 pr_info("Disabling non-boot CPUs ...\n"); in freeze_secondary_cpus()
1439 error = -EBUSY; in freeze_secondary_cpus()
1457 pr_err("Non-boot CPUs are not disabled\n"); in freeze_secondary_cpus()
1488 pr_info("Enabling non-boot CPUs ...\n"); in thaw_secondary_cpus()
1513 return -ENOMEM; in alloc_frozen_cpus()
1626 * On the tear-down path, timers_dead_cpu() must be invoked
1747 return -EINVAL; in cpuhp_cb_check()
1771 return -EINVAL; in cpuhp_reserve_state()
1775 if (!step->name) in cpuhp_reserve_state()
1779 return -ENOSPC; in cpuhp_reserve_state()
1808 if (name && sp->name) in cpuhp_store_callbacks()
1809 return -EBUSY; in cpuhp_store_callbacks()
1811 sp->startup.single = startup; in cpuhp_store_callbacks()
1812 sp->teardown.single = teardown; in cpuhp_store_callbacks()
1813 sp->name = name; in cpuhp_store_callbacks()
1814 sp->multi_instance = multi_instance; in cpuhp_store_callbacks()
1815 INIT_HLIST_HEAD(&sp->list); in cpuhp_store_callbacks()
1821 return cpuhp_get_step(state)->teardown.single; in cpuhp_get_teardown_cb()
1838 if ((bringup && !sp->startup.single) || in cpuhp_issue_call()
1839 (!bringup && !sp->teardown.single)) in cpuhp_issue_call()
1870 int cpustate = st->state; in cpuhp_rollback_install()
1892 if (sp->multi_instance == false) in __cpuhp_state_add_instance_cpuslocked()
1893 return -EINVAL; in __cpuhp_state_add_instance_cpuslocked()
1897 if (!invoke || !sp->startup.multi) in __cpuhp_state_add_instance_cpuslocked()
1906 int cpustate = st->state; in __cpuhp_state_add_instance_cpuslocked()
1913 if (sp->teardown.multi) in __cpuhp_state_add_instance_cpuslocked()
1920 hlist_add_head(node, &sp->list); in __cpuhp_state_add_instance_cpuslocked()
1939 * __cpuhp_setup_state_cpuslocked - Setup the callbacks for an hotplug machine state
1967 return -EINVAL; in __cpuhp_setup_state_cpuslocked()
1989 int cpustate = st->state; in __cpuhp_setup_state_cpuslocked()
2038 if (!sp->multi_instance) in __cpuhp_state_remove_instance()
2039 return -EINVAL; in __cpuhp_state_remove_instance()
2053 int cpustate = st->state; in __cpuhp_state_remove_instance()
2069 * __cpuhp_remove_state_cpuslocked - Remove the callbacks for an hotplug machine state
2088 if (sp->multi_instance) { in __cpuhp_remove_state_cpuslocked()
2089 WARN(!hlist_empty(&sp->list), in __cpuhp_remove_state_cpuslocked()
2105 int cpustate = st->state; in __cpuhp_remove_state_cpuslocked()
2129 dev->offline = true; in cpuhp_offline_cpu_device()
2131 kobject_uevent(&dev->kobj, KOBJ_OFFLINE); in cpuhp_offline_cpu_device()
2138 dev->offline = false; in cpuhp_online_cpu_device()
2140 kobject_uevent(&dev->kobj, KOBJ_ONLINE); in cpuhp_online_cpu_device()
2198 struct device_attribute *attr, char *buf) in show_cpuhp_state() argument
2200 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id); in show_cpuhp_state()
2202 return sprintf(buf, "%d\n", st->state); in show_cpuhp_state()
2207 struct device_attribute *attr, in write_cpuhp_target() argument
2210 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id); in write_cpuhp_target()
2220 return -EINVAL; in write_cpuhp_target()
2223 return -EINVAL; in write_cpuhp_target()
2232 ret = !sp->name || sp->cant_stop ? -EINVAL : 0; in write_cpuhp_target()
2237 if (st->state < target) in write_cpuhp_target()
2238 ret = cpu_up(dev->id, target); in write_cpuhp_target()
2239 else if (st->state > target) in write_cpuhp_target()
2240 ret = cpu_down(dev->id, target); in write_cpuhp_target()
2241 else if (WARN_ON(st->target != target)) in write_cpuhp_target()
2242 st->target = target; in write_cpuhp_target()
2249 struct device_attribute *attr, char *buf) in show_cpuhp_target() argument
2251 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id); in show_cpuhp_target()
2253 return sprintf(buf, "%d\n", st->target); in show_cpuhp_target()
2259 struct device_attribute *attr, in write_cpuhp_fail() argument
2262 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id); in write_cpuhp_fail()
2271 return -EINVAL; in write_cpuhp_fail()
2277 return -EINVAL; in write_cpuhp_fail()
2284 if (!sp->startup.single && !sp->teardown.single) in write_cpuhp_fail()
2285 ret = -EINVAL; in write_cpuhp_fail()
2290 st->fail = fail; in write_cpuhp_fail()
2296 struct device_attribute *attr, char *buf) in show_cpuhp_fail() argument
2298 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id); in show_cpuhp_fail()
2300 return sprintf(buf, "%d\n", st->fail); in show_cpuhp_fail()
2306 &dev_attr_state.attr,
2307 &dev_attr_target.attr,
2308 &dev_attr_fail.attr,
2319 struct device_attribute *attr, char *buf) in show_cpuhp_states() argument
2328 if (sp->name) { in show_cpuhp_states()
2329 cur = sprintf(buf, "%3d: %s\n", i, sp->name); in show_cpuhp_states()
2340 &dev_attr_states.attr,
2353 __store_smt_control(struct device *dev, struct device_attribute *attr, in __store_smt_control() argument
2365 return -EINVAL; in __store_smt_control()
2368 return -EPERM; in __store_smt_control()
2371 return -ENODEV; in __store_smt_control()
2395 __store_smt_control(struct device *dev, struct device_attribute *attr, in __store_smt_control() argument
2398 return -ENODEV; in __store_smt_control()
2411 show_smt_control(struct device *dev, struct device_attribute *attr, char *buf) in show_smt_control() argument
2415 return snprintf(buf, PAGE_SIZE - 2, "%s\n", state); in show_smt_control()
2419 store_smt_control(struct device *dev, struct device_attribute *attr, in store_smt_control() argument
2422 return __store_smt_control(dev, attr, buf, count); in store_smt_control()
2427 show_smt_active(struct device *dev, struct device_attribute *attr, char *buf) in show_smt_active() argument
2429 return snprintf(buf, PAGE_SIZE - 2, "%d\n", sched_smt_active()); in show_smt_active()
2434 &dev_attr_control.attr,
2435 &dev_attr_active.attr,
2447 return sysfs_create_group(&cpu_subsys.dev_root->kobj, in cpu_smt_sysfs_init()
2459 ret = sysfs_create_group(&cpu_subsys.dev_root->kobj, in cpuhp_sysfs_init()
2469 ret = sysfs_create_group(&dev->kobj, &cpuhp_cpu_attr_group); in cpuhp_sysfs_init()
2486 /* cpu_bit_bitmap[0] is empty - so we can back into it */