Lines Matching refs:st
157 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); in cpuhp_invoke_callback() local
163 if (st->fail == state) { in cpuhp_invoke_callback()
164 st->fail = CPUHP_INVALID; in cpuhp_invoke_callback()
179 trace_cpuhp_enter(cpu, st->target, state, cb); in cpuhp_invoke_callback()
181 trace_cpuhp_exit(cpu, st->state, state, ret); in cpuhp_invoke_callback()
192 trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node); in cpuhp_invoke_callback()
194 trace_cpuhp_exit(cpu, st->state, state, ret); in cpuhp_invoke_callback()
206 trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node); in cpuhp_invoke_callback()
208 trace_cpuhp_exit(cpu, st->state, state, ret); in cpuhp_invoke_callback()
236 trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node); in cpuhp_invoke_callback()
238 trace_cpuhp_exit(cpu, st->state, state, ret); in cpuhp_invoke_callback()
257 static inline void wait_for_ap_thread(struct cpuhp_cpu_state *st, bool bringup) in wait_for_ap_thread() argument
259 struct completion *done = bringup ? &st->done_up : &st->done_down; in wait_for_ap_thread()
263 static inline void complete_ap_thread(struct cpuhp_cpu_state *st, bool bringup) in complete_ap_thread() argument
265 struct completion *done = bringup ? &st->done_up : &st->done_down; in complete_ap_thread()
480 static inline enum cpuhp_state cpuhp_set_state(struct cpuhp_cpu_state *st, enum cpuhp_state target) in cpuhp_set_state() argument
482 enum cpuhp_state prev_state = st->state; in cpuhp_set_state()
484 st->rollback = false; in cpuhp_set_state()
485 st->last = NULL; in cpuhp_set_state()
487 st->target = target; in cpuhp_set_state()
488 st->single = false; in cpuhp_set_state()
489 st->bringup = st->state < target; in cpuhp_set_state()
494 static inline void cpuhp_reset_state(struct cpuhp_cpu_state *st, enum cpuhp_state prev_state) in cpuhp_reset_state() argument
496 st->rollback = true; in cpuhp_reset_state()
502 if (!st->last) { in cpuhp_reset_state()
503 if (st->bringup) { in cpuhp_reset_state()
504 st->state--; in cpuhp_reset_state()
506 st->state++; in cpuhp_reset_state()
510 st->target = prev_state; in cpuhp_reset_state()
511 st->bringup = !st->bringup; in cpuhp_reset_state()
515 static void _cpuhp_kick_ap(struct cpuhp_cpu_state *st) in _cpuhp_kick_ap() argument
517 if (!st->single && st->state == st->target) { in _cpuhp_kick_ap()
521 st->result = 0; in _cpuhp_kick_ap()
527 st->should_run = true; in _cpuhp_kick_ap()
528 wake_up_process(st->thread); in _cpuhp_kick_ap()
529 wait_for_ap_thread(st, st->bringup); in _cpuhp_kick_ap()
532 static int cpuhp_kick_ap(struct cpuhp_cpu_state *st, enum cpuhp_state target) in cpuhp_kick_ap() argument
537 prev_state = cpuhp_set_state(st, target); in cpuhp_kick_ap()
538 _cpuhp_kick_ap(st); in cpuhp_kick_ap()
539 if ((ret = st->result)) { in cpuhp_kick_ap()
540 cpuhp_reset_state(st, prev_state); in cpuhp_kick_ap()
541 _cpuhp_kick_ap(st); in cpuhp_kick_ap()
549 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); in bringup_wait_for_ap() local
552 wait_for_ap_thread(st, true); in bringup_wait_for_ap()
558 kthread_unpark(st->thread); in bringup_wait_for_ap()
571 if (st->target <= CPUHP_AP_ONLINE_IDLE) { in bringup_wait_for_ap()
575 return cpuhp_kick_ap(st, st->target); in bringup_wait_for_ap()
625 static void undo_cpu_up(unsigned int cpu, struct cpuhp_cpu_state *st) in undo_cpu_up() argument
627 for (st->state--; st->state > st->target; st->state--) { in undo_cpu_up()
628 cpuhp_invoke_callback(cpu, st->state, false, NULL, NULL); in undo_cpu_up()
632 static inline bool can_rollback_cpu(struct cpuhp_cpu_state *st) in can_rollback_cpu() argument
644 return st->state <= CPUHP_BRINGUP_CPU; in can_rollback_cpu()
647 static int cpuhp_up_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st, enum cpuhp_state target) in cpuhp_up_callbacks() argument
649 enum cpuhp_state prev_state = st->state; in cpuhp_up_callbacks()
652 while (st->state < target) { in cpuhp_up_callbacks()
653 st->state++; in cpuhp_up_callbacks()
654 ret = cpuhp_invoke_callback(cpu, st->state, true, NULL, NULL); in cpuhp_up_callbacks()
656 if (can_rollback_cpu(st)) { in cpuhp_up_callbacks()
657 st->target = prev_state; in cpuhp_up_callbacks()
658 undo_cpu_up(cpu, st); in cpuhp_up_callbacks()
671 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); in cpuhp_create() local
673 init_completion(&st->done_up); in cpuhp_create()
674 init_completion(&st->done_down); in cpuhp_create()
679 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state); in cpuhp_should_run() local
681 return st->should_run; in cpuhp_should_run()
700 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state); in cpuhp_thread_fun() local
701 bool bringup = st->bringup; in cpuhp_thread_fun()
704 if (WARN_ON_ONCE(!st->should_run)) { in cpuhp_thread_fun()
722 if (st->single) { in cpuhp_thread_fun()
723 state = st->cb_state; in cpuhp_thread_fun()
724 st->should_run = false; in cpuhp_thread_fun()
727 st->state++; in cpuhp_thread_fun()
728 state = st->state; in cpuhp_thread_fun()
729 st->should_run = (st->state < st->target); in cpuhp_thread_fun()
730 WARN_ON_ONCE(st->state > st->target); in cpuhp_thread_fun()
732 state = st->state; in cpuhp_thread_fun()
733 st->state--; in cpuhp_thread_fun()
734 st->should_run = (st->state > st->target); in cpuhp_thread_fun()
735 WARN_ON_ONCE(st->state < st->target); in cpuhp_thread_fun()
743 st->result = cpuhp_invoke_callback(cpu, state, bringup, st->node, &st->last); in cpuhp_thread_fun()
749 WARN_ON_ONCE(st->result); in cpuhp_thread_fun()
751 st->result = cpuhp_invoke_callback(cpu, state, bringup, st->node, &st->last); in cpuhp_thread_fun()
754 if (st->result) { in cpuhp_thread_fun()
760 WARN_ON_ONCE(st->rollback); in cpuhp_thread_fun()
761 st->should_run = false; in cpuhp_thread_fun()
767 if (!st->should_run) { in cpuhp_thread_fun()
768 complete_ap_thread(st, bringup); in cpuhp_thread_fun()
775 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); in cpuhp_invoke_ap_callback() local
792 if (!st->thread) { in cpuhp_invoke_ap_callback()
796 st->rollback = false; in cpuhp_invoke_ap_callback()
797 st->last = NULL; in cpuhp_invoke_ap_callback()
799 st->node = node; in cpuhp_invoke_ap_callback()
800 st->bringup = bringup; in cpuhp_invoke_ap_callback()
801 st->cb_state = state; in cpuhp_invoke_ap_callback()
802 st->single = true; in cpuhp_invoke_ap_callback()
804 _cpuhp_kick_ap(st); in cpuhp_invoke_ap_callback()
809 if ((ret = st->result) && st->last) { in cpuhp_invoke_ap_callback()
810 st->rollback = true; in cpuhp_invoke_ap_callback()
811 st->bringup = !bringup; in cpuhp_invoke_ap_callback()
813 _cpuhp_kick_ap(st); in cpuhp_invoke_ap_callback()
820 st->node = st->last = NULL; in cpuhp_invoke_ap_callback()
826 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); in cpuhp_kick_ap_work() local
827 enum cpuhp_state prev_state = st->state; in cpuhp_kick_ap_work()
836 trace_cpuhp_enter(cpu, st->target, prev_state, cpuhp_kick_ap_work); in cpuhp_kick_ap_work()
837 ret = cpuhp_kick_ap(st, st->target); in cpuhp_kick_ap_work()
838 trace_cpuhp_exit(cpu, st->state, prev_state, ret); in cpuhp_kick_ap_work()
956 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state); in take_cpu_down() local
957 enum cpuhp_state target = max((int)st->target, CPUHP_AP_OFFLINE); in take_cpu_down()
971 WARN_ON(st->state != CPUHP_TEARDOWN_CPU); in take_cpu_down()
972 st->state--; in take_cpu_down()
974 for (; st->state > target; st->state--) { in take_cpu_down()
975 ret = cpuhp_invoke_callback(cpu, st->state, false, NULL, NULL); in take_cpu_down()
993 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); in takedown_cpu() local
1025 wait_for_ap_thread(st, false); in takedown_cpu()
1026 BUG_ON(st->state != CPUHP_AP_IDLE_DEAD); in takedown_cpu()
1042 struct cpuhp_cpu_state *st = arg; in cpuhp_complete_idle_dead() local
1044 complete_ap_thread(st, false); in cpuhp_complete_idle_dead()
1049 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state); in cpuhp_report_idle_dead() local
1051 BUG_ON(st->state != CPUHP_AP_OFFLINE); in cpuhp_report_idle_dead()
1053 st->state = CPUHP_AP_IDLE_DEAD; in cpuhp_report_idle_dead()
1058 smp_call_function_single(cpumask_first(cpu_online_mask), cpuhp_complete_idle_dead, st, 0); in cpuhp_report_idle_dead()
1061 static void undo_cpu_down(unsigned int cpu, struct cpuhp_cpu_state *st) in undo_cpu_down() argument
1063 for (st->state++; st->state < st->target; st->state++) { in undo_cpu_down()
1064 cpuhp_invoke_callback(cpu, st->state, true, NULL, NULL); in undo_cpu_down()
1068 static int cpuhp_down_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st, enum cpuhp_state targ… in cpuhp_down_callbacks() argument
1070 enum cpuhp_state prev_state = st->state; in cpuhp_down_callbacks()
1073 for (; st->state > target; st->state--) { in cpuhp_down_callbacks()
1074 ret = cpuhp_invoke_callback(cpu, st->state, false, NULL, NULL); in cpuhp_down_callbacks()
1076 st->target = prev_state; in cpuhp_down_callbacks()
1077 if (st->state < prev_state) { in cpuhp_down_callbacks()
1078 undo_cpu_down(cpu, st); in cpuhp_down_callbacks()
1089 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); in _cpu_down() local
1110 prev_state = cpuhp_set_state(st, target); in _cpu_down()
1115 if (st->state > CPUHP_TEARDOWN_CPU) { in _cpu_down()
1116 st->target = max((int)target, CPUHP_TEARDOWN_CPU); in _cpu_down()
1130 if (st->state > CPUHP_TEARDOWN_CPU) { in _cpu_down()
1134 st->target = target; in _cpu_down()
1140 ret = cpuhp_down_callbacks(cpu, st, target); in _cpu_down()
1141 if (ret && st->state == CPUHP_TEARDOWN_CPU && st->state < prev_state) { in _cpu_down()
1142 cpuhp_reset_state(st, prev_state); in _cpu_down()
1143 _cpuhp_kick_ap(st); in _cpu_down()
1258 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); in notify_cpu_starting() local
1259 enum cpuhp_state target = min((int)st->target, CPUHP_AP_ONLINE); in notify_cpu_starting()
1264 while (st->state < target) { in notify_cpu_starting()
1265 st->state++; in notify_cpu_starting()
1266 ret = cpuhp_invoke_callback(cpu, st->state, true, NULL, NULL); in notify_cpu_starting()
1281 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state); in cpuhp_online_idle() local
1294 st->state = CPUHP_AP_ONLINE_IDLE; in cpuhp_online_idle()
1295 complete_ap_thread(st, true); in cpuhp_online_idle()
1301 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); in _cpu_up() local
1316 if (st->state >= target) { in _cpu_up()
1320 if (st->state == CPUHP_OFFLINE) { in _cpu_up()
1331 cpuhp_set_state(st, target); in _cpu_up()
1336 if (st->state > CPUHP_BRINGUP_CPU) { in _cpu_up()
1353 ret = cpuhp_up_callbacks(cpu, st, target); in _cpu_up()
1950 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); in cpuhp_rollback_install() local
1951 int cpustate = st->state; in cpuhp_rollback_install()
1989 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); in __cpuhp_state_add_instance_cpuslocked() local
1990 int cpustate = st->state; in __cpuhp_state_add_instance_cpuslocked()
2073 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); in __cpuhp_setup_state_cpuslocked() local
2074 int cpustate = st->state; in __cpuhp_setup_state_cpuslocked()
2138 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); in __cpuhp_state_remove_instance() local
2139 int cpustate = st->state; in __cpuhp_state_remove_instance()
2191 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); in __cpuhp_remove_state_cpuslocked() local
2192 int cpustate = st->state; in __cpuhp_remove_state_cpuslocked()
2294 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id); in show_cpuhp_state() local
2296 return sprintf(buf, "%d\n", st->state); in show_cpuhp_state()
2302 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id); in write_cpuhp_target() local
2334 if (st->state < target) { in write_cpuhp_target()
2346 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id); in show_cpuhp_target() local
2348 return sprintf(buf, "%d\n", st->target); in show_cpuhp_target()
2354 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id); in write_cpuhp_fail() local
2387 st->fail = fail; in write_cpuhp_fail()
2394 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id); in show_cpuhp_fail() local
2396 return sprintf(buf, "%d\n", st->fail); in show_cpuhp_fail()