Lines Matching refs:st
161 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); in cpuhp_invoke_callback() local
167 if (st->fail == state) { in cpuhp_invoke_callback()
168 st->fail = CPUHP_INVALID; in cpuhp_invoke_callback()
181 trace_cpuhp_enter(cpu, st->target, state, cb); in cpuhp_invoke_callback()
183 trace_cpuhp_exit(cpu, st->state, state, ret); in cpuhp_invoke_callback()
193 trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node); in cpuhp_invoke_callback()
195 trace_cpuhp_exit(cpu, st->state, state, ret); in cpuhp_invoke_callback()
205 trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node); in cpuhp_invoke_callback()
207 trace_cpuhp_exit(cpu, st->state, state, ret); in cpuhp_invoke_callback()
230 trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node); in cpuhp_invoke_callback()
232 trace_cpuhp_exit(cpu, st->state, state, ret); in cpuhp_invoke_callback()
251 static inline void wait_for_ap_thread(struct cpuhp_cpu_state *st, bool bringup) in wait_for_ap_thread() argument
253 struct completion *done = bringup ? &st->done_up : &st->done_down; in wait_for_ap_thread()
257 static inline void complete_ap_thread(struct cpuhp_cpu_state *st, bool bringup) in complete_ap_thread() argument
259 struct completion *done = bringup ? &st->done_up : &st->done_down; in complete_ap_thread()
465 cpuhp_set_state(struct cpuhp_cpu_state *st, enum cpuhp_state target) in cpuhp_set_state() argument
467 enum cpuhp_state prev_state = st->state; in cpuhp_set_state()
469 st->rollback = false; in cpuhp_set_state()
470 st->last = NULL; in cpuhp_set_state()
472 st->target = target; in cpuhp_set_state()
473 st->single = false; in cpuhp_set_state()
474 st->bringup = st->state < target; in cpuhp_set_state()
480 cpuhp_reset_state(struct cpuhp_cpu_state *st, enum cpuhp_state prev_state) in cpuhp_reset_state() argument
482 st->rollback = true; in cpuhp_reset_state()
488 if (!st->last) { in cpuhp_reset_state()
489 if (st->bringup) in cpuhp_reset_state()
490 st->state--; in cpuhp_reset_state()
492 st->state++; in cpuhp_reset_state()
495 st->target = prev_state; in cpuhp_reset_state()
496 st->bringup = !st->bringup; in cpuhp_reset_state()
500 static void __cpuhp_kick_ap(struct cpuhp_cpu_state *st) in __cpuhp_kick_ap() argument
502 if (!st->single && st->state == st->target) in __cpuhp_kick_ap()
505 st->result = 0; in __cpuhp_kick_ap()
511 st->should_run = true; in __cpuhp_kick_ap()
512 wake_up_process(st->thread); in __cpuhp_kick_ap()
513 wait_for_ap_thread(st, st->bringup); in __cpuhp_kick_ap()
516 static int cpuhp_kick_ap(struct cpuhp_cpu_state *st, enum cpuhp_state target) in cpuhp_kick_ap() argument
521 prev_state = cpuhp_set_state(st, target); in cpuhp_kick_ap()
522 __cpuhp_kick_ap(st); in cpuhp_kick_ap()
523 if ((ret = st->result)) { in cpuhp_kick_ap()
524 cpuhp_reset_state(st, prev_state); in cpuhp_kick_ap()
525 __cpuhp_kick_ap(st); in cpuhp_kick_ap()
533 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); in bringup_wait_for_ap() local
536 wait_for_ap_thread(st, true); in bringup_wait_for_ap()
541 kthread_unpark(st->thread); in bringup_wait_for_ap()
553 if (st->target <= CPUHP_AP_ONLINE_IDLE) in bringup_wait_for_ap()
556 return cpuhp_kick_ap(st, st->target); in bringup_wait_for_ap()
604 static void undo_cpu_up(unsigned int cpu, struct cpuhp_cpu_state *st) in undo_cpu_up() argument
606 for (st->state--; st->state > st->target; st->state--) in undo_cpu_up()
607 cpuhp_invoke_callback(cpu, st->state, false, NULL, NULL); in undo_cpu_up()
610 static inline bool can_rollback_cpu(struct cpuhp_cpu_state *st) in can_rollback_cpu() argument
621 return st->state <= CPUHP_BRINGUP_CPU; in can_rollback_cpu()
624 static int cpuhp_up_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st, in cpuhp_up_callbacks() argument
627 enum cpuhp_state prev_state = st->state; in cpuhp_up_callbacks()
630 while (st->state < target) { in cpuhp_up_callbacks()
631 st->state++; in cpuhp_up_callbacks()
632 ret = cpuhp_invoke_callback(cpu, st->state, true, NULL, NULL); in cpuhp_up_callbacks()
634 if (can_rollback_cpu(st)) { in cpuhp_up_callbacks()
635 st->target = prev_state; in cpuhp_up_callbacks()
636 undo_cpu_up(cpu, st); in cpuhp_up_callbacks()
649 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); in cpuhp_create() local
651 init_completion(&st->done_up); in cpuhp_create()
652 init_completion(&st->done_down); in cpuhp_create()
657 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state); in cpuhp_should_run() local
659 return st->should_run; in cpuhp_should_run()
678 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state); in cpuhp_thread_fun() local
679 bool bringup = st->bringup; in cpuhp_thread_fun()
682 if (WARN_ON_ONCE(!st->should_run)) in cpuhp_thread_fun()
699 if (st->single) { in cpuhp_thread_fun()
700 state = st->cb_state; in cpuhp_thread_fun()
701 st->should_run = false; in cpuhp_thread_fun()
704 st->state++; in cpuhp_thread_fun()
705 state = st->state; in cpuhp_thread_fun()
706 st->should_run = (st->state < st->target); in cpuhp_thread_fun()
707 WARN_ON_ONCE(st->state > st->target); in cpuhp_thread_fun()
709 state = st->state; in cpuhp_thread_fun()
710 st->state--; in cpuhp_thread_fun()
711 st->should_run = (st->state > st->target); in cpuhp_thread_fun()
712 WARN_ON_ONCE(st->state < st->target); in cpuhp_thread_fun()
720 st->result = cpuhp_invoke_callback(cpu, state, bringup, st->node, &st->last); in cpuhp_thread_fun()
726 WARN_ON_ONCE(st->result); in cpuhp_thread_fun()
728 st->result = cpuhp_invoke_callback(cpu, state, bringup, st->node, &st->last); in cpuhp_thread_fun()
731 if (st->result) { in cpuhp_thread_fun()
737 WARN_ON_ONCE(st->rollback); in cpuhp_thread_fun()
738 st->should_run = false; in cpuhp_thread_fun()
744 if (!st->should_run) in cpuhp_thread_fun()
745 complete_ap_thread(st, bringup); in cpuhp_thread_fun()
753 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); in cpuhp_invoke_ap_callback() local
769 if (!st->thread) in cpuhp_invoke_ap_callback()
772 st->rollback = false; in cpuhp_invoke_ap_callback()
773 st->last = NULL; in cpuhp_invoke_ap_callback()
775 st->node = node; in cpuhp_invoke_ap_callback()
776 st->bringup = bringup; in cpuhp_invoke_ap_callback()
777 st->cb_state = state; in cpuhp_invoke_ap_callback()
778 st->single = true; in cpuhp_invoke_ap_callback()
780 __cpuhp_kick_ap(st); in cpuhp_invoke_ap_callback()
785 if ((ret = st->result) && st->last) { in cpuhp_invoke_ap_callback()
786 st->rollback = true; in cpuhp_invoke_ap_callback()
787 st->bringup = !bringup; in cpuhp_invoke_ap_callback()
789 __cpuhp_kick_ap(st); in cpuhp_invoke_ap_callback()
796 st->node = st->last = NULL; in cpuhp_invoke_ap_callback()
802 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); in cpuhp_kick_ap_work() local
803 enum cpuhp_state prev_state = st->state; in cpuhp_kick_ap_work()
812 trace_cpuhp_enter(cpu, st->target, prev_state, cpuhp_kick_ap_work); in cpuhp_kick_ap_work()
813 ret = cpuhp_kick_ap(st, st->target); in cpuhp_kick_ap_work()
814 trace_cpuhp_exit(cpu, st->state, prev_state, ret); in cpuhp_kick_ap_work()
929 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state); in take_cpu_down() local
930 enum cpuhp_state target = max((int)st->target, CPUHP_AP_OFFLINE); in take_cpu_down()
943 WARN_ON(st->state != CPUHP_TEARDOWN_CPU); in take_cpu_down()
944 st->state--; in take_cpu_down()
946 for (; st->state > target; st->state--) { in take_cpu_down()
947 ret = cpuhp_invoke_callback(cpu, st->state, false, NULL, NULL); in take_cpu_down()
965 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); in takedown_cpu() local
997 wait_for_ap_thread(st, false); in takedown_cpu()
998 BUG_ON(st->state != CPUHP_AP_IDLE_DEAD); in takedown_cpu()
1014 struct cpuhp_cpu_state *st = arg; in cpuhp_complete_idle_dead() local
1016 complete_ap_thread(st, false); in cpuhp_complete_idle_dead()
1021 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state); in cpuhp_report_idle_dead() local
1023 BUG_ON(st->state != CPUHP_AP_OFFLINE); in cpuhp_report_idle_dead()
1025 st->state = CPUHP_AP_IDLE_DEAD; in cpuhp_report_idle_dead()
1031 cpuhp_complete_idle_dead, st, 0); in cpuhp_report_idle_dead()
1034 static void undo_cpu_down(unsigned int cpu, struct cpuhp_cpu_state *st) in undo_cpu_down() argument
1036 for (st->state++; st->state < st->target; st->state++) in undo_cpu_down()
1037 cpuhp_invoke_callback(cpu, st->state, true, NULL, NULL); in undo_cpu_down()
1040 static int cpuhp_down_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st, in cpuhp_down_callbacks() argument
1043 enum cpuhp_state prev_state = st->state; in cpuhp_down_callbacks()
1046 for (; st->state > target; st->state--) { in cpuhp_down_callbacks()
1047 ret = cpuhp_invoke_callback(cpu, st->state, false, NULL, NULL); in cpuhp_down_callbacks()
1049 st->target = prev_state; in cpuhp_down_callbacks()
1050 if (st->state < prev_state) in cpuhp_down_callbacks()
1051 undo_cpu_down(cpu, st); in cpuhp_down_callbacks()
1062 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); in _cpu_down() local
1075 prev_state = cpuhp_set_state(st, target); in _cpu_down()
1080 if (st->state > CPUHP_TEARDOWN_CPU) { in _cpu_down()
1081 st->target = max((int)target, CPUHP_TEARDOWN_CPU); in _cpu_down()
1094 if (st->state > CPUHP_TEARDOWN_CPU) in _cpu_down()
1097 st->target = target; in _cpu_down()
1103 ret = cpuhp_down_callbacks(cpu, st, target); in _cpu_down()
1104 if (ret && st->state == CPUHP_TEARDOWN_CPU && st->state < prev_state) { in _cpu_down()
1105 cpuhp_reset_state(st, prev_state); in _cpu_down()
1106 __cpuhp_kick_ap(st); in _cpu_down()
1313 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); in pause_cpus() local
1315 st->state = CPUHP_AP_ACTIVE - 1; in pause_cpus()
1316 st->target = st->state; in pause_cpus()
1388 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); in resume_cpus() local
1390 st->state = CPUHP_ONLINE; in resume_cpus()
1391 st->target = st->state; in resume_cpus()
1461 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); in notify_cpu_starting() local
1462 enum cpuhp_state target = min((int)st->target, CPUHP_AP_ONLINE); in notify_cpu_starting()
1467 while (st->state < target) { in notify_cpu_starting()
1468 st->state++; in notify_cpu_starting()
1469 ret = cpuhp_invoke_callback(cpu, st->state, true, NULL, NULL); in notify_cpu_starting()
1484 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state); in cpuhp_online_idle() local
1496 st->state = CPUHP_AP_ONLINE_IDLE; in cpuhp_online_idle()
1497 complete_ap_thread(st, true); in cpuhp_online_idle()
1522 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); in _cpu_up() local
1537 if (st->state >= target) in _cpu_up()
1540 if (st->state == CPUHP_OFFLINE) { in _cpu_up()
1551 cpuhp_set_state(st, target); in _cpu_up()
1556 if (st->state > CPUHP_BRINGUP_CPU) { in _cpu_up()
1572 ret = cpuhp_up_callbacks(cpu, st, target); in _cpu_up()
2170 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); in cpuhp_rollback_install() local
2171 int cpustate = st->state; in cpuhp_rollback_install()
2206 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); in __cpuhp_state_add_instance_cpuslocked() local
2207 int cpustate = st->state; in __cpuhp_state_add_instance_cpuslocked()
2289 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); in __cpuhp_setup_state_cpuslocked() local
2290 int cpustate = st->state; in __cpuhp_setup_state_cpuslocked()
2353 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); in __cpuhp_state_remove_instance() local
2354 int cpustate = st->state; in __cpuhp_state_remove_instance()
2405 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); in __cpuhp_remove_state_cpuslocked() local
2406 int cpustate = st->state; in __cpuhp_remove_state_cpuslocked()
2501 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id); in show_cpuhp_state() local
2503 return sprintf(buf, "%d\n", st->state); in show_cpuhp_state()
2511 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id); in write_cpuhp_target() local
2538 if (st->state < target) in write_cpuhp_target()
2540 else if (st->state > target) in write_cpuhp_target()
2542 else if (WARN_ON(st->target != target)) in write_cpuhp_target()
2543 st->target = target; in write_cpuhp_target()
2552 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id); in show_cpuhp_target() local
2554 return sprintf(buf, "%d\n", st->target); in show_cpuhp_target()
2563 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id); in write_cpuhp_fail() local
2591 st->fail = fail; in write_cpuhp_fail()
2599 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id); in show_cpuhp_fail() local
2601 return sprintf(buf, "%d\n", st->fail); in show_cpuhp_fail()