Lines Matching refs:cpu
101 WARN_ON_ONCE(smp_processor_id() != rdp->cpu); in rcu_nocb_bypass_lock()
120 WARN_ON_ONCE(smp_processor_id() != rdp->cpu); in rcu_nocb_wait_contended()
213 bool rcu_is_nocb_cpu(int cpu) in rcu_is_nocb_cpu() argument
216 return cpumask_test_cpu(cpu, rcu_nocb_mask); in rcu_is_nocb_cpu()
229 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in __wake_nocb_gp()
245 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("DoWake")); in __wake_nocb_gp()
292 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, reason); in wake_nocb_gp_defer()
429 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in rcu_nocb_try_bypass()
444 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in rcu_nocb_try_bypass()
467 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("FirstBQ")); in rcu_nocb_try_bypass()
477 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in rcu_nocb_try_bypass()
481 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in rcu_nocb_try_bypass()
508 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in __call_rcu_nocb_wake()
520 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in __call_rcu_nocb_wake()
546 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("WakeNot")); in __call_rcu_nocb_wake()
550 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("WakeNot")); in __call_rcu_nocb_wake()
610 int __maybe_unused cpu = my_rdp->cpu; in nocb_gp_wait() local
634 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("Check")); in nocb_gp_wait()
656 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in nocb_gp_wait()
683 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in nocb_gp_wait()
717 trace_rcu_nocb_wake(rcu_state.name, cpu, TPS("Poll")); in nocb_gp_wait()
721 trace_rcu_nocb_wake(rcu_state.name, cpu, TPS("Sleep")); in nocb_gp_wait()
724 trace_rcu_nocb_wake(rcu_state.name, cpu, TPS("EndSleep")); in nocb_gp_wait()
836 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("CBSleep")); in nocb_cb_wait()
852 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("WokeEmpty")); in nocb_cb_wait()
896 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("DeferredWake")); in do_nocb_deferred_wakeup_common()
908 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("Timer")); in do_nocb_deferred_wakeup_timer()
978 WARN_ON_ONCE(rdp->cpu != raw_smp_processor_id()); in rcu_nocb_rdp_deoffload()
980 pr_info("De-offloading %d\n", rdp->cpu); in rcu_nocb_rdp_deoffload()
1020 int rcu_nocb_cpu_deoffload(int cpu) in rcu_nocb_cpu_deoffload() argument
1022 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_nocb_cpu_deoffload()
1028 if (cpu_online(cpu)) { in rcu_nocb_cpu_deoffload()
1029 ret = work_on_cpu(cpu, rcu_nocb_rdp_deoffload, rdp); in rcu_nocb_cpu_deoffload()
1031 cpumask_clear_cpu(cpu, rcu_nocb_mask); in rcu_nocb_cpu_deoffload()
1051 WARN_ON_ONCE(rdp->cpu != raw_smp_processor_id()); in rcu_nocb_rdp_offload()
1059 pr_info("Offloading %d\n", rdp->cpu); in rcu_nocb_rdp_offload()
1090 int rcu_nocb_cpu_offload(int cpu) in rcu_nocb_cpu_offload() argument
1092 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_nocb_cpu_offload()
1098 if (cpu_online(cpu)) { in rcu_nocb_cpu_offload()
1099 ret = work_on_cpu(cpu, rcu_nocb_rdp_offload, rdp); in rcu_nocb_cpu_offload()
1101 cpumask_set_cpu(cpu, rcu_nocb_mask); in rcu_nocb_cpu_offload()
1116 int cpu; in rcu_init_nohz() local
1152 for_each_cpu(cpu, rcu_nocb_mask) { in rcu_init_nohz()
1153 rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_init_nohz()
1181 static void rcu_spawn_one_nocb_kthread(int cpu) in rcu_spawn_one_nocb_kthread() argument
1183 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_spawn_one_nocb_kthread()
1191 if (!rcu_is_nocb_cpu(cpu) || rdp->nocb_cb_kthread) in rcu_spawn_one_nocb_kthread()
1198 "rcuog/%d", rdp_gp->cpu); in rcu_spawn_one_nocb_kthread()
1206 "rcuo%c/%d", rcu_state.abbr, cpu); in rcu_spawn_one_nocb_kthread()
1217 static void rcu_spawn_cpu_nocb_kthread(int cpu) in rcu_spawn_cpu_nocb_kthread() argument
1220 rcu_spawn_one_nocb_kthread(cpu); in rcu_spawn_cpu_nocb_kthread()
1231 int cpu; in rcu_spawn_nocb_kthreads() local
1233 for_each_online_cpu(cpu) in rcu_spawn_nocb_kthreads()
1234 rcu_spawn_cpu_nocb_kthread(cpu); in rcu_spawn_nocb_kthreads()
1246 int cpu; in rcu_organize_nocb_kthreads() local
1268 for_each_cpu(cpu, rcu_nocb_mask) { in rcu_organize_nocb_kthreads()
1269 rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_organize_nocb_kthreads()
1270 if (rdp->cpu >= nl) { in rcu_organize_nocb_kthreads()
1273 nl = DIV_ROUND_UP(rdp->cpu + 1, ls) * ls; in rcu_organize_nocb_kthreads()
1283 __func__, cpu); in rcu_organize_nocb_kthreads()
1291 pr_cont(" %d", cpu); in rcu_organize_nocb_kthreads()
1332 rdp->cpu, in show_rcu_nocb_gp_state()
1365 rdp->cpu, rdp->nocb_gp_rdp->cpu, in show_rcu_nocb_state()
1366 rdp->nocb_next_cb_rdp ? rdp->nocb_next_cb_rdp->cpu : -1, in show_rcu_nocb_state()
1484 static void rcu_spawn_cpu_nocb_kthread(int cpu) in rcu_spawn_cpu_nocb_kthread() argument