Lines Matching refs:cpu
104 WARN_ON_ONCE(smp_processor_id() != rdp->cpu); in rcu_nocb_bypass_lock()
123 WARN_ON_ONCE(smp_processor_id() != rdp->cpu); in rcu_nocb_wait_contended()
224 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in __wake_nocb_gp()
240 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("DoWake")); in __wake_nocb_gp()
316 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, reason); in wake_nocb_gp_defer()
473 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in rcu_nocb_try_bypass()
492 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in rcu_nocb_try_bypass()
524 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("FirstBQ")); in rcu_nocb_try_bypass()
541 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in rcu_nocb_try_bypass()
545 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in rcu_nocb_try_bypass()
574 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in __call_rcu_nocb_wake()
593 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in __call_rcu_nocb_wake()
619 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("WakeNot")); in __call_rcu_nocb_wake()
623 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("WakeNot")); in __call_rcu_nocb_wake()
665 static void nocb_gp_sleep(struct rcu_data *my_rdp, int cpu) in nocb_gp_sleep() argument
667 trace_rcu_nocb_wake(rcu_state.name, cpu, TPS("Sleep")); in nocb_gp_sleep()
670 trace_rcu_nocb_wake(rcu_state.name, cpu, TPS("EndSleep")); in nocb_gp_sleep()
680 int __maybe_unused cpu = my_rdp->cpu; in nocb_gp_wait() local
719 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("Check")); in nocb_gp_wait()
746 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in nocb_gp_wait()
776 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in nocb_gp_wait()
816 trace_rcu_nocb_wake(rcu_state.name, cpu, TPS("Poll")); in nocb_gp_wait()
823 nocb_gp_sleep(my_rdp, cpu); in nocb_gp_wait()
829 nocb_gp_sleep(my_rdp, cpu); in nocb_gp_wait()
939 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("WokeEmpty")); in nocb_cb_wait()
988 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("CBSleep")); in nocb_cb_wait()
1037 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("DeferredWake")); in do_nocb_deferred_wakeup_common()
1049 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("Timer")); in do_nocb_deferred_wakeup_timer()
1124 WARN_ON_ONCE((rdp->cpu != raw_smp_processor_id()) && cpu_online(rdp->cpu)); in rcu_nocb_rdp_deoffload()
1126 pr_info("De-offloading %d\n", rdp->cpu); in rcu_nocb_rdp_deoffload()
1206 int rcu_nocb_cpu_deoffload(int cpu) in rcu_nocb_cpu_deoffload() argument
1208 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_nocb_cpu_deoffload()
1214 if (cpu_online(cpu)) { in rcu_nocb_cpu_deoffload()
1215 ret = work_on_cpu(cpu, rcu_nocb_rdp_deoffload, rdp); in rcu_nocb_cpu_deoffload()
1217 cpumask_clear_cpu(cpu, rcu_nocb_mask); in rcu_nocb_cpu_deoffload()
1219 pr_info("NOCB: Cannot CB-deoffload offline CPU %d\n", rdp->cpu); in rcu_nocb_cpu_deoffload()
1238 WARN_ON_ONCE(rdp->cpu != raw_smp_processor_id()); in rcu_nocb_rdp_offload()
1249 pr_info("Offloading %d\n", rdp->cpu); in rcu_nocb_rdp_offload()
1291 int rcu_nocb_cpu_offload(int cpu) in rcu_nocb_cpu_offload() argument
1293 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_nocb_cpu_offload()
1299 if (cpu_online(cpu)) { in rcu_nocb_cpu_offload()
1300 ret = work_on_cpu(cpu, rcu_nocb_rdp_offload, rdp); in rcu_nocb_cpu_offload()
1302 cpumask_set_cpu(cpu, rcu_nocb_mask); in rcu_nocb_cpu_offload()
1304 pr_info("NOCB: Cannot CB-offload offline CPU %d\n", rdp->cpu); in rcu_nocb_cpu_offload()
1318 int cpu; in lazy_rcu_shrink_count() local
1322 for_each_possible_cpu(cpu) { in lazy_rcu_shrink_count()
1323 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in lazy_rcu_shrink_count()
1334 int cpu; in lazy_rcu_shrink_scan() local
1339 for_each_possible_cpu(cpu) { in lazy_rcu_shrink_scan()
1340 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in lazy_rcu_shrink_scan()
1366 int cpu; in rcu_init_nohz() local
1422 for_each_cpu(cpu, rcu_nocb_mask) { in rcu_init_nohz()
1423 rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_init_nohz()
1453 static void rcu_spawn_cpu_nocb_kthread(int cpu) in rcu_spawn_cpu_nocb_kthread() argument
1455 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_spawn_cpu_nocb_kthread()
1473 "rcuog/%d", rdp_gp->cpu); in rcu_spawn_cpu_nocb_kthread()
1486 "rcuo%c/%d", rcu_state.abbr, cpu); in rcu_spawn_cpu_nocb_kthread()
1500 cpumask_clear_cpu(cpu, rcu_nocb_mask); in rcu_spawn_cpu_nocb_kthread()
1514 int cpu; in rcu_organize_nocb_kthreads() local
1535 for_each_possible_cpu(cpu) { in rcu_organize_nocb_kthreads()
1536 rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_organize_nocb_kthreads()
1537 if (rdp->cpu >= nl) { in rcu_organize_nocb_kthreads()
1540 nl = DIV_ROUND_UP(rdp->cpu + 1, ls) * ls; in rcu_organize_nocb_kthreads()
1550 __func__, cpu); in rcu_organize_nocb_kthreads()
1556 pr_cont(" %d", cpu); in rcu_organize_nocb_kthreads()
1559 if (cpumask_test_cpu(cpu, rcu_nocb_mask)) in rcu_organize_nocb_kthreads()
1599 rdp->cpu, in show_rcu_nocb_gp_state()
1638 rdp->cpu, rdp->nocb_gp_rdp->cpu, in show_rcu_nocb_state()
1639 nocb_next_rdp ? nocb_next_rdp->cpu : -1, in show_rcu_nocb_state()
1762 static void rcu_spawn_cpu_nocb_kthread(int cpu) in rcu_spawn_cpu_nocb_kthread() argument