Lines Matching refs:rnp
84 static void rcu_report_exp_rnp(struct rcu_node *rnp, bool wake);
130 static void rcu_preempt_ctxt_queue(struct rcu_node *rnp, struct rcu_data *rdp) in rcu_preempt_ctxt_queue() argument
131 __releases(rnp->lock) /* But leaves rrupts disabled. */ in rcu_preempt_ctxt_queue()
133 int blkd_state = (rnp->gp_tasks ? RCU_GP_TASKS : 0) + in rcu_preempt_ctxt_queue()
134 (rnp->exp_tasks ? RCU_EXP_TASKS : 0) + in rcu_preempt_ctxt_queue()
135 (rnp->qsmask & rdp->grpmask ? RCU_GP_BLKD : 0) + in rcu_preempt_ctxt_queue()
136 (rnp->expmask & rdp->grpmask ? RCU_EXP_BLKD : 0); in rcu_preempt_ctxt_queue()
139 raw_lockdep_assert_held_rcu_node(rnp); in rcu_preempt_ctxt_queue()
140 WARN_ON_ONCE(rdp->mynode != rnp); in rcu_preempt_ctxt_queue()
141 WARN_ON_ONCE(!rcu_is_leaf_node(rnp)); in rcu_preempt_ctxt_queue()
143 WARN_ON_ONCE(rnp->qsmaskinitnext & ~rnp->qsmaskinit & rnp->qsmask & in rcu_preempt_ctxt_queue()
164 list_add(&t->rcu_node_entry, &rnp->blkd_tasks); in rcu_preempt_ctxt_queue()
182 list_add_tail(&t->rcu_node_entry, &rnp->blkd_tasks); in rcu_preempt_ctxt_queue()
195 list_add(&t->rcu_node_entry, rnp->exp_tasks); in rcu_preempt_ctxt_queue()
206 list_add(&t->rcu_node_entry, rnp->gp_tasks); in rcu_preempt_ctxt_queue()
222 if (!rnp->gp_tasks && (blkd_state & RCU_GP_BLKD)) { in rcu_preempt_ctxt_queue()
223 WRITE_ONCE(rnp->gp_tasks, &t->rcu_node_entry); in rcu_preempt_ctxt_queue()
224 WARN_ON_ONCE(rnp->completedqs == rnp->gp_seq); in rcu_preempt_ctxt_queue()
226 if (!rnp->exp_tasks && (blkd_state & RCU_EXP_BLKD)) in rcu_preempt_ctxt_queue()
227 rnp->exp_tasks = &t->rcu_node_entry; in rcu_preempt_ctxt_queue()
229 !(rnp->qsmask & rdp->grpmask)); in rcu_preempt_ctxt_queue()
231 !(rnp->expmask & rdp->grpmask)); in rcu_preempt_ctxt_queue()
232 raw_spin_unlock_rcu_node(rnp); /* interrupts remain disabled. */ in rcu_preempt_ctxt_queue()
289 struct rcu_node *rnp; in rcu_note_context_switch() local
298 rnp = rdp->mynode; in rcu_note_context_switch()
299 raw_spin_lock_rcu_node(rnp); in rcu_note_context_switch()
301 t->rcu_blocked_node = rnp; in rcu_note_context_switch()
308 WARN_ON_ONCE((rdp->grpmask & rcu_rnp_online_cpus(rnp)) == 0); in rcu_note_context_switch()
312 (rnp->qsmask & rdp->grpmask) in rcu_note_context_switch()
313 ? rnp->gp_seq in rcu_note_context_switch()
314 : rcu_seq_snap(&rnp->gp_seq)); in rcu_note_context_switch()
315 rcu_preempt_ctxt_queue(rnp, rdp); in rcu_note_context_switch()
341 static int rcu_preempt_blocked_readers_cgp(struct rcu_node *rnp) in rcu_preempt_blocked_readers_cgp() argument
343 return READ_ONCE(rnp->gp_tasks) != NULL; in rcu_preempt_blocked_readers_cgp()
400 struct rcu_node *rnp) in rcu_next_node_entry() argument
405 if (np == &rnp->blkd_tasks) in rcu_next_node_entry()
414 static bool rcu_preempt_has_tasks(struct rcu_node *rnp) in rcu_preempt_has_tasks() argument
416 return !list_empty(&rnp->blkd_tasks); in rcu_preempt_has_tasks()
433 struct rcu_node *rnp; in rcu_preempt_deferred_qs_irqrestore() local
481 rnp = t->rcu_blocked_node; in rcu_preempt_deferred_qs_irqrestore()
482 raw_spin_lock_rcu_node(rnp); /* irqs already disabled. */ in rcu_preempt_deferred_qs_irqrestore()
483 WARN_ON_ONCE(rnp != t->rcu_blocked_node); in rcu_preempt_deferred_qs_irqrestore()
484 WARN_ON_ONCE(!rcu_is_leaf_node(rnp)); in rcu_preempt_deferred_qs_irqrestore()
485 empty_norm = !rcu_preempt_blocked_readers_cgp(rnp); in rcu_preempt_deferred_qs_irqrestore()
486 WARN_ON_ONCE(rnp->completedqs == rnp->gp_seq && in rcu_preempt_deferred_qs_irqrestore()
487 (!empty_norm || rnp->qsmask)); in rcu_preempt_deferred_qs_irqrestore()
488 empty_exp = sync_rcu_preempt_exp_done(rnp); in rcu_preempt_deferred_qs_irqrestore()
490 np = rcu_next_node_entry(t, rnp); in rcu_preempt_deferred_qs_irqrestore()
494 rnp->gp_seq, t->pid); in rcu_preempt_deferred_qs_irqrestore()
495 if (&t->rcu_node_entry == rnp->gp_tasks) in rcu_preempt_deferred_qs_irqrestore()
496 WRITE_ONCE(rnp->gp_tasks, np); in rcu_preempt_deferred_qs_irqrestore()
497 if (&t->rcu_node_entry == rnp->exp_tasks) in rcu_preempt_deferred_qs_irqrestore()
498 rnp->exp_tasks = np; in rcu_preempt_deferred_qs_irqrestore()
501 drop_boost_mutex = rt_mutex_owner(&rnp->boost_mtx) == t; in rcu_preempt_deferred_qs_irqrestore()
502 if (&t->rcu_node_entry == rnp->boost_tasks) in rcu_preempt_deferred_qs_irqrestore()
503 rnp->boost_tasks = np; in rcu_preempt_deferred_qs_irqrestore()
512 empty_exp_now = sync_rcu_preempt_exp_done(rnp); in rcu_preempt_deferred_qs_irqrestore()
513 if (!empty_norm && !rcu_preempt_blocked_readers_cgp(rnp)) { in rcu_preempt_deferred_qs_irqrestore()
515 rnp->gp_seq, in rcu_preempt_deferred_qs_irqrestore()
516 0, rnp->qsmask, in rcu_preempt_deferred_qs_irqrestore()
517 rnp->level, in rcu_preempt_deferred_qs_irqrestore()
518 rnp->grplo, in rcu_preempt_deferred_qs_irqrestore()
519 rnp->grphi, in rcu_preempt_deferred_qs_irqrestore()
520 !!rnp->gp_tasks); in rcu_preempt_deferred_qs_irqrestore()
521 rcu_report_unblock_qs_rnp(rnp, flags); in rcu_preempt_deferred_qs_irqrestore()
523 raw_spin_unlock_irqrestore_rcu_node(rnp, flags); in rcu_preempt_deferred_qs_irqrestore()
531 rcu_report_exp_rnp(rnp, true); in rcu_preempt_deferred_qs_irqrestore()
535 rt_mutex_futex_unlock(&rnp->boost_mtx); in rcu_preempt_deferred_qs_irqrestore()
612 struct rcu_node *rnp = rdp->mynode; in rcu_read_unlock_special() local
616 (rdp->grpmask & READ_ONCE(rnp->expmask)) || in rcu_read_unlock_special()
658 static void rcu_preempt_check_blocked_tasks(struct rcu_node *rnp) in rcu_preempt_check_blocked_tasks() argument
663 if (WARN_ON_ONCE(rcu_preempt_blocked_readers_cgp(rnp))) in rcu_preempt_check_blocked_tasks()
664 dump_blkd_tasks(rnp, 10); in rcu_preempt_check_blocked_tasks()
665 if (rcu_preempt_has_tasks(rnp) && in rcu_preempt_check_blocked_tasks()
666 (rnp->qsmaskinit || rnp->wait_blkd_tasks)) { in rcu_preempt_check_blocked_tasks()
667 WRITE_ONCE(rnp->gp_tasks, rnp->blkd_tasks.next); in rcu_preempt_check_blocked_tasks()
668 t = container_of(rnp->gp_tasks, struct task_struct, in rcu_preempt_check_blocked_tasks()
671 rnp->gp_seq, t->pid); in rcu_preempt_check_blocked_tasks()
673 WARN_ON_ONCE(rnp->qsmask); in rcu_preempt_check_blocked_tasks()
744 dump_blkd_tasks(struct rcu_node *rnp, int ncheck) in dump_blkd_tasks() argument
753 raw_lockdep_assert_held_rcu_node(rnp); in dump_blkd_tasks()
755 __func__, rnp->grplo, rnp->grphi, rnp->level, in dump_blkd_tasks()
756 (long)rnp->gp_seq, (long)rnp->completedqs); in dump_blkd_tasks()
757 for (rnp1 = rnp; rnp1; rnp1 = rnp1->parent) in dump_blkd_tasks()
761 __func__, READ_ONCE(rnp->gp_tasks), rnp->boost_tasks, in dump_blkd_tasks()
762 rnp->exp_tasks); in dump_blkd_tasks()
765 list_for_each(lhp, &rnp->blkd_tasks) { in dump_blkd_tasks()
771 for (cpu = rnp->grplo; cpu <= rnp->grphi; cpu++) { in dump_blkd_tasks()
773 onl = !!(rdp->grpmask & rcu_rnp_online_cpus(rnp)); in dump_blkd_tasks()
866 static int rcu_preempt_blocked_readers_cgp(struct rcu_node *rnp) in rcu_preempt_blocked_readers_cgp() argument
874 static bool rcu_preempt_has_tasks(struct rcu_node *rnp) in rcu_preempt_has_tasks() argument
894 static void rcu_preempt_check_blocked_tasks(struct rcu_node *rnp) in rcu_preempt_check_blocked_tasks() argument
896 WARN_ON_ONCE(rnp->qsmask); in rcu_preempt_check_blocked_tasks()
935 dump_blkd_tasks(struct rcu_node *rnp, int ncheck) in dump_blkd_tasks() argument
937 WARN_ON_ONCE(!list_empty(&rnp->blkd_tasks)); in dump_blkd_tasks()
965 static int rcu_boost(struct rcu_node *rnp) in rcu_boost() argument
971 if (READ_ONCE(rnp->exp_tasks) == NULL && in rcu_boost()
972 READ_ONCE(rnp->boost_tasks) == NULL) in rcu_boost()
975 raw_spin_lock_irqsave_rcu_node(rnp, flags); in rcu_boost()
981 if (rnp->exp_tasks == NULL && rnp->boost_tasks == NULL) { in rcu_boost()
982 raw_spin_unlock_irqrestore_rcu_node(rnp, flags); in rcu_boost()
992 if (rnp->exp_tasks != NULL) in rcu_boost()
993 tb = rnp->exp_tasks; in rcu_boost()
995 tb = rnp->boost_tasks; in rcu_boost()
1014 rt_mutex_init_proxy_locked(&rnp->boost_mtx, t); in rcu_boost()
1015 raw_spin_unlock_irqrestore_rcu_node(rnp, flags); in rcu_boost()
1017 rt_mutex_lock(&rnp->boost_mtx); in rcu_boost()
1018 rt_mutex_unlock(&rnp->boost_mtx); /* Then keep lockdep happy. */ in rcu_boost()
1020 return READ_ONCE(rnp->exp_tasks) != NULL || in rcu_boost()
1021 READ_ONCE(rnp->boost_tasks) != NULL; in rcu_boost()
1029 struct rcu_node *rnp = (struct rcu_node *)arg; in rcu_boost_kthread() local
1035 rnp->boost_kthread_status = RCU_KTHREAD_WAITING; in rcu_boost_kthread()
1037 rcu_wait(rnp->boost_tasks || rnp->exp_tasks); in rcu_boost_kthread()
1039 rnp->boost_kthread_status = RCU_KTHREAD_RUNNING; in rcu_boost_kthread()
1040 more2boost = rcu_boost(rnp); in rcu_boost_kthread()
1046 rnp->boost_kthread_status = RCU_KTHREAD_YIELDING; in rcu_boost_kthread()
1068 static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags) in rcu_initiate_boost() argument
1069 __releases(rnp->lock) in rcu_initiate_boost()
1071 raw_lockdep_assert_held_rcu_node(rnp); in rcu_initiate_boost()
1072 if (!rcu_preempt_blocked_readers_cgp(rnp) && rnp->exp_tasks == NULL) { in rcu_initiate_boost()
1073 raw_spin_unlock_irqrestore_rcu_node(rnp, flags); in rcu_initiate_boost()
1076 if (rnp->exp_tasks != NULL || in rcu_initiate_boost()
1077 (rnp->gp_tasks != NULL && in rcu_initiate_boost()
1078 rnp->boost_tasks == NULL && in rcu_initiate_boost()
1079 rnp->qsmask == 0 && in rcu_initiate_boost()
1080 ULONG_CMP_GE(jiffies, rnp->boost_time))) { in rcu_initiate_boost()
1081 if (rnp->exp_tasks == NULL) in rcu_initiate_boost()
1082 rnp->boost_tasks = rnp->gp_tasks; in rcu_initiate_boost()
1083 raw_spin_unlock_irqrestore_rcu_node(rnp, flags); in rcu_initiate_boost()
1084 rcu_wake_cond(rnp->boost_kthread_task, in rcu_initiate_boost()
1085 rnp->boost_kthread_status); in rcu_initiate_boost()
1087 raw_spin_unlock_irqrestore_rcu_node(rnp, flags); in rcu_initiate_boost()
1105 static void rcu_preempt_boost_start_gp(struct rcu_node *rnp) in rcu_preempt_boost_start_gp() argument
1107 rnp->boost_time = jiffies + RCU_BOOST_DELAY_JIFFIES; in rcu_preempt_boost_start_gp()
1115 static void rcu_spawn_one_boost_kthread(struct rcu_node *rnp) in rcu_spawn_one_boost_kthread() argument
1117 int rnp_index = rnp - rcu_get_root(); in rcu_spawn_one_boost_kthread()
1125 if (!rcu_scheduler_fully_active || rcu_rnp_online_cpus(rnp) == 0) in rcu_spawn_one_boost_kthread()
1130 if (rnp->boost_kthread_task != NULL) in rcu_spawn_one_boost_kthread()
1133 t = kthread_create(rcu_boost_kthread, (void *)rnp, in rcu_spawn_one_boost_kthread()
1138 raw_spin_lock_irqsave_rcu_node(rnp, flags); in rcu_spawn_one_boost_kthread()
1139 rnp->boost_kthread_task = t; in rcu_spawn_one_boost_kthread()
1140 raw_spin_unlock_irqrestore_rcu_node(rnp, flags); in rcu_spawn_one_boost_kthread()
1155 static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu) in rcu_boost_kthread_setaffinity() argument
1157 struct task_struct *t = rnp->boost_kthread_task; in rcu_boost_kthread_setaffinity()
1158 unsigned long mask = rcu_rnp_online_cpus(rnp); in rcu_boost_kthread_setaffinity()
1166 for_each_leaf_node_possible_cpu(rnp, cpu) in rcu_boost_kthread_setaffinity()
1167 if ((mask & leaf_node_cpu_bit(rnp, cpu)) && in rcu_boost_kthread_setaffinity()
1181 struct rcu_node *rnp; in rcu_spawn_boost_kthreads() local
1183 rcu_for_each_leaf_node(rnp) in rcu_spawn_boost_kthreads()
1184 rcu_spawn_one_boost_kthread(rnp); in rcu_spawn_boost_kthreads()
1190 struct rcu_node *rnp = rdp->mynode; in rcu_prepare_kthreads() local
1194 rcu_spawn_one_boost_kthread(rnp); in rcu_prepare_kthreads()
1199 static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags) in rcu_initiate_boost() argument
1200 __releases(rnp->lock) in rcu_initiate_boost()
1202 raw_spin_unlock_irqrestore_rcu_node(rnp, flags); in rcu_initiate_boost()
1210 static void rcu_preempt_boost_start_gp(struct rcu_node *rnp) in rcu_preempt_boost_start_gp() argument
1214 static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu) in rcu_boost_kthread_setaffinity() argument
1304 struct rcu_node *rnp; in rcu_try_advance_all_cbs() local
1311 rnp = rdp->mynode; in rcu_try_advance_all_cbs()
1319 rcu_seq_current(&rnp->gp_seq)) || in rcu_try_advance_all_cbs()
1385 struct rcu_node *rnp; in rcu_prepare_for_idle() local
1422 rnp = rdp->mynode; in rcu_prepare_for_idle()
1423 raw_spin_lock_rcu_node(rnp); /* irqs already disabled. */ in rcu_prepare_for_idle()
1424 needwake = rcu_accelerate_cbs(rnp, rdp); in rcu_prepare_for_idle()
1425 raw_spin_unlock_rcu_node(rnp); /* irqs remain disabled. */ in rcu_prepare_for_idle()
1620 static struct swait_queue_head *rcu_nocb_gp_get(struct rcu_node *rnp) in rcu_nocb_gp_get() argument
1622 return &rnp->nocb_gp_wq[rcu_seq_ctr(rnp->gp_seq) & 0x1]; in rcu_nocb_gp_get()
1625 static void rcu_init_one_nocb(struct rcu_node *rnp) in rcu_init_one_nocb() argument
1627 init_swait_queue_head(&rnp->nocb_gp_wq[0]); in rcu_init_one_nocb()
1628 init_swait_queue_head(&rnp->nocb_gp_wq[1]); in rcu_init_one_nocb()
1957 struct rcu_node *rnp; in nocb_gp_wait() local
1984 rnp = rdp->mynode; in nocb_gp_wait()
1995 rcu_seq_done(&rnp->gp_seq, cur_gp_seq))) { in nocb_gp_wait()
1996 raw_spin_lock_rcu_node(rnp); /* irqs disabled. */ in nocb_gp_wait()
1997 needwake_gp = rcu_advance_cbs(rnp, rdp); in nocb_gp_wait()
1998 raw_spin_unlock_rcu_node(rnp); /* irqs disabled. */ in nocb_gp_wait()
2049 rnp = my_rdp->mynode; in nocb_gp_wait()
2050 trace_rcu_this_gp(rnp, my_rdp, wait_gp_seq, TPS("StartWait")); in nocb_gp_wait()
2052 rnp->nocb_gp_wq[rcu_seq_ctr(wait_gp_seq) & 0x1], in nocb_gp_wait()
2053 rcu_seq_done(&rnp->gp_seq, wait_gp_seq) || in nocb_gp_wait()
2055 trace_rcu_this_gp(rnp, my_rdp, wait_gp_seq, TPS("EndWait")); in nocb_gp_wait()
2097 struct rcu_node *rnp = rdp->mynode; in nocb_cb_wait() local
2108 rcu_seq_done(&rnp->gp_seq, cur_gp_seq) && in nocb_cb_wait()
2109 raw_spin_trylock_rcu_node(rnp)) { /* irqs already disabled. */ in nocb_cb_wait()
2111 raw_spin_unlock_rcu_node(rnp); /* irqs remain disabled. */ in nocb_cb_wait()
2399 struct rcu_node *rnp = rdp->mynode; in show_rcu_nocb_gp_state() local
2410 ".W"[swait_active(&rnp->nocb_gp_wq[0])], in show_rcu_nocb_gp_state()
2411 ".W"[swait_active(&rnp->nocb_gp_wq[1])], in show_rcu_nocb_gp_state()
2415 rnp->grplo, rnp->grphi, READ_ONCE(rdp->nocb_gp_loops)); in show_rcu_nocb_gp_state()
2495 static struct swait_queue_head *rcu_nocb_gp_get(struct rcu_node *rnp) in rcu_nocb_gp_get() argument
2500 static void rcu_init_one_nocb(struct rcu_node *rnp) in rcu_init_one_nocb() argument