152 	struct rcu_node *rnp;  in rcu_iw_handler()  local
155 	rnp = rdp->mynode;  in rcu_iw_handler()
156 	raw_spin_lock_rcu_node(rnp);  in rcu_iw_handler()
158 		rdp->rcu_iw_gp_seq = rnp->gp_seq;  in rcu_iw_handler()
161 	raw_spin_unlock_rcu_node(rnp);  in rcu_iw_handler()
174 static void rcu_print_detail_task_stall_rnp(struct rcu_node *rnp)  in rcu_print_detail_task_stall_rnp()  argument
179 	raw_spin_lock_irqsave_rcu_node(rnp, flags);  in rcu_print_detail_task_stall_rnp()
180 	if (!rcu_preempt_blocked_readers_cgp(rnp)) {  in rcu_print_detail_task_stall_rnp()
181 		raw_spin_unlock_irqrestore_rcu_node(rnp, flags);  in rcu_print_detail_task_stall_rnp()
184 	t = list_entry(rnp->gp_tasks->prev,  in rcu_print_detail_task_stall_rnp()
186 	list_for_each_entry_continue(t, &rnp->blkd_tasks, rcu_node_entry) {  in rcu_print_detail_task_stall_rnp()
194 	raw_spin_unlock_irqrestore_rcu_node(rnp, flags);  in rcu_print_detail_task_stall_rnp()
201 static int rcu_print_task_stall(struct rcu_node *rnp)  in rcu_print_task_stall()  argument
206 	if (!rcu_preempt_blocked_readers_cgp(rnp))  in rcu_print_task_stall()
209 	       rnp->level, rnp->grplo, rnp->grphi);  in rcu_print_task_stall()
210 	t = list_entry(rnp->gp_tasks->prev,  in rcu_print_task_stall()
212 	list_for_each_entry_continue(t, &rnp->blkd_tasks, rcu_node_entry) {  in rcu_print_task_stall()
226 static void rcu_print_detail_task_stall_rnp(struct rcu_node *rnp)  in rcu_print_detail_task_stall_rnp()  argument
234 static int rcu_print_task_stall(struct rcu_node *rnp)  in rcu_print_task_stall()  argument
250 	struct rcu_node *rnp;  in rcu_dump_cpu_stacks()  local
252 	rcu_for_each_leaf_node(rnp) {  in rcu_dump_cpu_stacks()
253 		raw_spin_lock_irqsave_rcu_node(rnp, flags);  in rcu_dump_cpu_stacks()
254 		for_each_leaf_node_possible_cpu(rnp, cpu)  in rcu_dump_cpu_stacks()
255 			if (rnp->qsmask & leaf_node_cpu_bit(rnp, cpu))  in rcu_dump_cpu_stacks()
258 		raw_spin_unlock_irqrestore_rcu_node(rnp, flags);  in rcu_dump_cpu_stacks()
363 	struct rcu_node *rnp;  in print_other_cpu_stall()  local
377 	rcu_for_each_leaf_node(rnp) {  in print_other_cpu_stall()
378 		raw_spin_lock_irqsave_rcu_node(rnp, flags);  in print_other_cpu_stall()
379 		ndetected += rcu_print_task_stall(rnp);  in print_other_cpu_stall()
380 		if (rnp->qsmask != 0) {  in print_other_cpu_stall()
381 			for_each_leaf_node_possible_cpu(rnp, cpu)  in print_other_cpu_stall()
382 				if (rnp->qsmask & leaf_node_cpu_bit(rnp, cpu)) {  in print_other_cpu_stall()
387 		raw_spin_unlock_irqrestore_rcu_node(rnp, flags);  in print_other_cpu_stall()
399 		rcu_for_each_leaf_node(rnp)  in print_other_cpu_stall()
400 			rcu_print_detail_task_stall_rnp(rnp);  in print_other_cpu_stall()
432 	struct rcu_node *rnp = rcu_get_root();  in print_cpu_stall()  local
459 	raw_spin_lock_irqsave_rcu_node(rnp, flags);  in print_cpu_stall()
464 	raw_spin_unlock_irqrestore_rcu_node(rnp, flags);  in print_cpu_stall()
487 	struct rcu_node *rnp;  in check_cpu_stall()  local
524 	rnp = rdp->mynode;  in check_cpu_stall()
527 	    (READ_ONCE(rnp->qsmask) & rdp->grpmask) &&  in check_cpu_stall()
578 	struct rcu_node *rnp;  in show_rcu_gp_kthreads()  local
592 	rcu_for_each_node_breadth_first(rnp) {  in show_rcu_gp_kthreads()
593 		if (ULONG_CMP_GE(rcu_state.gp_seq, rnp->gp_seq_needed))  in show_rcu_gp_kthreads()
596 			rnp->grplo, rnp->grphi, (long)rnp->gp_seq,  in show_rcu_gp_kthreads()
597 			(long)rnp->gp_seq_needed);  in show_rcu_gp_kthreads()
598 		if (!rcu_is_leaf_node(rnp))  in show_rcu_gp_kthreads()
600 		for_each_leaf_node_possible_cpu(rnp, cpu) {  in show_rcu_gp_kthreads()
623 static void rcu_check_gp_start_stall(struct rcu_node *rnp, struct rcu_data *rdp,  in rcu_check_gp_start_stall()  argument
640 	raw_spin_lock_irqsave_rcu_node(rnp, flags);  in rcu_check_gp_start_stall()
647 		raw_spin_unlock_irqrestore_rcu_node(rnp, flags);  in rcu_check_gp_start_stall()
652 	if (rnp_root != rnp)  in rcu_check_gp_start_stall()
660 		if (rnp_root != rnp)  in rcu_check_gp_start_stall()
663 		raw_spin_unlock_irqrestore_rcu_node(rnp, flags);  in rcu_check_gp_start_stall()
667 	if (rnp_root != rnp)  in rcu_check_gp_start_stall()
669 	raw_spin_unlock_irqrestore_rcu_node(rnp, flags);  in rcu_check_gp_start_stall()