596 				  unsigned long s)  in srcu_funnel_exp_start()  argument
601 		if (rcu_seq_done(&ssp->srcu_gp_seq, s) ||  in srcu_funnel_exp_start()
602 		    ULONG_CMP_GE(READ_ONCE(snp->srcu_gp_seq_needed_exp), s))  in srcu_funnel_exp_start()
605 		if (ULONG_CMP_GE(snp->srcu_gp_seq_needed_exp, s)) {  in srcu_funnel_exp_start()
609 		WRITE_ONCE(snp->srcu_gp_seq_needed_exp, s);  in srcu_funnel_exp_start()
613 	if (ULONG_CMP_LT(ssp->srcu_gp_seq_needed_exp, s))  in srcu_funnel_exp_start()
614 		WRITE_ONCE(ssp->srcu_gp_seq_needed_exp, s);  in srcu_funnel_exp_start()
629 				 unsigned long s, bool do_norm)  in srcu_funnel_gp_start()  argument
632 	int idx = rcu_seq_ctr(s) % ARRAY_SIZE(sdp->mynode->srcu_have_cbs);  in srcu_funnel_gp_start()
638 		if (rcu_seq_done(&ssp->srcu_gp_seq, s) && snp != sdp->mynode)  in srcu_funnel_gp_start()
641 		if (ULONG_CMP_GE(snp->srcu_have_cbs[idx], s)) {  in srcu_funnel_gp_start()
643 			if (snp == sdp->mynode && snp_seq == s)  in srcu_funnel_gp_start()
646 			if (snp == sdp->mynode && snp_seq != s) {  in srcu_funnel_gp_start()
653 				srcu_funnel_exp_start(ssp, snp, s);  in srcu_funnel_gp_start()
656 		snp->srcu_have_cbs[idx] = s;  in srcu_funnel_gp_start()
659 		if (!do_norm && ULONG_CMP_LT(snp->srcu_gp_seq_needed_exp, s))  in srcu_funnel_gp_start()
660 			WRITE_ONCE(snp->srcu_gp_seq_needed_exp, s);  in srcu_funnel_gp_start()
666 	if (ULONG_CMP_LT(ssp->srcu_gp_seq_needed, s)) {  in srcu_funnel_gp_start()
671 		smp_store_release(&ssp->srcu_gp_seq_needed, s); /*^^^*/  in srcu_funnel_gp_start()
673 	if (!do_norm && ULONG_CMP_LT(ssp->srcu_gp_seq_needed_exp, s))  in srcu_funnel_gp_start()
674 		WRITE_ONCE(ssp->srcu_gp_seq_needed_exp, s);  in srcu_funnel_gp_start()
677 	if (!rcu_seq_done(&ssp->srcu_gp_seq, s) &&  in srcu_funnel_gp_start()
815 	unsigned long s;  in srcu_gp_start_if_needed()  local
826 	s = rcu_seq_snap(&ssp->srcu_gp_seq);  in srcu_gp_start_if_needed()
827 	(void)rcu_segcblist_accelerate(&sdp->srcu_cblist, s);  in srcu_gp_start_if_needed()
828 	if (ULONG_CMP_LT(sdp->srcu_gp_seq_needed, s)) {  in srcu_gp_start_if_needed()
829 		sdp->srcu_gp_seq_needed = s;  in srcu_gp_start_if_needed()
832 	if (!do_norm && ULONG_CMP_LT(sdp->srcu_gp_seq_needed_exp, s)) {  in srcu_gp_start_if_needed()
833 		sdp->srcu_gp_seq_needed_exp = s;  in srcu_gp_start_if_needed()
838 		srcu_funnel_gp_start(ssp, sdp, s, do_norm);  in srcu_gp_start_if_needed()
840 		srcu_funnel_exp_start(ssp, sdp->mynode, s);  in srcu_gp_start_if_needed()
842 	return s;  in srcu_gp_start_if_needed()
1108 	unsigned long s = rcu_seq_snap(&ssp->srcu_barrier_seq);  in srcu_barrier()  local
1112 	if (rcu_seq_done(&ssp->srcu_barrier_seq, s)) {  in srcu_barrier()