46 	unsigned long s;  in rcu_exp_gp_seq_snap()  local
49 	s = rcu_seq_snap(&rcu_state.expedited_sequence);  in rcu_exp_gp_seq_snap()
50 	trace_rcu_exp_grace_period(rcu_state.name, s, TPS("snap"));  in rcu_exp_gp_seq_snap()
51 	return s;  in rcu_exp_gp_seq_snap()
59 static bool rcu_exp_gp_seq_done(unsigned long s)  in rcu_exp_gp_seq_done()  argument
61 	return rcu_seq_done(&rcu_state.expedited_sequence, s);  in rcu_exp_gp_seq_done()
258 static bool sync_exp_work_done(unsigned long s)  in sync_exp_work_done()  argument
260 	if (rcu_exp_gp_seq_done(s)) {  in sync_exp_work_done()
261 		trace_rcu_exp_grace_period(rcu_state.name, s, TPS("done"));  in sync_exp_work_done()
275 static bool exp_funnel_lock(unsigned long s)  in exp_funnel_lock()  argument
282 	if (ULONG_CMP_LT(READ_ONCE(rnp->exp_seq_rq), s) &&  in exp_funnel_lock()
284 	     ULONG_CMP_LT(READ_ONCE(rnp_root->exp_seq_rq), s)) &&  in exp_funnel_lock()
296 		if (sync_exp_work_done(s))  in exp_funnel_lock()
301 		if (ULONG_CMP_GE(rnp->exp_seq_rq, s)) {  in exp_funnel_lock()
308 			wait_event(rnp->exp_wq[rcu_seq_ctr(s) & 0x3],  in exp_funnel_lock()
309 				   sync_exp_work_done(s));  in exp_funnel_lock()
312 		rnp->exp_seq_rq = s; /* Followers can wait on us. */  in exp_funnel_lock()
319 	if (sync_exp_work_done(s)) {  in exp_funnel_lock()
324 	trace_rcu_exp_grace_period(rcu_state.name, s, TPS("start"));  in exp_funnel_lock()
539 static void rcu_exp_wait_wake(unsigned long s)  in rcu_exp_wait_wake()  argument
545 	trace_rcu_exp_grace_period(rcu_state.name, s, TPS("end"));  in rcu_exp_wait_wake()
554 		if (ULONG_CMP_LT(READ_ONCE(rnp->exp_seq_rq), s)) {  in rcu_exp_wait_wake()
557 			if (ULONG_CMP_LT(rnp->exp_seq_rq, s))  in rcu_exp_wait_wake()
558 				rnp->exp_seq_rq = s;  in rcu_exp_wait_wake()
564 	trace_rcu_exp_grace_period(rcu_state.name, s, TPS("endwake"));  in rcu_exp_wait_wake()
572 static void rcu_exp_sel_wait_wake(unsigned long s)  in rcu_exp_sel_wait_wake()  argument
578 	rcu_exp_wait_wake(s);  in rcu_exp_sel_wait_wake()
798 	unsigned long s;  in synchronize_rcu_expedited()  local
816 	s = rcu_exp_gp_seq_snap();  in synchronize_rcu_expedited()
817 	if (exp_funnel_lock(s))  in synchronize_rcu_expedited()
823 		rcu_exp_sel_wait_wake(s);  in synchronize_rcu_expedited()
826 		rew.rew_s = s;  in synchronize_rcu_expedited()
833 	wait_event(rnp->exp_wq[rcu_seq_ctr(s) & 0x3],  in synchronize_rcu_expedited()
834 		   sync_exp_work_done(s));  in synchronize_rcu_expedited()