50 	unsigned long s;  in rcu_exp_gp_seq_snap()  local
53 	s = rcu_seq_snap(&rcu_state.expedited_sequence);  in rcu_exp_gp_seq_snap()
54 	trace_rcu_exp_grace_period(rcu_state.name, s, TPS("snap"));  in rcu_exp_gp_seq_snap()
55 	return s;  in rcu_exp_gp_seq_snap()
63 static bool rcu_exp_gp_seq_done(unsigned long s)  in rcu_exp_gp_seq_done()  argument
65 	return rcu_seq_done(&rcu_state.expedited_sequence, s);  in rcu_exp_gp_seq_done()
264 static bool sync_exp_work_done(unsigned long s)  in sync_exp_work_done()  argument
266 	if (rcu_exp_gp_seq_done(s)) {  in sync_exp_work_done()
267 		trace_rcu_exp_grace_period(rcu_state.name, s, TPS("done"));  in sync_exp_work_done()
281 static bool exp_funnel_lock(unsigned long s)  in exp_funnel_lock()  argument
288 	if (ULONG_CMP_LT(READ_ONCE(rnp->exp_seq_rq), s) &&  in exp_funnel_lock()
290 	     ULONG_CMP_LT(READ_ONCE(rnp_root->exp_seq_rq), s)) &&  in exp_funnel_lock()
302 		if (sync_exp_work_done(s))  in exp_funnel_lock()
307 		if (ULONG_CMP_GE(rnp->exp_seq_rq, s)) {  in exp_funnel_lock()
314 			wait_event(rnp->exp_wq[rcu_seq_ctr(s) & 0x3],  in exp_funnel_lock()
315 				   sync_exp_work_done(s));  in exp_funnel_lock()
318 		WRITE_ONCE(rnp->exp_seq_rq, s); /* Followers can wait on us. */  in exp_funnel_lock()
325 	if (sync_exp_work_done(s)) {  in exp_funnel_lock()
330 	trace_rcu_exp_grace_period(rcu_state.name, s, TPS("start"));  in exp_funnel_lock()
419 static void rcu_exp_sel_wait_wake(unsigned long s);
685 static void rcu_exp_wait_wake(unsigned long s)  in rcu_exp_wait_wake()  argument
696 	trace_rcu_exp_grace_period(rcu_state.name, s, TPS("end"));  in rcu_exp_wait_wake()
699 		if (ULONG_CMP_LT(READ_ONCE(rnp->exp_seq_rq), s)) {  in rcu_exp_wait_wake()
702 			if (ULONG_CMP_LT(rnp->exp_seq_rq, s))  in rcu_exp_wait_wake()
703 				WRITE_ONCE(rnp->exp_seq_rq, s);  in rcu_exp_wait_wake()
707 		wake_up_all(&rnp->exp_wq[rcu_seq_ctr(s) & 0x3]);  in rcu_exp_wait_wake()
709 	trace_rcu_exp_grace_period(rcu_state.name, s, TPS("endwake"));  in rcu_exp_wait_wake()
717 static void rcu_exp_sel_wait_wake(unsigned long s)  in rcu_exp_sel_wait_wake()  argument
723 	rcu_exp_wait_wake(s);  in rcu_exp_sel_wait_wake()
917 	unsigned long s;  in synchronize_rcu_expedited()  local
948 	s = rcu_exp_gp_seq_snap();  in synchronize_rcu_expedited()
949 	if (exp_funnel_lock(s))  in synchronize_rcu_expedited()
955 		rcu_exp_sel_wait_wake(s);  in synchronize_rcu_expedited()
958 		rew.rew_s = s;  in synchronize_rcu_expedited()
964 	wait_event(rnp->exp_wq[rcu_seq_ctr(s) & 0x3],  in synchronize_rcu_expedited()
965 		   sync_exp_work_done(s));  in synchronize_rcu_expedited()
985 	unsigned long s;  in sync_rcu_do_polled_gp()  local
988 	s = rnp->exp_seq_poll_rq;  in sync_rcu_do_polled_gp()
991 	if (s == RCU_GET_STATE_COMPLETED)  in sync_rcu_do_polled_gp()
993 	while (!poll_state_synchronize_rcu(s)) {  in sync_rcu_do_polled_gp()
996 …pr_info("%s: i = %d s = %lx gp_seq_polled = %lx\n", __func__, i, s, READ_ONCE(rcu_state.gp_seq_pol…  in sync_rcu_do_polled_gp()
1000 	s = rnp->exp_seq_poll_rq;  in sync_rcu_do_polled_gp()
1001 	if (poll_state_synchronize_rcu(s))  in sync_rcu_do_polled_gp()
1020 	unsigned long s;  in start_poll_synchronize_rcu_expedited()  local
1022 	s = get_state_synchronize_rcu();  in start_poll_synchronize_rcu_expedited()
1027 	if (!poll_state_synchronize_rcu(s)) {  in start_poll_synchronize_rcu_expedited()
1028 		rnp->exp_seq_poll_rq = s;  in start_poll_synchronize_rcu_expedited()
1035 	return s;  in start_poll_synchronize_rcu_expedited()