247 static unsigned long srcu_readers_lock_idx(struct srcu_struct *ssp, int idx)  in srcu_readers_lock_idx()  argument
255 		sum += READ_ONCE(cpuc->srcu_lock_count[idx]);  in srcu_readers_lock_idx()
264 static unsigned long srcu_readers_unlock_idx(struct srcu_struct *ssp, int idx)  in srcu_readers_unlock_idx()  argument
272 		sum += READ_ONCE(cpuc->srcu_unlock_count[idx]);  in srcu_readers_unlock_idx()
281 static bool srcu_readers_active_idx_check(struct srcu_struct *ssp, int idx)  in srcu_readers_active_idx_check()  argument
285 	unlocks = srcu_readers_unlock_idx(ssp, idx);  in srcu_readers_active_idx_check()
321 	return srcu_readers_lock_idx(ssp, idx) == unlocks;  in srcu_readers_active_idx_check()
405 	int idx;  in __srcu_read_lock()  local
407 	idx = READ_ONCE(ssp->srcu_idx) & 0x1;  in __srcu_read_lock()
408 	this_cpu_inc(ssp->sda->srcu_lock_count[idx]);  in __srcu_read_lock()
410 	return idx;  in __srcu_read_lock()
419 void __srcu_read_unlock(struct srcu_struct *ssp, int idx)  in __srcu_read_unlock()  argument
422 	this_cpu_inc(ssp->sda->srcu_unlock_count[idx]);  in __srcu_read_unlock()
520 	int idx;  in srcu_gp_end()  local
530 	idx = rcu_seq_state(ssp->srcu_gp_seq);  in srcu_gp_end()
531 	WARN_ON_ONCE(idx != SRCU_STATE_SCAN2);  in srcu_gp_end()
543 	idx = rcu_seq_ctr(gpseq) % ARRAY_SIZE(snp->srcu_have_cbs);  in srcu_gp_end()
549 			cbs = snp->srcu_have_cbs[idx] == gpseq;  in srcu_gp_end()
550 		snp->srcu_have_cbs[idx] = gpseq;  in srcu_gp_end()
551 		rcu_seq_set_state(&snp->srcu_have_cbs[idx], 1);  in srcu_gp_end()
554 		mask = snp->srcu_data_have_cbs[idx];  in srcu_gp_end()
555 		snp->srcu_data_have_cbs[idx] = 0;  in srcu_gp_end()
635 	int idx = rcu_seq_ctr(s) % ARRAY_SIZE(sdp->mynode->srcu_have_cbs);  in srcu_funnel_gp_start()  local
644 		if (ULONG_CMP_GE(snp->srcu_have_cbs[idx], s)) {  in srcu_funnel_gp_start()
645 			snp_seq = snp->srcu_have_cbs[idx];  in srcu_funnel_gp_start()
647 				snp->srcu_data_have_cbs[idx] |= sdp->grpmask;  in srcu_funnel_gp_start()
659 		snp->srcu_have_cbs[idx] = s;  in srcu_funnel_gp_start()
661 			snp->srcu_data_have_cbs[idx] |= sdp->grpmask;  in srcu_funnel_gp_start()
698 static bool try_check_zero(struct srcu_struct *ssp, int idx, int trycount)  in try_check_zero()  argument
701 		if (srcu_readers_active_idx_check(ssp, idx))  in try_check_zero()
838 	int idx;  in __call_srcu()  local
852 	idx = srcu_read_lock(ssp);  in __call_srcu()
874 	srcu_read_unlock(ssp, idx);  in __call_srcu()
1092 	int idx;  in srcu_advance_state()  local
1106 	idx = rcu_seq_state(smp_load_acquire(&ssp->srcu_gp_seq)); /* ^^^ */  in srcu_advance_state()
1107 	if (idx == SRCU_STATE_IDLE) {  in srcu_advance_state()
1115 		idx = rcu_seq_state(READ_ONCE(ssp->srcu_gp_seq));  in srcu_advance_state()
1116 		if (idx == SRCU_STATE_IDLE)  in srcu_advance_state()
1119 		if (idx != SRCU_STATE_IDLE) {  in srcu_advance_state()
1126 		idx = 1 ^ (ssp->srcu_idx & 1);  in srcu_advance_state()
1127 		if (!try_check_zero(ssp, idx, 1)) {  in srcu_advance_state()
1141 		idx = 1 ^ (ssp->srcu_idx & 1);  in srcu_advance_state()
1142 		if (!try_check_zero(ssp, idx, 2)) {  in srcu_advance_state()
1255 	int idx;  in srcu_torture_stats_print()  local
1258 	idx = ssp->srcu_idx & 0x1;  in srcu_torture_stats_print()
1260 		 tt, tf, rcu_seq_current(&ssp->srcu_gp_seq), idx);  in srcu_torture_stats_print()
1268 		u0 = sdp->srcu_unlock_count[!idx];  in srcu_torture_stats_print()
1269 		u1 = sdp->srcu_unlock_count[idx];  in srcu_torture_stats_print()
1277 		l0 = sdp->srcu_lock_count[!idx];  in srcu_torture_stats_print()
1278 		l1 = sdp->srcu_lock_count[idx];  in srcu_torture_stats_print()