244 static unsigned long srcu_readers_lock_idx(struct srcu_struct *ssp, int idx)  in srcu_readers_lock_idx()  argument
252 		sum += READ_ONCE(cpuc->srcu_lock_count[idx]);  in srcu_readers_lock_idx()
261 static unsigned long srcu_readers_unlock_idx(struct srcu_struct *ssp, int idx)  in srcu_readers_unlock_idx()  argument
269 		sum += READ_ONCE(cpuc->srcu_unlock_count[idx]);  in srcu_readers_unlock_idx()
278 static bool srcu_readers_active_idx_check(struct srcu_struct *ssp, int idx)  in srcu_readers_active_idx_check()  argument
282 	unlocks = srcu_readers_unlock_idx(ssp, idx);  in srcu_readers_active_idx_check()
318 	return srcu_readers_lock_idx(ssp, idx) == unlocks;  in srcu_readers_active_idx_check()
402 	int idx;  in __srcu_read_lock()  local
404 	idx = READ_ONCE(ssp->srcu_idx) & 0x1;  in __srcu_read_lock()
405 	this_cpu_inc(ssp->sda->srcu_lock_count[idx]);  in __srcu_read_lock()
407 	return idx;  in __srcu_read_lock()
416 void __srcu_read_unlock(struct srcu_struct *ssp, int idx)  in __srcu_read_unlock()  argument
419 	this_cpu_inc(ssp->sda->srcu_unlock_count[idx]);  in __srcu_read_unlock()
517 	int idx;  in srcu_gp_end()  local
527 	idx = rcu_seq_state(ssp->srcu_gp_seq);  in srcu_gp_end()
528 	WARN_ON_ONCE(idx != SRCU_STATE_SCAN2);  in srcu_gp_end()
540 	idx = rcu_seq_ctr(gpseq) % ARRAY_SIZE(snp->srcu_have_cbs);  in srcu_gp_end()
546 			cbs = snp->srcu_have_cbs[idx] == gpseq;  in srcu_gp_end()
547 		snp->srcu_have_cbs[idx] = gpseq;  in srcu_gp_end()
548 		rcu_seq_set_state(&snp->srcu_have_cbs[idx], 1);  in srcu_gp_end()
551 		mask = snp->srcu_data_have_cbs[idx];  in srcu_gp_end()
552 		snp->srcu_data_have_cbs[idx] = 0;  in srcu_gp_end()
632 	int idx = rcu_seq_ctr(s) % ARRAY_SIZE(sdp->mynode->srcu_have_cbs);  in srcu_funnel_gp_start()  local
641 		if (ULONG_CMP_GE(snp->srcu_have_cbs[idx], s)) {  in srcu_funnel_gp_start()
642 			snp_seq = snp->srcu_have_cbs[idx];  in srcu_funnel_gp_start()
644 				snp->srcu_data_have_cbs[idx] |= sdp->grpmask;  in srcu_funnel_gp_start()
656 		snp->srcu_have_cbs[idx] = s;  in srcu_funnel_gp_start()
658 			snp->srcu_data_have_cbs[idx] |= sdp->grpmask;  in srcu_funnel_gp_start()
695 static bool try_check_zero(struct srcu_struct *ssp, int idx, int trycount)  in try_check_zero()  argument
698 		if (srcu_readers_active_idx_check(ssp, idx))  in try_check_zero()
812 	int idx;  in srcu_gp_start_if_needed()  local
819 	idx = srcu_read_lock(ssp);  in srcu_gp_start_if_needed()
841 	srcu_read_unlock(ssp, idx);  in srcu_gp_start_if_needed()
1175 	int idx;  in srcu_advance_state()  local
1189 	idx = rcu_seq_state(smp_load_acquire(&ssp->srcu_gp_seq)); /* ^^^ */  in srcu_advance_state()
1190 	if (idx == SRCU_STATE_IDLE) {  in srcu_advance_state()
1198 		idx = rcu_seq_state(READ_ONCE(ssp->srcu_gp_seq));  in srcu_advance_state()
1199 		if (idx == SRCU_STATE_IDLE)  in srcu_advance_state()
1202 		if (idx != SRCU_STATE_IDLE) {  in srcu_advance_state()
1209 		idx = 1 ^ (ssp->srcu_idx & 1);  in srcu_advance_state()
1210 		if (!try_check_zero(ssp, idx, 1)) {  in srcu_advance_state()
1226 		idx = 1 ^ (ssp->srcu_idx & 1);  in srcu_advance_state()
1227 		if (!try_check_zero(ssp, idx, 2)) {  in srcu_advance_state()
1343 	int idx;  in srcu_torture_stats_print()  local
1346 	idx = ssp->srcu_idx & 0x1;  in srcu_torture_stats_print()
1348 		 tt, tf, rcu_seq_current(&ssp->srcu_gp_seq), idx);  in srcu_torture_stats_print()
1356 		u0 = data_race(sdp->srcu_unlock_count[!idx]);  in srcu_torture_stats_print()
1357 		u1 = data_race(sdp->srcu_unlock_count[idx]);  in srcu_torture_stats_print()
1365 		l0 = data_race(sdp->srcu_lock_count[!idx]);  in srcu_torture_stats_print()
1366 		l1 = data_race(sdp->srcu_lock_count[idx]);  in srcu_torture_stats_print()