204 #define RCU_DATA_CPU(cpu)	(&per_cpu(rcu_data, cpu))  argument
210 #define RCU_TRACE_CPU(f, cpu) RCU_TRACE(f, &(RCU_DATA_CPU(cpu)->trace));  argument
379 	int cpu;  in __rcu_advance_callbacks()  local
421 	cpu = raw_smp_processor_id();  in __rcu_advance_callbacks()
422 	if (per_cpu(rcu_flip_flag, cpu) == rcu_flipped) {  in __rcu_advance_callbacks()
424 		per_cpu(rcu_flip_flag, cpu) = rcu_flip_seen;  in __rcu_advance_callbacks()
446 	int cpu = smp_processor_id();  in rcu_irq_enter()  local
447 	struct rcu_dyntick_sched *rdssp = &per_cpu(rcu_dyntick_sched, cpu);  in rcu_irq_enter()
449 	if (per_cpu(rcu_update_flag, cpu))  in rcu_irq_enter()
450 		per_cpu(rcu_update_flag, cpu)++;  in rcu_irq_enter()
497 		per_cpu(rcu_update_flag, cpu)++;  in rcu_irq_enter()
516 	int cpu = smp_processor_id();  in rcu_irq_exit()  local
517 	struct rcu_dyntick_sched *rdssp = &per_cpu(rcu_dyntick_sched, cpu);  in rcu_irq_exit()
528 	if (per_cpu(rcu_update_flag, cpu)) {  in rcu_irq_exit()
529 		if (--per_cpu(rcu_update_flag, cpu))  in rcu_irq_exit()
565 static void dyntick_save_progress_counter(int cpu)  in dyntick_save_progress_counter()  argument
567 	struct rcu_dyntick_sched *rdssp = &per_cpu(rcu_dyntick_sched, cpu);  in dyntick_save_progress_counter()
573 rcu_try_flip_waitack_needed(int cpu)  in rcu_try_flip_waitack_needed()  argument
577 	struct rcu_dyntick_sched *rdssp = &per_cpu(rcu_dyntick_sched, cpu);  in rcu_try_flip_waitack_needed()
610 rcu_try_flip_waitmb_needed(int cpu)  in rcu_try_flip_waitmb_needed()  argument
614 	struct rcu_dyntick_sched *rdssp = &per_cpu(rcu_dyntick_sched, cpu);  in rcu_try_flip_waitmb_needed()
644 static void dyntick_save_progress_counter_sched(int cpu)  in dyntick_save_progress_counter_sched()  argument
646 	struct rcu_dyntick_sched *rdssp = &per_cpu(rcu_dyntick_sched, cpu);  in dyntick_save_progress_counter_sched()
651 static int rcu_qsctr_inc_needed_dyntick(int cpu)  in rcu_qsctr_inc_needed_dyntick()  argument
655 	struct rcu_dyntick_sched *rdssp = &per_cpu(rcu_dyntick_sched, cpu);  in rcu_qsctr_inc_needed_dyntick()
689 # define dyntick_save_progress_counter(cpu)		do { } while (0)  argument
690 # define rcu_try_flip_waitack_needed(cpu)		(1)  argument
691 # define rcu_try_flip_waitmb_needed(cpu)		(1)  argument
693 # define dyntick_save_progress_counter_sched(cpu)	do { } while (0)  argument
694 # define rcu_qsctr_inc_needed_dyntick(cpu)		(1)  argument
698 static void save_qsctr_sched(int cpu)  in save_qsctr_sched()  argument
700 	struct rcu_dyntick_sched *rdssp = &per_cpu(rcu_dyntick_sched, cpu);  in save_qsctr_sched()
705 static inline int rcu_qsctr_inc_needed(int cpu)  in rcu_qsctr_inc_needed()  argument
707 	struct rcu_dyntick_sched *rdssp = &per_cpu(rcu_dyntick_sched, cpu);  in rcu_qsctr_inc_needed()
737 	int cpu;  in rcu_try_flip_idle()  local
762 	for_each_cpu(cpu, to_cpumask(rcu_cpu_online_map)) {  in rcu_try_flip_idle()
763 		per_cpu(rcu_flip_flag, cpu) = rcu_flipped;  in rcu_try_flip_idle()
764 		dyntick_save_progress_counter(cpu);  in rcu_try_flip_idle()
777 	int cpu;  in rcu_try_flip_waitack()  local
780 	for_each_cpu(cpu, to_cpumask(rcu_cpu_online_map))  in rcu_try_flip_waitack()
781 		if (rcu_try_flip_waitack_needed(cpu) &&  in rcu_try_flip_waitack()
782 		    per_cpu(rcu_flip_flag, cpu) != rcu_flip_seen) {  in rcu_try_flip_waitack()
805 	int cpu;  in rcu_try_flip_waitzero()  local
812 	for_each_cpu(cpu, to_cpumask(rcu_cpu_online_map))  in rcu_try_flip_waitzero()
813 		sum += RCU_DATA_CPU(cpu)->rcu_flipctr[lastidx];  in rcu_try_flip_waitzero()
827 	for_each_cpu(cpu, to_cpumask(rcu_cpu_online_map)) {  in rcu_try_flip_waitzero()
828 		per_cpu(rcu_mb_flag, cpu) = rcu_mb_needed;  in rcu_try_flip_waitzero()
829 		dyntick_save_progress_counter(cpu);  in rcu_try_flip_waitzero()
844 	int cpu;  in rcu_try_flip_waitmb()  local
847 	for_each_cpu(cpu, to_cpumask(rcu_cpu_online_map))  in rcu_try_flip_waitmb()
848 		if (rcu_try_flip_waitmb_needed(cpu) &&  in rcu_try_flip_waitmb()
849 		    per_cpu(rcu_mb_flag, cpu) != rcu_mb_done) {  in rcu_try_flip_waitmb()
913 static void rcu_check_mb(int cpu)  in rcu_check_mb()  argument
915 	if (per_cpu(rcu_mb_flag, cpu) == rcu_mb_needed) {  in rcu_check_mb()
917 		per_cpu(rcu_mb_flag, cpu) = rcu_mb_done;  in rcu_check_mb()
921 void rcu_check_callbacks(int cpu, int user)  in rcu_check_callbacks()  argument
924 	struct rcu_data *rdp = RCU_DATA_CPU(cpu);  in rcu_check_callbacks()
940 	    (idle_cpu(cpu) && !in_softirq() &&  in rcu_check_callbacks()
943 	     	rcu_qsctr_inc(cpu);  in rcu_check_callbacks()
946 	rcu_check_mb(cpu);  in rcu_check_callbacks()
964 void rcu_advance_callbacks(int cpu, int user)  in rcu_advance_callbacks()  argument
967 	struct rcu_data *rdp = RCU_DATA_CPU(cpu);  in rcu_advance_callbacks()
990 void rcu_offline_cpu(int cpu)  in rcu_offline_cpu()  argument
995 	struct rcu_data *rdp = RCU_DATA_CPU(cpu);  in rcu_offline_cpu()
1022 	rcu_check_mb(cpu);  in rcu_offline_cpu()
1023 	if (per_cpu(rcu_flip_flag, cpu) == rcu_flipped) {  in rcu_offline_cpu()
1025 		per_cpu(rcu_flip_flag, cpu) = rcu_flip_seen;  in rcu_offline_cpu()
1030 	RCU_DATA_ME()->rcu_flipctr[0] += RCU_DATA_CPU(cpu)->rcu_flipctr[0];  in rcu_offline_cpu()
1031 	RCU_DATA_ME()->rcu_flipctr[1] += RCU_DATA_CPU(cpu)->rcu_flipctr[1];  in rcu_offline_cpu()
1033 	RCU_DATA_CPU(cpu)->rcu_flipctr[0] = 0;  in rcu_offline_cpu()
1034 	RCU_DATA_CPU(cpu)->rcu_flipctr[1] = 0;  in rcu_offline_cpu()
1036 	cpumask_clear_cpu(cpu, to_cpumask(rcu_cpu_online_map));  in rcu_offline_cpu()
1064 void rcu_offline_cpu(int cpu)  in rcu_offline_cpu()  argument
1070 void __cpuinit rcu_online_cpu(int cpu)  in rcu_online_cpu()  argument
1076 	cpumask_set_cpu(cpu, to_cpumask(rcu_cpu_online_map));  in rcu_online_cpu()
1088 	rdp = RCU_DATA_CPU(cpu);  in rcu_online_cpu()
1202 	int cpu;  in rcu_sched_grace_period()  local
1214 		for_each_online_cpu(cpu) {  in rcu_sched_grace_period()
1215 			dyntick_save_progress_counter_sched(cpu);  in rcu_sched_grace_period()
1216 			save_qsctr_sched(cpu);  in rcu_sched_grace_period()
1241 		for_each_online_cpu(cpu) {  in rcu_sched_grace_period()
1242 			while (rcu_qsctr_inc_needed(cpu) &&  in rcu_sched_grace_period()
1243 			       rcu_qsctr_inc_needed_dyntick(cpu)) {  in rcu_sched_grace_period()
1251 		for_each_online_cpu(cpu) {  in rcu_sched_grace_period()
1253 			rdp = RCU_DATA_CPU(cpu);  in rcu_sched_grace_period()
1346 int rcu_needs_cpu(int cpu)  in rcu_needs_cpu()  argument
1348 	struct rcu_data *rdp = RCU_DATA_CPU(cpu);  in rcu_needs_cpu()
1357 int rcu_pending(int cpu)  in rcu_pending()  argument
1359 	struct rcu_data *rdp = RCU_DATA_CPU(cpu);  in rcu_pending()
1372 	if ((per_cpu(rcu_flip_flag, cpu) == rcu_flipped) ||  in rcu_pending()
1373 	    (per_cpu(rcu_mb_flag, cpu) == rcu_mb_needed))  in rcu_pending()
1389 	long cpu = (long)hcpu;  in rcu_cpu_notify()  local
1394 		rcu_online_cpu(cpu);  in rcu_cpu_notify()
1400 		rcu_offline_cpu(cpu);  in rcu_cpu_notify()
1414 	int cpu;  in __rcu_init()  local
1419 	for_each_possible_cpu(cpu) {  in __rcu_init()
1420 		rdp = RCU_DATA_CPU(cpu);  in __rcu_init()
1454 	for_each_online_cpu(cpu)  in __rcu_init()
1455 		rcu_cpu_notify(&rcu_nb, CPU_UP_PREPARE,	(void *)(long) cpu);  in __rcu_init()
1473 long *rcupreempt_flipctr(int cpu)  in rcupreempt_flipctr()  argument
1475 	return &RCU_DATA_CPU(cpu)->rcu_flipctr[0];  in rcupreempt_flipctr()
1479 int rcupreempt_flip_flag(int cpu)  in rcupreempt_flip_flag()  argument
1481 	return per_cpu(rcu_flip_flag, cpu);  in rcupreempt_flip_flag()
1485 int rcupreempt_mb_flag(int cpu)  in rcupreempt_mb_flag()  argument
1487 	return per_cpu(rcu_mb_flag, cpu);  in rcupreempt_mb_flag()
1497 struct rcupreempt_trace *rcupreempt_trace_cpu(int cpu)  in rcupreempt_trace_cpu()  argument
1499 	struct rcu_data *rdp = RCU_DATA_CPU(cpu);  in rcupreempt_trace_cpu()