345 static void rcu_eqs_enter_common(struct rcu_dynticks *rdtp, long long oldval,  in rcu_eqs_enter_common()  argument
348 	trace_rcu_dyntick("Start", oldval, rdtp->dynticks_nesting);  in rcu_eqs_enter_common()
361 	atomic_inc(&rdtp->dynticks);  in rcu_eqs_enter_common()
363 	WARN_ON_ONCE(atomic_read(&rdtp->dynticks) & 0x1);  in rcu_eqs_enter_common()
384 	struct rcu_dynticks *rdtp;  in rcu_eqs_enter()  local
386 	rdtp = &__get_cpu_var(rcu_dynticks);  in rcu_eqs_enter()
387 	oldval = rdtp->dynticks_nesting;  in rcu_eqs_enter()
390 		rdtp->dynticks_nesting = 0;  in rcu_eqs_enter()
392 		rdtp->dynticks_nesting -= DYNTICK_TASK_NEST_VALUE;  in rcu_eqs_enter()
393 	rcu_eqs_enter_common(rdtp, oldval, user);  in rcu_eqs_enter()
443 	struct rcu_dynticks *rdtp;  in rcu_user_enter_after_irq()  local
446 	rdtp = &__get_cpu_var(rcu_dynticks);  in rcu_user_enter_after_irq()
448 	WARN_ON_ONCE(!(rdtp->dynticks_nesting & DYNTICK_TASK_MASK));  in rcu_user_enter_after_irq()
449 	rdtp->dynticks_nesting = 1;  in rcu_user_enter_after_irq()
474 	struct rcu_dynticks *rdtp;  in rcu_irq_exit()  local
477 	rdtp = &__get_cpu_var(rcu_dynticks);  in rcu_irq_exit()
478 	oldval = rdtp->dynticks_nesting;  in rcu_irq_exit()
479 	rdtp->dynticks_nesting--;  in rcu_irq_exit()
480 	WARN_ON_ONCE(rdtp->dynticks_nesting < 0);  in rcu_irq_exit()
481 	if (rdtp->dynticks_nesting)  in rcu_irq_exit()
482 		trace_rcu_dyntick("--=", oldval, rdtp->dynticks_nesting);  in rcu_irq_exit()
484 		rcu_eqs_enter_common(rdtp, oldval, true);  in rcu_irq_exit()
495 static void rcu_eqs_exit_common(struct rcu_dynticks *rdtp, long long oldval,  in rcu_eqs_exit_common()  argument
499 	atomic_inc(&rdtp->dynticks);  in rcu_eqs_exit_common()
502 	WARN_ON_ONCE(!(atomic_read(&rdtp->dynticks) & 0x1));  in rcu_eqs_exit_common()
504 	trace_rcu_dyntick("End", oldval, rdtp->dynticks_nesting);  in rcu_eqs_exit_common()
509 				  oldval, rdtp->dynticks_nesting);  in rcu_eqs_exit_common()
523 	struct rcu_dynticks *rdtp;  in rcu_eqs_exit()  local
526 	rdtp = &__get_cpu_var(rcu_dynticks);  in rcu_eqs_exit()
527 	oldval = rdtp->dynticks_nesting;  in rcu_eqs_exit()
530 		rdtp->dynticks_nesting += DYNTICK_TASK_NEST_VALUE;  in rcu_eqs_exit()
532 		rdtp->dynticks_nesting = DYNTICK_TASK_EXIT_IDLE;  in rcu_eqs_exit()
533 	rcu_eqs_exit_common(rdtp, oldval, user);  in rcu_eqs_exit()
581 	struct rcu_dynticks *rdtp;  in rcu_user_exit_after_irq()  local
584 	rdtp = &__get_cpu_var(rcu_dynticks);  in rcu_user_exit_after_irq()
586 	WARN_ON_ONCE(rdtp->dynticks_nesting & DYNTICK_TASK_NEST_MASK);  in rcu_user_exit_after_irq()
587 	rdtp->dynticks_nesting += DYNTICK_TASK_EXIT_IDLE;  in rcu_user_exit_after_irq()
614 	struct rcu_dynticks *rdtp;  in rcu_irq_enter()  local
618 	rdtp = &__get_cpu_var(rcu_dynticks);  in rcu_irq_enter()
619 	oldval = rdtp->dynticks_nesting;  in rcu_irq_enter()
620 	rdtp->dynticks_nesting++;  in rcu_irq_enter()
621 	WARN_ON_ONCE(rdtp->dynticks_nesting == 0);  in rcu_irq_enter()
623 		trace_rcu_dyntick("++=", oldval, rdtp->dynticks_nesting);  in rcu_irq_enter()
625 		rcu_eqs_exit_common(rdtp, oldval, true);  in rcu_irq_enter()
638 	struct rcu_dynticks *rdtp = &__get_cpu_var(rcu_dynticks);  in rcu_nmi_enter()  local
640 	if (rdtp->dynticks_nmi_nesting == 0 &&  in rcu_nmi_enter()
641 	    (atomic_read(&rdtp->dynticks) & 0x1))  in rcu_nmi_enter()
643 	rdtp->dynticks_nmi_nesting++;  in rcu_nmi_enter()
645 	atomic_inc(&rdtp->dynticks);  in rcu_nmi_enter()
648 	WARN_ON_ONCE(!(atomic_read(&rdtp->dynticks) & 0x1));  in rcu_nmi_enter()
660 	struct rcu_dynticks *rdtp = &__get_cpu_var(rcu_dynticks);  in rcu_nmi_exit()  local
662 	if (rdtp->dynticks_nmi_nesting == 0 ||  in rcu_nmi_exit()
663 	    --rdtp->dynticks_nmi_nesting != 0)  in rcu_nmi_exit()
667 	atomic_inc(&rdtp->dynticks);  in rcu_nmi_exit()
669 	WARN_ON_ONCE(atomic_read(&rdtp->dynticks) & 0x1);  in rcu_nmi_exit()