Lines Matching refs:rcu_state
87 struct rcu_state rcu_state = { variable
88 .level = { &rcu_state.node[0] },
91 .barrier_mutex = __MUTEX_INITIALIZER(rcu_state.barrier_mutex),
94 .exp_mutex = __MUTEX_INITIALIZER(rcu_state.exp_mutex),
95 .exp_wake_mutex = __MUTEX_INITIALIZER(rcu_state.exp_wake_mutex),
96 .ofl_lock = __RAW_SPIN_LOCK_UNLOCKED(rcu_state.ofl_lock),
203 return rcu_seq_state(rcu_seq_current(&rcu_state.gp_seq)); in rcu_gp_in_progress()
506 return READ_ONCE(rcu_state.gp_seq); in rcu_get_gp_seq()
518 return rcu_state.expedited_sequence; in rcu_exp_batches_completed()
527 return &rcu_state.node[0]; in rcu_get_root()
548 *flags = READ_ONCE(rcu_state.gp_flags); in rcutorture_get_gp_data()
549 *gp_seq = rcu_seq_current(&rcu_state.gp_seq); in rcutorture_get_gp_data()
995 trace_rcu_fqs(rcu_state.name, rdp->gp_seq, rdp->cpu, TPS("dti")); in dyntick_save_progress_counter()
1024 trace_rcu_fqs(rcu_state.name, rdp->gp_seq, rdp->cpu, TPS("dti")); in rcu_implicit_dynticks_qs()
1031 time_after(jiffies, rcu_state.gp_start + HZ)) { in rcu_implicit_dynticks_qs()
1065 (time_after(jiffies, rcu_state.gp_start + jtsq * 2) || in rcu_implicit_dynticks_qs()
1066 time_after(jiffies, rcu_state.jiffies_resched))) { in rcu_implicit_dynticks_qs()
1070 } else if (time_after(jiffies, rcu_state.gp_start + jtsq)) { in rcu_implicit_dynticks_qs()
1096 if (time_after(jiffies, rcu_state.jiffies_resched)) { in rcu_implicit_dynticks_qs()
1119 trace_rcu_future_grace_period(rcu_state.name, rnp->gp_seq, gp_seq_req, in trace_rcu_this_gp()
1191 WRITE_ONCE(rcu_state.gp_flags, rcu_state.gp_flags | RCU_GP_FLAG_INIT); in rcu_start_this_gp()
1192 rcu_state.gp_req_activity = jiffies; in rcu_start_this_gp()
1193 if (!rcu_state.gp_kthread) { in rcu_start_this_gp()
1197 trace_rcu_grace_period(rcu_state.name, READ_ONCE(rcu_state.gp_seq), TPS("newreq")); in rcu_start_this_gp()
1243 if ((current == rcu_state.gp_kthread && in rcu_gp_kthread_wake()
1245 !READ_ONCE(rcu_state.gp_flags) || in rcu_gp_kthread_wake()
1246 !rcu_state.gp_kthread) in rcu_gp_kthread_wake()
1248 WRITE_ONCE(rcu_state.gp_wake_time, jiffies); in rcu_gp_kthread_wake()
1249 WRITE_ONCE(rcu_state.gp_wake_seq, READ_ONCE(rcu_state.gp_seq)); in rcu_gp_kthread_wake()
1250 swake_up_one(&rcu_state.gp_wq); in rcu_gp_kthread_wake()
1287 gp_seq_req = rcu_seq_snap(&rcu_state.gp_seq); in rcu_accelerate_cbs()
1293 trace_rcu_grace_period(rcu_state.name, rdp->gp_seq, TPS("AccWaitCB")); in rcu_accelerate_cbs()
1295 trace_rcu_grace_period(rcu_state.name, rdp->gp_seq, TPS("AccReadyCB")); in rcu_accelerate_cbs()
1313 c = rcu_seq_snap(&rcu_state.gp_seq); in rcu_accelerate_cbs_unlocked()
1394 trace_rcu_grace_period(rcu_state.name, rdp->gp_seq, TPS("cpuend")); in __note_gp_changes()
1408 trace_rcu_grace_period(rcu_state.name, rnp->gp_seq, TPS("cpustart")); in __note_gp_changes()
1445 !(rcu_seq_ctr(rcu_state.gp_seq) % in rcu_gp_slow()
1461 WRITE_ONCE(rcu_state.gp_activity, jiffies); in rcu_gp_init()
1463 if (!READ_ONCE(rcu_state.gp_flags)) { in rcu_gp_init()
1468 WRITE_ONCE(rcu_state.gp_flags, 0); /* Clear all flags: New GP. */ in rcu_gp_init()
1482 rcu_seq_start(&rcu_state.gp_seq); in rcu_gp_init()
1483 trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, TPS("start")); in rcu_gp_init()
1492 rcu_state.gp_state = RCU_GP_ONOFF; in rcu_gp_init()
1494 raw_spin_lock(&rcu_state.ofl_lock); in rcu_gp_init()
1500 raw_spin_unlock(&rcu_state.ofl_lock); in rcu_gp_init()
1536 raw_spin_unlock(&rcu_state.ofl_lock); in rcu_gp_init()
1552 rcu_state.gp_state = RCU_GP_INIT; in rcu_gp_init()
1559 WRITE_ONCE(rnp->gp_seq, rcu_state.gp_seq); in rcu_gp_init()
1563 trace_rcu_grace_period_init(rcu_state.name, rnp->gp_seq, in rcu_gp_init()
1574 WRITE_ONCE(rcu_state.gp_activity, jiffies); in rcu_gp_init()
1589 *gfp = READ_ONCE(rcu_state.gp_flags); in rcu_gp_fqs_check_wake()
1607 WRITE_ONCE(rcu_state.gp_activity, jiffies); in rcu_gp_fqs()
1608 WRITE_ONCE(rcu_state.n_force_qs, rcu_state.n_force_qs + 1); in rcu_gp_fqs()
1617 if (READ_ONCE(rcu_state.gp_flags) & RCU_GP_FLAG_FQS) { in rcu_gp_fqs()
1619 WRITE_ONCE(rcu_state.gp_flags, in rcu_gp_fqs()
1620 READ_ONCE(rcu_state.gp_flags) & ~RCU_GP_FLAG_FQS); in rcu_gp_fqs()
1641 rcu_state.jiffies_force_qs = jiffies + j; in rcu_gp_fqs_loop()
1642 WRITE_ONCE(rcu_state.jiffies_kick_kthreads, in rcu_gp_fqs_loop()
1645 trace_rcu_grace_period(rcu_state.name, in rcu_gp_fqs_loop()
1646 READ_ONCE(rcu_state.gp_seq), in rcu_gp_fqs_loop()
1648 rcu_state.gp_state = RCU_GP_WAIT_FQS; in rcu_gp_fqs_loop()
1650 rcu_state.gp_wq, rcu_gp_fqs_check_wake(&gf), j); in rcu_gp_fqs_loop()
1651 rcu_state.gp_state = RCU_GP_DOING_FQS; in rcu_gp_fqs_loop()
1658 if (ULONG_CMP_GE(jiffies, rcu_state.jiffies_force_qs) || in rcu_gp_fqs_loop()
1660 trace_rcu_grace_period(rcu_state.name, in rcu_gp_fqs_loop()
1661 READ_ONCE(rcu_state.gp_seq), in rcu_gp_fqs_loop()
1665 trace_rcu_grace_period(rcu_state.name, in rcu_gp_fqs_loop()
1666 READ_ONCE(rcu_state.gp_seq), in rcu_gp_fqs_loop()
1669 WRITE_ONCE(rcu_state.gp_activity, jiffies); in rcu_gp_fqs_loop()
1675 WRITE_ONCE(rcu_state.gp_activity, jiffies); in rcu_gp_fqs_loop()
1677 trace_rcu_grace_period(rcu_state.name, in rcu_gp_fqs_loop()
1678 READ_ONCE(rcu_state.gp_seq), in rcu_gp_fqs_loop()
1682 if (time_after(jiffies, rcu_state.jiffies_force_qs)) in rcu_gp_fqs_loop()
1685 j = rcu_state.jiffies_force_qs - j; in rcu_gp_fqs_loop()
1703 WRITE_ONCE(rcu_state.gp_activity, jiffies); in rcu_gp_cleanup()
1705 rcu_state.gp_end = jiffies; in rcu_gp_cleanup()
1706 gp_duration = rcu_state.gp_end - rcu_state.gp_start; in rcu_gp_cleanup()
1707 if (gp_duration > rcu_state.gp_max) in rcu_gp_cleanup()
1708 rcu_state.gp_max = gp_duration; in rcu_gp_cleanup()
1729 new_gp_seq = rcu_state.gp_seq; in rcu_gp_cleanup()
1746 WRITE_ONCE(rcu_state.gp_activity, jiffies); in rcu_gp_cleanup()
1753 trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, TPS("end")); in rcu_gp_cleanup()
1754 rcu_seq_end(&rcu_state.gp_seq); in rcu_gp_cleanup()
1755 rcu_state.gp_state = RCU_GP_IDLE; in rcu_gp_cleanup()
1767 WRITE_ONCE(rcu_state.gp_flags, RCU_GP_FLAG_INIT); in rcu_gp_cleanup()
1768 rcu_state.gp_req_activity = jiffies; in rcu_gp_cleanup()
1769 trace_rcu_grace_period(rcu_state.name, in rcu_gp_cleanup()
1770 READ_ONCE(rcu_state.gp_seq), in rcu_gp_cleanup()
1773 WRITE_ONCE(rcu_state.gp_flags, in rcu_gp_cleanup()
1774 rcu_state.gp_flags & RCU_GP_FLAG_INIT); in rcu_gp_cleanup()
1789 trace_rcu_grace_period(rcu_state.name, in rcu_gp_kthread()
1790 READ_ONCE(rcu_state.gp_seq), in rcu_gp_kthread()
1792 rcu_state.gp_state = RCU_GP_WAIT_GPS; in rcu_gp_kthread()
1793 swait_event_idle_exclusive(rcu_state.gp_wq, in rcu_gp_kthread()
1794 READ_ONCE(rcu_state.gp_flags) & in rcu_gp_kthread()
1796 rcu_state.gp_state = RCU_GP_DONE_GPS; in rcu_gp_kthread()
1801 WRITE_ONCE(rcu_state.gp_activity, jiffies); in rcu_gp_kthread()
1803 trace_rcu_grace_period(rcu_state.name, in rcu_gp_kthread()
1804 READ_ONCE(rcu_state.gp_seq), in rcu_gp_kthread()
1812 rcu_state.gp_state = RCU_GP_CLEANUP; in rcu_gp_kthread()
1814 rcu_state.gp_state = RCU_GP_CLEANED; in rcu_gp_kthread()
1832 WRITE_ONCE(rcu_state.gp_flags, in rcu_report_qs_rsp()
1833 READ_ONCE(rcu_state.gp_flags) | RCU_GP_FLAG_FQS); in rcu_report_qs_rsp()
1876 trace_rcu_quiescent_state_report(rcu_state.name, rnp->gp_seq, in rcu_report_qs_rnp()
2046 trace_rcu_grace_period(rcu_state.name, rnp->gp_seq, in rcutree_dying_cpu()
2133 trace_rcu_batch_start(rcu_state.name, in rcu_do_batch()
2136 trace_rcu_batch_end(rcu_state.name, 0, in rcu_do_batch()
2155 trace_rcu_batch_start(rcu_state.name, in rcu_do_batch()
2167 if (__rcu_reclaim(rcu_state.name, rhp)) in rcu_do_batch()
2197 trace_rcu_batch_end(rcu_state.name, count, !!rcl.head, need_resched(), in rcu_do_batch()
2213 rdp->n_force_qs_snap = READ_ONCE(rcu_state.n_force_qs); in rcu_do_batch()
2324 ret = (READ_ONCE(rcu_state.gp_flags) & RCU_GP_FLAG_FQS) || in rcu_force_quiescent_state()
2337 if (READ_ONCE(rcu_state.gp_flags) & RCU_GP_FLAG_FQS) { in rcu_force_quiescent_state()
2341 WRITE_ONCE(rcu_state.gp_flags, in rcu_force_quiescent_state()
2342 READ_ONCE(rcu_state.gp_flags) | RCU_GP_FLAG_FQS); in rcu_force_quiescent_state()
2541 if (READ_ONCE(rcu_state.n_force_qs) == rdp->n_force_qs_snap && in __call_rcu_core()
2544 rdp->n_force_qs_snap = READ_ONCE(rcu_state.n_force_qs); in __call_rcu_core()
2604 trace_rcu_kfree_callback(rcu_state.name, head, in __call_rcu()
2609 trace_rcu_callback(rcu_state.name, head, in __call_rcu()
2764 return rcu_seq_snap(&rcu_state.gp_seq); in get_state_synchronize_rcu()
2784 if (!rcu_seq_done(&rcu_state.gp_seq, oldstate)) in cond_synchronize_rcu()
2845 trace_rcu_barrier(rcu_state.name, s, cpu, in rcu_barrier_trace()
2846 atomic_read(&rcu_state.barrier_cpu_count), done); in rcu_barrier_trace()
2855 if (atomic_dec_and_test(&rcu_state.barrier_cpu_count)) { in rcu_barrier_callback()
2857 rcu_state.barrier_sequence); in rcu_barrier_callback()
2858 complete(&rcu_state.barrier_completion); in rcu_barrier_callback()
2860 rcu_barrier_trace(TPS("CB"), -1, rcu_state.barrier_sequence); in rcu_barrier_callback()
2871 rcu_barrier_trace(TPS("IRQ"), -1, rcu_state.barrier_sequence); in rcu_barrier_func()
2877 atomic_inc(&rcu_state.barrier_cpu_count); in rcu_barrier_func()
2881 rcu_state.barrier_sequence); in rcu_barrier_func()
2898 unsigned long s = rcu_seq_snap(&rcu_state.barrier_sequence); in rcu_barrier()
2903 mutex_lock(&rcu_state.barrier_mutex); in rcu_barrier()
2906 if (rcu_seq_done(&rcu_state.barrier_sequence, s)) { in rcu_barrier()
2908 rcu_state.barrier_sequence); in rcu_barrier()
2910 mutex_unlock(&rcu_state.barrier_mutex); in rcu_barrier()
2915 rcu_seq_start(&rcu_state.barrier_sequence); in rcu_barrier()
2916 rcu_barrier_trace(TPS("Inc1"), -1, rcu_state.barrier_sequence); in rcu_barrier()
2924 init_completion(&rcu_state.barrier_completion); in rcu_barrier()
2925 atomic_set(&rcu_state.barrier_cpu_count, 1); in rcu_barrier()
2940 rcu_state.barrier_sequence); in rcu_barrier()
2944 rcu_state.barrier_sequence); in rcu_barrier()
2953 if (atomic_dec_and_test(&rcu_state.barrier_cpu_count)) in rcu_barrier()
2954 complete(&rcu_state.barrier_completion); in rcu_barrier()
2957 wait_for_completion(&rcu_state.barrier_completion); in rcu_barrier()
2960 rcu_barrier_trace(TPS("Inc2"), -1, rcu_state.barrier_sequence); in rcu_barrier()
2961 rcu_seq_end(&rcu_state.barrier_sequence); in rcu_barrier()
2964 mutex_unlock(&rcu_state.barrier_mutex); in rcu_barrier()
3008 rdp->rcu_ofl_gp_seq = rcu_state.gp_seq; in rcu_boot_init_percpu_data()
3010 rdp->rcu_onl_gp_seq = rcu_state.gp_seq; in rcu_boot_init_percpu_data()
3035 rdp->n_force_qs_snap = READ_ONCE(rcu_state.n_force_qs); in rcutree_prepare_cpu()
3058 trace_rcu_grace_period(rcu_state.name, rdp->gp_seq, TPS("cpuonl")); in rcutree_prepare_cpu()
3155 smp_store_release(&rcu_state.ncpus, rcu_state.ncpus + nbits); /* ^^^ */ in rcu_cpu_starting()
3157 rdp->rcu_onl_gp_seq = READ_ONCE(rcu_state.gp_seq); in rcu_cpu_starting()
3158 rdp->rcu_onl_gp_flags = READ_ONCE(rcu_state.gp_flags); in rcu_cpu_starting()
3191 raw_spin_lock(&rcu_state.ofl_lock); in rcu_report_dead()
3193 rdp->rcu_ofl_gp_seq = READ_ONCE(rcu_state.gp_seq); in rcu_report_dead()
3194 rdp->rcu_ofl_gp_flags = READ_ONCE(rcu_state.gp_flags); in rcu_report_dead()
3202 raw_spin_unlock(&rcu_state.ofl_lock); in rcu_report_dead()
3306 t = kthread_create(rcu_gp_kthread, NULL, "%s", rcu_state.name); in rcu_spawn_gp_kthread()
3315 rcu_state.gp_kthread = t; in rcu_spawn_gp_kthread()
3369 rcu_state.level[i] = in rcu_init_one()
3370 rcu_state.level[i - 1] + num_rcu_lvl[i - 1]; in rcu_init_one()
3377 rnp = rcu_state.level[i]; in rcu_init_one()
3385 rnp->gp_seq = rcu_state.gp_seq; in rcu_init_one()
3386 rnp->gp_seq_needed = rcu_state.gp_seq; in rcu_init_one()
3387 rnp->completedqs = rcu_state.gp_seq; in rcu_init_one()
3401 rnp->parent = rcu_state.level[i - 1] + in rcu_init_one()
3415 init_swait_queue_head(&rcu_state.gp_wq); in rcu_init_one()
3416 init_swait_queue_head(&rcu_state.expedited_wq); in rcu_init_one()