Lines Matching refs:rdp
170 struct rcu_data *rdp = &per_cpu(rcu_preempt_data, cpu); in rcu_preempt_qs() local
172 if (rdp->passed_quiesce == 0) in rcu_preempt_qs()
173 trace_rcu_grace_period("rcu_preempt", rdp->gpnum, "cpuqs"); in rcu_preempt_qs()
174 rdp->passed_quiesce = 1; in rcu_preempt_qs()
195 struct rcu_data *rdp; in rcu_preempt_note_context_switch() local
202 rdp = per_cpu_ptr(rcu_preempt_state.rda, cpu); in rcu_preempt_note_context_switch()
203 rnp = rdp->mynode; in rcu_preempt_note_context_switch()
226 WARN_ON_ONCE((rdp->grpmask & rnp->qsmaskinit) == 0); in rcu_preempt_note_context_switch()
228 if ((rnp->qsmask & rdp->grpmask) && rnp->gp_tasks != NULL) { in rcu_preempt_note_context_switch()
237 if (rnp->qsmask & rdp->grpmask) in rcu_preempt_note_context_switch()
240 trace_rcu_preempt_task(rdp->rsp->name, in rcu_preempt_note_context_switch()
242 (rnp->qsmask & rdp->grpmask) in rcu_preempt_note_context_switch()
572 struct rcu_data *rdp) in rcu_preempt_offline_tasks() argument
586 WARN_ON_ONCE(rnp != rdp->mynode); in rcu_preempt_offline_tasks()
1032 struct rcu_data *rdp) in rcu_preempt_offline_tasks() argument
1487 struct rcu_data *rdp = per_cpu_ptr(rcu_state->rda, cpu); in rcu_prepare_kthreads() local
1488 struct rcu_node *rnp = rdp->mynode; in rcu_prepare_kthreads()
1617 struct rcu_data *rdp; in rcu_try_advance_all_cbs() local
1622 rdp = this_cpu_ptr(rsp->rda); in rcu_try_advance_all_cbs()
1623 rnp = rdp->mynode; in rcu_try_advance_all_cbs()
1630 if (rdp->completed != rnp->completed && in rcu_try_advance_all_cbs()
1631 rdp->nxttail[RCU_DONE_TAIL] != rdp->nxttail[RCU_NEXT_TAIL]) in rcu_try_advance_all_cbs()
1632 rcu_process_gp_end(rsp, rdp); in rcu_try_advance_all_cbs()
1634 if (cpu_has_callbacks_ready_to_invoke(rdp)) in rcu_try_advance_all_cbs()
1691 struct rcu_data *rdp; in rcu_prepare_for_idle() local
1731 rdp = per_cpu_ptr(rsp->rda, cpu); in rcu_prepare_for_idle()
1732 if (!*rdp->nxttail[RCU_DONE_TAIL]) in rcu_prepare_for_idle()
1734 rnp = rdp->mynode; in rcu_prepare_for_idle()
1736 rcu_accelerate_cbs(rsp, rnp, rdp); in rcu_prepare_for_idle()
1748 struct rcu_data *rdp; in rcu_cleanup_after_idle() local
1755 rdp = per_cpu_ptr(rsp->rda, cpu); in rcu_cleanup_after_idle()
1756 if (cpu_has_callbacks_ready_to_invoke(rdp)) in rcu_cleanup_after_idle()
1800 struct rcu_data *rdp; in rcu_oom_notify_cpu() local
1803 rdp = __this_cpu_ptr(rsp->rda); in rcu_oom_notify_cpu()
1804 if (rdp->qlen_lazy != 0) { in rcu_oom_notify_cpu()
1806 rsp->call(&rdp->oom_head, rcu_oom_callback); in rcu_oom_notify_cpu()
1904 struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu); in print_cpu_stall_info() local
1905 struct rcu_dynticks *rdtp = rdp->dynticks; in print_cpu_stall_info()
1909 if (rsp->gpnum == rdp->gpnum) { in print_cpu_stall_info()
1911 ticks_value = rdp->ticks_this_gp; in print_cpu_stall_info()
1914 ticks_value = rsp->gpnum - rdp->gpnum; in print_cpu_stall_info()
1921 rdp->softirq_snap, kstat_softirqs_cpu(RCU_SOFTIRQ, cpu), in print_cpu_stall_info()
1932 static void zero_cpu_stall_ticks(struct rcu_data *rdp) in zero_cpu_stall_ticks() argument
1934 rdp->ticks_this_gp = 0; in zero_cpu_stall_ticks()
1935 rdp->softirq_snap = kstat_softirqs_cpu(RCU_SOFTIRQ, smp_processor_id()); in zero_cpu_stall_ticks()
1964 static void zero_cpu_stall_ticks(struct rcu_data *rdp) in zero_cpu_stall_ticks() argument
2071 static void __call_rcu_nocb_enqueue(struct rcu_data *rdp, in __call_rcu_nocb_enqueue() argument
2081 old_rhpp = xchg(&rdp->nocb_tail, rhtp); in __call_rcu_nocb_enqueue()
2083 atomic_long_add(rhcount, &rdp->nocb_q_count); in __call_rcu_nocb_enqueue()
2084 atomic_long_add(rhcount_lazy, &rdp->nocb_q_count_lazy); in __call_rcu_nocb_enqueue()
2087 t = ACCESS_ONCE(rdp->nocb_kthread); in __call_rcu_nocb_enqueue()
2090 len = atomic_long_read(&rdp->nocb_q_count); in __call_rcu_nocb_enqueue()
2091 if (old_rhpp == &rdp->nocb_head) { in __call_rcu_nocb_enqueue()
2092 wake_up(&rdp->nocb_wq); /* ... only if queue was empty ... */ in __call_rcu_nocb_enqueue()
2093 rdp->qlen_last_fqs_check = 0; in __call_rcu_nocb_enqueue()
2094 } else if (len > rdp->qlen_last_fqs_check + qhimark) { in __call_rcu_nocb_enqueue()
2096 rdp->qlen_last_fqs_check = LONG_MAX / 2; in __call_rcu_nocb_enqueue()
2110 static bool __call_rcu_nocb(struct rcu_data *rdp, struct rcu_head *rhp, in __call_rcu_nocb() argument
2114 if (!rcu_is_nocb_cpu(rdp->cpu)) in __call_rcu_nocb()
2116 __call_rcu_nocb_enqueue(rdp, rhp, &rhp->next, 1, lazy); in __call_rcu_nocb()
2118 trace_rcu_kfree_callback(rdp->rsp->name, rhp, in __call_rcu_nocb()
2120 rdp->qlen_lazy, rdp->qlen); in __call_rcu_nocb()
2122 trace_rcu_callback(rdp->rsp->name, rhp, in __call_rcu_nocb()
2123 rdp->qlen_lazy, rdp->qlen); in __call_rcu_nocb()
2132 struct rcu_data *rdp) in rcu_nocb_adopt_orphan_cbs() argument
2145 __call_rcu_nocb_enqueue(rdp, rsp->orphan_donelist, in rcu_nocb_adopt_orphan_cbs()
2152 __call_rcu_nocb_enqueue(rdp, rsp->orphan_nxtlist, in rcu_nocb_adopt_orphan_cbs()
2165 static void rcu_nocb_wait_gp(struct rcu_data *rdp) in rcu_nocb_wait_gp() argument
2170 struct rcu_node *rnp = rdp->mynode; in rcu_nocb_wait_gp()
2173 c = rcu_start_future_gp(rnp, rdp); in rcu_nocb_wait_gp()
2180 trace_rcu_future_gp(rnp, rdp, c, "StartWait"); in rcu_nocb_wait_gp()
2188 trace_rcu_future_gp(rnp, rdp, c, "ResumeWait"); in rcu_nocb_wait_gp()
2190 trace_rcu_future_gp(rnp, rdp, c, "EndWait"); in rcu_nocb_wait_gp()
2204 struct rcu_data *rdp = arg; in rcu_nocb_kthread() local
2210 wait_event_interruptible(rdp->nocb_wq, rdp->nocb_head); in rcu_nocb_kthread()
2211 list = ACCESS_ONCE(rdp->nocb_head); in rcu_nocb_kthread()
2222 ACCESS_ONCE(rdp->nocb_head) = NULL; in rcu_nocb_kthread()
2223 tail = xchg(&rdp->nocb_tail, &rdp->nocb_head); in rcu_nocb_kthread()
2224 c = atomic_long_xchg(&rdp->nocb_q_count, 0); in rcu_nocb_kthread()
2225 cl = atomic_long_xchg(&rdp->nocb_q_count_lazy, 0); in rcu_nocb_kthread()
2226 ACCESS_ONCE(rdp->nocb_p_count) += c; in rcu_nocb_kthread()
2227 ACCESS_ONCE(rdp->nocb_p_count_lazy) += cl; in rcu_nocb_kthread()
2228 rcu_nocb_wait_gp(rdp); in rcu_nocb_kthread()
2231 trace_rcu_batch_start(rdp->rsp->name, cl, c, -1); in rcu_nocb_kthread()
2242 if (__rcu_reclaim(rdp->rsp->name, list)) in rcu_nocb_kthread()
2248 trace_rcu_batch_end(rdp->rsp->name, c, !!list, 0, 0, 1); in rcu_nocb_kthread()
2249 ACCESS_ONCE(rdp->nocb_p_count) -= c; in rcu_nocb_kthread()
2250 ACCESS_ONCE(rdp->nocb_p_count_lazy) -= cl; in rcu_nocb_kthread()
2251 rdp->n_nocbs_invoked += c; in rcu_nocb_kthread()
2257 static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp) in rcu_boot_init_nocb_percpu_data() argument
2259 rdp->nocb_tail = &rdp->nocb_head; in rcu_boot_init_nocb_percpu_data()
2260 init_waitqueue_head(&rdp->nocb_wq); in rcu_boot_init_nocb_percpu_data()
2267 struct rcu_data *rdp; in rcu_spawn_nocb_kthreads() local
2273 rdp = per_cpu_ptr(rsp->rda, cpu); in rcu_spawn_nocb_kthreads()
2274 t = kthread_run(rcu_nocb_kthread, rdp, in rcu_spawn_nocb_kthreads()
2277 ACCESS_ONCE(rdp->nocb_kthread) = t; in rcu_spawn_nocb_kthreads()
2282 static bool init_nocb_callback_list(struct rcu_data *rdp) in init_nocb_callback_list() argument
2285 !cpumask_test_cpu(rdp->cpu, rcu_nocb_mask)) in init_nocb_callback_list()
2287 rdp->nxttail[RCU_NEXT_TAIL] = NULL; in init_nocb_callback_list()
2310 static bool __call_rcu_nocb(struct rcu_data *rdp, struct rcu_head *rhp, in __call_rcu_nocb() argument
2317 struct rcu_data *rdp) in rcu_nocb_adopt_orphan_cbs() argument
2322 static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp) in rcu_boot_init_nocb_percpu_data() argument
2330 static bool init_nocb_callback_list(struct rcu_data *rdp) in init_nocb_callback_list() argument