| /kernel/rcu/ |
| D | rcu_segcblist.c | 32 void rcu_cblist_enqueue(struct rcu_cblist *rclp, struct rcu_head *rhp) in rcu_cblist_enqueue() argument
34 *rclp->tail = rhp; in rcu_cblist_enqueue()
35 rclp->tail = &rhp->next; in rcu_cblist_enqueue()
49 struct rcu_head *rhp) in rcu_cblist_flush_enqueue() argument
58 if (!rhp) { in rcu_cblist_flush_enqueue()
61 rhp->next = NULL; in rcu_cblist_flush_enqueue()
62 srclp->head = rhp; in rcu_cblist_flush_enqueue()
63 srclp->tail = &rhp->next; in rcu_cblist_flush_enqueue()
78 struct rcu_head *rhp; in rcu_cblist_dequeue() local
80 rhp = rclp->head; in rcu_cblist_dequeue()
[all …]
|
| D | srcutiny.c | 113 struct rcu_head *rhp; in srcu_drive_gp() local
135 rhp = lh; in srcu_drive_gp()
138 rhp->func(rhp); in srcu_drive_gp()
158 void call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp, in call_srcu() argument
163 rhp->func = func; in call_srcu()
164 rhp->next = NULL; in call_srcu()
166 *ssp->srcu_cb_tail = rhp; in call_srcu()
167 ssp->srcu_cb_tail = &rhp->next; in call_srcu()
|
| D | rcuperf.c | 347 static void rcu_perf_async_cb(struct rcu_head *rhp) in rcu_perf_async_cb() argument
350 kfree(rhp); in rcu_perf_async_cb()
362 struct rcu_head *rhp = NULL; in rcu_perf_writer() local
404 if (!rhp) in rcu_perf_writer()
405 rhp = kmalloc(sizeof(*rhp), GFP_KERNEL); in rcu_perf_writer()
406 if (rhp && atomic_read(this_cpu_ptr(&n_async_inflight)) < gp_async_max) { in rcu_perf_writer()
409 cur_ops->async(rhp, rcu_perf_async_cb); in rcu_perf_writer()
410 rhp = NULL; in rcu_perf_writer()
416 kfree(rhp); /* Because we are stopping. */ in rcu_perf_writer()
|
| D | srcutree.c | 802 static void srcu_leak_callback(struct rcu_head *rhp) in srcu_leak_callback() argument
834 static void __call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp, in __call_srcu() argument
845 if (debug_rcu_head_queue(rhp)) { in __call_srcu()
847 WRITE_ONCE(rhp->func, srcu_leak_callback); in __call_srcu()
851 rhp->func = func; in __call_srcu()
856 rcu_segcblist_enqueue(&sdp->srcu_cblist, rhp, false); in __call_srcu()
894 void call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp, in call_srcu() argument
897 __call_srcu(ssp, rhp, func, true); in call_srcu()
1006 static void srcu_barrier_cb(struct rcu_head *rhp) in srcu_barrier_cb() argument
1011 sdp = container_of(rhp, struct srcu_data, srcu_barrier_head); in srcu_barrier_cb()
[all …]
|
| D | rcu_segcblist.h | 28 void rcu_cblist_enqueue(struct rcu_cblist *rclp, struct rcu_head *rhp);
31 struct rcu_head *rhp);
109 struct rcu_head *rhp, bool lazy);
111 struct rcu_head *rhp, bool lazy);
|
| D | update.c | 439 void do_trace_rcu_torture_read(const char *rcutorturename, struct rcu_head *rhp, in do_trace_rcu_torture_read() argument
443 trace_rcu_torture_read(rcutorturename, rhp, secs, c_old, c); in do_trace_rcu_torture_read()
447 #define do_trace_rcu_torture_read(rcutorturename, rhp, secs, c_old, c) \ argument
522 void call_rcu_tasks(struct rcu_head *rhp, rcu_callback_t func) in call_rcu_tasks() argument
527 rhp->next = NULL; in call_rcu_tasks()
528 rhp->func = func; in call_rcu_tasks()
531 *rcu_tasks_cbs_tail = rhp; in call_rcu_tasks()
532 rcu_tasks_cbs_tail = &rhp->next; in call_rcu_tasks()
|
| D | sync.c | 43 static void rcu_sync_func(struct rcu_head *rhp);
73 static void rcu_sync_func(struct rcu_head *rhp) in rcu_sync_func() argument
75 struct rcu_sync *rsp = container_of(rhp, struct rcu_sync, cb_head); in rcu_sync_func()
|
| D | rcutorture.c | 1128 static void rcu_torture_timer_cb(struct rcu_head *rhp) in rcu_torture_timer_cb() argument
1130 kfree(rhp); in rcu_torture_timer_cb()
1340 struct rcu_head *rhp = kmalloc(sizeof(*rhp), GFP_NOWAIT); in rcu_torture_timer() local
1342 if (rhp) in rcu_torture_timer()
1343 cur_ops->call(rhp, rcu_torture_timer_cb); in rcu_torture_timer()
1644 static void rcu_torture_fwd_prog_cb(struct rcu_head *rhp) in rcu_torture_fwd_prog_cb() argument
1646 struct fwd_cb_state *fcsp = container_of(rhp, struct fwd_cb_state, rh); in rcu_torture_fwd_prog_cb()
1703 static void rcu_torture_fwd_cb_cr(struct rcu_head *rhp) in rcu_torture_fwd_cb_cr() argument
1707 struct rcu_fwd_cb *rfcp = container_of(rhp, struct rcu_fwd_cb, rh); in rcu_torture_fwd_cb_cr()
2260 static void rcu_torture_leak_cb(struct rcu_head *rhp) in rcu_torture_leak_cb() argument
[all …]
|
| D | tree_plugin.h | 1690 static bool rcu_nocb_do_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp, in rcu_nocb_do_flush_bypass() argument
1698 if (rhp && !rcu_cblist_n_cbs(&rdp->nocb_bypass)) { in rcu_nocb_do_flush_bypass()
1703 if (rhp) in rcu_nocb_do_flush_bypass()
1705 rcu_cblist_flush_enqueue(&rcl, &rdp->nocb_bypass, rhp); in rcu_nocb_do_flush_bypass()
1720 static bool rcu_nocb_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp, in rcu_nocb_flush_bypass() argument
1727 return rcu_nocb_do_flush_bypass(rdp, rhp, j); in rcu_nocb_flush_bypass()
1761 static bool rcu_nocb_try_bypass(struct rcu_data *rdp, struct rcu_head *rhp, in rcu_nocb_try_bypass() argument
1817 if (!rcu_nocb_flush_bypass(rdp, rhp, j)) { in rcu_nocb_try_bypass()
1840 rcu_cblist_enqueue(&rdp->nocb_bypass, rhp); in rcu_nocb_try_bypass()
2485 static bool rcu_nocb_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp, in rcu_nocb_flush_bypass() argument
[all …]
|
| D | rcu.h | 460 struct rcu_head *rhp,
474 struct rcu_head *rhp,
479 #define do_trace_rcu_torture_read(rcutorturename, rhp, secs, c_old, c) \ argument
|
| D | tree.h | 434 static bool rcu_nocb_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp,
436 static bool rcu_nocb_try_bypass(struct rcu_data *rdp, struct rcu_head *rhp,
|
| D | tree.c | 2116 struct rcu_head *rhp; in rcu_do_batch() local
2154 rhp = rcu_cblist_dequeue(&rcl); in rcu_do_batch()
2155 for (; rhp; rhp = rcu_cblist_dequeue(&rcl)) { in rcu_do_batch()
2156 debug_rcu_head_unqueue(rhp); in rcu_do_batch()
2157 if (__rcu_reclaim(rcu_state.name, rhp)) in rcu_do_batch()
2544 static void rcu_leak_callback(struct rcu_head *rhp) in rcu_leak_callback() argument
2844 static void rcu_barrier_callback(struct rcu_head *rhp) in rcu_barrier_callback() argument
|
| /kernel/ |
| D | pid.c | 118 static void delayed_put_pid(struct rcu_head *rhp) in delayed_put_pid() argument
120 struct pid *pid = container_of(rhp, struct pid, rcu); in delayed_put_pid()
|
| D | exit.c | 176 static void delayed_put_task_struct(struct rcu_head *rhp) in delayed_put_task_struct() argument
178 struct task_struct *tsk = container_of(rhp, struct task_struct, rcu); in delayed_put_task_struct()
|
| D | fork.c | 1748 static void __delayed_free_task(struct rcu_head *rhp) in __delayed_free_task() argument
1750 struct task_struct *tsk = container_of(rhp, struct task_struct, rcu); in __delayed_free_task()
|
| /kernel/irq/ |
| D | irqdesc.c | 431 static void delayed_free_desc(struct rcu_head *rhp) in delayed_free_desc() argument
433 struct irq_desc *desc = container_of(rhp, struct irq_desc, rcu); in delayed_free_desc()
|
| /kernel/sched/ |
| D | core.c | 6982 static void sched_free_group_rcu(struct rcu_head *rhp) in sched_free_group_rcu() argument
6985 sched_free_group(container_of(rhp, struct task_group, rcu)); in sched_free_group_rcu()
|