/kernel/rcu/ |
D | rcu_segcblist.c | 28 void rcu_cblist_enqueue(struct rcu_cblist *rclp, struct rcu_head *rhp) in rcu_cblist_enqueue() argument 30 *rclp->tail = rhp; in rcu_cblist_enqueue() 31 rclp->tail = &rhp->next; in rcu_cblist_enqueue() 45 struct rcu_head *rhp) in rcu_cblist_flush_enqueue() argument 53 if (!rhp) { in rcu_cblist_flush_enqueue() 56 rhp->next = NULL; in rcu_cblist_flush_enqueue() 57 srclp->head = rhp; in rcu_cblist_flush_enqueue() 58 srclp->tail = &rhp->next; in rcu_cblist_flush_enqueue() 69 struct rcu_head *rhp; in rcu_cblist_dequeue() local 71 rhp = rclp->head; in rcu_cblist_dequeue() [all …]
|
D | srcutiny.c | 116 struct rcu_head *rhp; in srcu_drive_gp() local 139 rhp = lh; in srcu_drive_gp() 142 rhp->func(rhp); in srcu_drive_gp() 178 void call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp, in call_srcu() argument 183 rhp->func = func; in call_srcu() 184 rhp->next = NULL; in call_srcu() 186 *ssp->srcu_cb_tail = rhp; in call_srcu() 187 ssp->srcu_cb_tail = &rhp->next; in call_srcu()
|
D | tasks.h | 284 static void call_rcu_tasks_generic(struct rcu_head *rhp, rcu_callback_t func, in call_rcu_tasks_generic() argument 295 rhp->next = NULL; in call_rcu_tasks_generic() 296 rhp->func = func; in call_rcu_tasks_generic() 319 rcu_segcblist_enqueue(&rtpcp->cblist, rhp); in call_rcu_tasks_generic() 338 static void rcu_barrier_tasks_generic_cb(struct rcu_head *rhp) in rcu_barrier_tasks_generic_cb() argument 343 rtpcp = container_of(rhp, struct rcu_tasks_percpu, barrier_q_head); in rcu_barrier_tasks_generic_cb() 464 struct rcu_head *rhp; in rcu_tasks_invoke_cbs() local 489 for (rhp = rcu_cblist_dequeue(&rcl); rhp; rhp = rcu_cblist_dequeue(&rcl)) { in rcu_tasks_invoke_cbs() 491 rhp->func(rhp); in rcu_tasks_invoke_cbs() 929 void call_rcu_tasks(struct rcu_head *rhp, rcu_callback_t func); [all …]
|
D | rcutorture.c | 1661 static void rcu_torture_timer_cb(struct rcu_head *rhp) in rcu_torture_timer_cb() argument 1663 kfree(rhp); in rcu_torture_timer_cb() 2014 struct rcu_head *rhp = kmalloc(sizeof(*rhp), GFP_NOWAIT); in rcu_torture_timer() local 2016 if (rhp) in rcu_torture_timer() 2017 cur_ops->call(rhp, rcu_torture_timer_cb); in rcu_torture_timer() 2255 struct rcu_head *rhp; in rcu_torture_mem_dump_obj() local 2262 rhp = kmem_cache_alloc(kcp, GFP_KERNEL); in rcu_torture_mem_dump_obj() 2263 if (WARN_ON_ONCE(!rhp)) { in rcu_torture_mem_dump_obj() 2267 …slab test: rcu_torture_stats = %px, &rhp = %px, rhp = %px, &z = %px\n", stats_task, &rhp, rhp, &z); in rcu_torture_mem_dump_obj() 2272 pr_alert("mem_dump_obj(%px):", &rhp); in rcu_torture_mem_dump_obj() [all …]
|
D | update.c | 585 void do_trace_rcu_torture_read(const char *rcutorturename, struct rcu_head *rhp, in do_trace_rcu_torture_read() argument 589 trace_rcu_torture_read(rcutorturename, rhp, secs, c_old, c); in do_trace_rcu_torture_read() 593 #define do_trace_rcu_torture_read(rcutorturename, rhp, secs, c_old, c) \ argument 668 struct early_boot_kfree_rcu *rhp; in early_boot_test_call_rcu() local 675 rhp = kmalloc(sizeof(*rhp), GFP_KERNEL); in early_boot_test_call_rcu() 676 if (!WARN_ON_ONCE(!rhp)) in early_boot_test_call_rcu() 677 kfree_rcu(rhp, rh); in early_boot_test_call_rcu()
|
D | srcutree.c | 1074 static void srcu_leak_callback(struct rcu_head *rhp) in srcu_leak_callback() argument 1082 struct rcu_head *rhp, bool do_norm) in srcu_gp_start_if_needed() argument 1101 if (rhp) in srcu_gp_start_if_needed() 1102 rcu_segcblist_enqueue(&sdp->srcu_cblist, rhp); in srcu_gp_start_if_needed() 1133 WARN_ON_ONCE(!rcu_segcblist_accelerate(&sdp->srcu_cblist, s) && rhp); in srcu_gp_start_if_needed() 1186 static void __call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp, in __call_srcu() argument 1189 if (debug_rcu_head_queue(rhp)) { in __call_srcu() 1191 WRITE_ONCE(rhp->func, srcu_leak_callback); in __call_srcu() 1195 rhp->func = func; in __call_srcu() 1196 (void)srcu_gp_start_if_needed(ssp, rhp, do_norm); in __call_srcu() [all …]
|
D | rcuscale.c | 398 static void rcu_scale_async_cb(struct rcu_head *rhp) in rcu_scale_async_cb() argument 401 kfree(rhp); in rcu_scale_async_cb() 413 struct rcu_head *rhp = NULL; in rcu_scale_writer() local 454 if (!rhp) in rcu_scale_writer() 455 rhp = kmalloc(sizeof(*rhp), GFP_KERNEL); in rcu_scale_writer() 456 if (rhp && atomic_read(this_cpu_ptr(&n_async_inflight)) < gp_async_max) { in rcu_scale_writer() 458 cur_ops->async(rhp, rcu_scale_async_cb); in rcu_scale_writer() 459 rhp = NULL; in rcu_scale_writer() 464 kfree(rhp); /* Because we are stopping. */ in rcu_scale_writer()
|
D | rcu_segcblist.h | 22 void rcu_cblist_enqueue(struct rcu_cblist *rclp, struct rcu_head *rhp); 25 struct rcu_head *rhp); 137 struct rcu_head *rhp); 139 struct rcu_head *rhp);
|
D | sync.c | 43 static void rcu_sync_func(struct rcu_head *rhp); 73 static void rcu_sync_func(struct rcu_head *rhp) in rcu_sync_func() argument 75 struct rcu_sync *rsp = container_of(rhp, struct rcu_sync, cb_head); in rcu_sync_func()
|
D | tree_nocb.h | 334 struct rcu_head *rhp = rhp_in; in rcu_nocb_do_flush_bypass() local 339 if (rhp && !rcu_cblist_n_cbs(&rdp->nocb_bypass)) { in rcu_nocb_do_flush_bypass() 344 if (rhp) in rcu_nocb_do_flush_bypass() 353 if (lazy && rhp) { in rcu_nocb_do_flush_bypass() 354 rcu_cblist_enqueue(&rdp->nocb_bypass, rhp); in rcu_nocb_do_flush_bypass() 355 rhp = NULL; in rcu_nocb_do_flush_bypass() 357 rcu_cblist_flush_enqueue(&rcl, &rdp->nocb_bypass, rhp); in rcu_nocb_do_flush_bypass() 374 static bool rcu_nocb_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp, in rcu_nocb_flush_bypass() argument 381 return rcu_nocb_do_flush_bypass(rdp, rhp, j, lazy); in rcu_nocb_flush_bypass() 415 static bool rcu_nocb_try_bypass(struct rcu_data *rdp, struct rcu_head *rhp, in rcu_nocb_try_bypass() argument [all …]
|
D | rcu.h | 495 struct rcu_head *rhp, 509 struct rcu_head *rhp, 514 #define do_trace_rcu_torture_read(rcutorturename, rhp, secs, c_old, c) \ argument
|
D | tree.h | 449 static bool rcu_nocb_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp, 451 static bool rcu_nocb_try_bypass(struct rcu_data *rdp, struct rcu_head *rhp,
|
D | tree.c | 2239 struct rcu_head *rhp; in rcu_do_batch() local 2283 rhp = rcu_cblist_dequeue(&rcl); in rcu_do_batch() 2285 for (; rhp; rhp = rcu_cblist_dequeue(&rcl)) { in rcu_do_batch() 2289 debug_rcu_head_unqueue(rhp); in rcu_do_batch() 2292 trace_rcu_invoke_callback(rcu_state.name, rhp); in rcu_do_batch() 2294 f = rhp->func; in rcu_do_batch() 2295 WRITE_ONCE(rhp->func, (rcu_callback_t)0L); in rcu_do_batch() 2296 f(rhp); in rcu_do_batch() 2733 static void rcu_leak_callback(struct rcu_head *rhp) in rcu_leak_callback() argument 3989 static void rcu_barrier_callback(struct rcu_head *rhp) in rcu_barrier_callback() argument
|
D | tiny.c | 161 static void tiny_rcu_leak_callback(struct rcu_head *rhp) in tiny_rcu_leak_callback() argument
|
/kernel/ |
D | pid.c | 120 static void delayed_put_pid(struct rcu_head *rhp) in delayed_put_pid() argument 122 struct pid *pid = container_of(rhp, struct pid, rcu); in delayed_put_pid()
|
D | fork.c | 947 void __put_task_struct_rcu_cb(struct rcu_head *rhp) in __put_task_struct_rcu_cb() argument 949 struct task_struct *task = container_of(rhp, struct task_struct, rcu); in __put_task_struct_rcu_cb() 2049 static void __delayed_free_task(struct rcu_head *rhp) in __delayed_free_task() argument 2051 struct task_struct *tsk = container_of(rhp, struct task_struct, rcu); in __delayed_free_task()
|
D | exit.c | 219 static void delayed_put_task_struct(struct rcu_head *rhp) in delayed_put_task_struct() argument 221 struct task_struct *tsk = container_of(rhp, struct task_struct, rcu); in delayed_put_task_struct()
|
/kernel/irq/ |
D | irqdesc.c | 435 static void delayed_free_desc(struct rcu_head *rhp) in delayed_free_desc() argument 437 struct irq_desc *desc = container_of(rhp, struct irq_desc, rcu); in delayed_free_desc()
|
/kernel/sched/ |
D | core.c | 10405 static void sched_unregister_group_rcu(struct rcu_head *rhp) in sched_unregister_group_rcu() argument 10408 sched_unregister_group(container_of(rhp, struct task_group, rcu)); in sched_unregister_group_rcu()
|