Lines Matching refs:head
2558 trace_rcu_batch_end(rcu_state.name, count, !!rcl.head, need_resched(), in rcu_do_batch()
2888 static void __call_rcu_core(struct rcu_data *rdp, struct rcu_head *head, in __call_rcu_core() argument
2922 rcu_segcblist_first_pend_cb(&rdp->cblist) != head) in __call_rcu_core()
2980 __call_rcu_common(struct rcu_head *head, rcu_callback_t func, bool lazy_in) in __call_rcu_common() argument
2989 WARN_ON_ONCE((unsigned long)head & (sizeof(void *) - 1)); in __call_rcu_common()
2991 if (debug_rcu_head_queue(head)) { in __call_rcu_common()
2998 pr_err("%s(): Double-freed CB %p->%pS()!!! ", __func__, head, head->func); in __call_rcu_common()
2999 mem_dump_obj(head); in __call_rcu_common()
3001 WRITE_ONCE(head->func, rcu_leak_callback); in __call_rcu_common()
3004 head->func = func; in __call_rcu_common()
3005 head->next = NULL; in __call_rcu_common()
3007 kasan_record_aux_stack_noalloc(head); in __call_rcu_common()
3023 if (rcu_nocb_try_bypass(rdp, head, &was_alldone, flags, lazy)) in __call_rcu_common()
3026 rcu_segcblist_enqueue(&rdp->cblist, head); in __call_rcu_common()
3028 trace_rcu_kvfree_callback(rcu_state.name, head, in __call_rcu_common()
3032 trace_rcu_callback(rcu_state.name, head, in __call_rcu_common()
3041 __call_rcu_core(rdp, head, flags); in __call_rcu_common()
3070 void call_rcu_hurry(struct rcu_head *head, rcu_callback_t func) in call_rcu_hurry() argument
3072 return __call_rcu_common(head, func, false); in call_rcu_hurry()
3123 void call_rcu(struct rcu_head *head, rcu_callback_t func) in call_rcu() argument
3125 __call_rcu_common(head, func, enable_rcu_lazy); in call_rcu()
3195 struct rcu_head *head; member
3297 struct rcu_head *head, *next; in kfree_rcu_work() local
3314 head = krwp->head_free; in kfree_rcu_work()
3362 for (; head; head = next) { in kfree_rcu_work()
3363 unsigned long offset = (unsigned long)head->func; in kfree_rcu_work()
3364 void *ptr = (void *)head - offset; in kfree_rcu_work()
3366 next = head->next; in kfree_rcu_work()
3369 trace_rcu_invoke_kvfree_callback(rcu_state.name, head, offset); in kfree_rcu_work()
3388 return !!krcp->head; in need_offload_krc()
3453 krwp->head_free = krcp->head; in kfree_rcu_monitor()
3454 WRITE_ONCE(krcp->head, NULL); in kfree_rcu_monitor()
3613 void kvfree_call_rcu(struct rcu_head *head, rcu_callback_t func) in kvfree_call_rcu() argument
3620 if (head) { in kvfree_call_rcu()
3621 ptr = (void *) head - (unsigned long) func; in kvfree_call_rcu()
3638 __func__, head); in kvfree_call_rcu()
3645 success = add_ptr_to_bulk_krc_lock(&krcp, &flags, ptr, !head); in kvfree_call_rcu()
3649 if (head == NULL) in kvfree_call_rcu()
3653 head->func = func; in kvfree_call_rcu()
3654 head->next = krcp->head; in kvfree_call_rcu()
3655 WRITE_ONCE(krcp->head, head); in kvfree_call_rcu()