| /kernel/ |
| D | irq_work.c | 30 static bool irq_work_claim(struct irq_work *work) in irq_work_claim() argument
34 oflags = atomic_fetch_or(IRQ_WORK_CLAIMED | CSD_TYPE_IRQ_WORK, &work->flags); in irq_work_claim()
53 static void __irq_work_queue_local(struct irq_work *work) in __irq_work_queue_local() argument
56 if (atomic_read(&work->flags) & IRQ_WORK_LAZY) { in __irq_work_queue_local()
57 if (llist_add(&work->llnode, this_cpu_ptr(&lazy_list)) && in __irq_work_queue_local()
61 if (llist_add(&work->llnode, this_cpu_ptr(&raised_list))) in __irq_work_queue_local()
67 bool irq_work_queue(struct irq_work *work) in irq_work_queue() argument
70 if (!irq_work_claim(work)) in irq_work_queue()
75 __irq_work_queue_local(work); in irq_work_queue()
88 bool irq_work_queue_on(struct irq_work *work, int cpu) in irq_work_queue_on() argument
[all …]
|
| D | task_work.c | 32 int task_work_add(struct task_struct *task, struct callback_head *work, in task_work_add() argument
38 kasan_record_aux_stack(work); in task_work_add()
44 work->next = head; in task_work_add()
45 } while (cmpxchg(&task->task_works, head, work) != head); in task_work_add()
79 struct callback_head *work; in task_work_cancel_match() local
91 while ((work = READ_ONCE(*pprev))) { in task_work_cancel_match()
92 if (!match(work, data)) in task_work_cancel_match()
93 pprev = &work->next; in task_work_cancel_match()
94 else if (cmpxchg(pprev, work, work->next) == work) in task_work_cancel_match()
99 return work; in task_work_cancel_match()
[all …]
|
| D | kthread.c | 720 struct kthread_work *work; in kthread_worker_fn() local
743 work = NULL; in kthread_worker_fn()
746 work = list_first_entry(&worker->work_list, in kthread_worker_fn()
748 list_del_init(&work->node); in kthread_worker_fn()
750 worker->current_work = work; in kthread_worker_fn()
753 if (work) { in kthread_worker_fn()
755 work->func(work); in kthread_worker_fn()
861 struct kthread_work *work) in queuing_blocked() argument
865 return !list_empty(&work->node) || work->canceling; in queuing_blocked()
869 struct kthread_work *work) in kthread_insert_work_sanity_check() argument
[all …]
|
| D | workqueue.c | 447 struct work_struct *work = addr; in work_is_static_object() local
449 return test_bit(WORK_STRUCT_STATIC_BIT, work_data_bits(work)); in work_is_static_object()
458 struct work_struct *work = addr; in work_fixup_init() local
462 cancel_work_sync(work); in work_fixup_init()
463 debug_object_init(work, &work_debug_descr); in work_fixup_init()
476 struct work_struct *work = addr; in work_fixup_free() local
480 cancel_work_sync(work); in work_fixup_free()
481 debug_object_free(work, &work_debug_descr); in work_fixup_free()
496 static inline void debug_work_activate(struct work_struct *work) in debug_work_activate() argument
498 debug_object_activate(work, &work_debug_descr); in debug_work_activate()
[all …]
|
| D | stop_machine.c | 60 struct cpu_stop_work *work, in __cpu_stop_queue_work() argument
63 list_add_tail(&work->list, &stopper->works); in __cpu_stop_queue_work()
68 static bool cpu_stop_queue_work(unsigned int cpu, struct cpu_stop_work *work) in cpu_stop_queue_work() argument
79 __cpu_stop_queue_work(stopper, work, &wakeq); in cpu_stop_queue_work()
80 else if (work->done) in cpu_stop_queue_work()
81 cpu_stop_signal_done(work->done); in cpu_stop_queue_work()
117 struct cpu_stop_work work = { .fn = fn, .arg = arg, .done = &done }; in stop_one_cpu() local
120 if (!cpu_stop_queue_work(cpu, &work)) in stop_one_cpu()
418 struct cpu_stop_work *work; in queue_stop_cpus_work() local
431 work = &per_cpu(cpu_stopper.stop_work, cpu); in queue_stop_cpus_work()
[all …]
|
| D | async.c | 70 struct work_struct work; member
109 static void async_run_entry_fn(struct work_struct *work) in async_run_entry_fn() argument
112 container_of(work, struct async_entry, work); in async_run_entry_fn()
158 INIT_WORK(&entry->work, async_run_entry_fn); in __async_schedule_node_domain()
176 queue_work_node(node, system_unbound_wq, &entry->work); in __async_schedule_node_domain()
|
| D | jump_label.c | 259 void jump_label_update_timeout(struct work_struct *work) in jump_label_update_timeout() argument
262 container_of(work, struct static_key_deferred, work.work); in jump_label_update_timeout()
281 struct delayed_work *work, in __static_key_slow_dec_deferred() argument
289 schedule_delayed_work(work, timeout); in __static_key_slow_dec_deferred()
293 void __static_key_deferred_flush(void *key, struct delayed_work *work) in __static_key_deferred_flush() argument
296 flush_delayed_work(work); in __static_key_deferred_flush()
305 INIT_DELAYED_WORK(&key->work, jump_label_update_timeout); in jump_label_rate_limit()
|
| D | umh.c | 158 static void call_usermodehelper_exec_work(struct work_struct *work) in call_usermodehelper_exec_work() argument
161 container_of(work, struct subprocess_info, work); in call_usermodehelper_exec_work()
367 INIT_WORK(&sub_info->work, call_usermodehelper_exec_work); in call_usermodehelper_setup()
433 queue_work(system_unbound_wq, &sub_info->work); in call_usermodehelper_exec()
|
| D | acct.c | 92 struct work_struct work; member
175 schedule_work(&acct->work); in acct_pin_kill()
183 static void close_work(struct work_struct *work) in close_work() argument
185 struct bsd_acct_struct *acct = container_of(work, struct bsd_acct_struct, work); in close_work()
246 INIT_WORK(&acct->work, close_work); in acct_on()
|
| D | smp.c | 999 struct work_struct work; member
1007 static void smp_call_on_cpu_callback(struct work_struct *work) in smp_call_on_cpu_callback() argument
1011 sscs = container_of(work, struct smp_call_on_cpu_struct, work); in smp_call_on_cpu_callback()
1030 INIT_WORK_ONSTACK(&sscs.work, smp_call_on_cpu_callback); in smp_call_on_cpu()
1035 queue_work_on(cpu, system_wq, &sscs.work); in smp_call_on_cpu()
|
| D | padata.c | 61 static void __init padata_mt_helper(struct work_struct *work);
328 queue_work_on(cb_cpu, pinst->serial_wq, &squeue->work); in padata_reorder()
348 static void invoke_padata_reorder(struct work_struct *work) in invoke_padata_reorder() argument
353 pd = container_of(work, struct parallel_data, reorder_work); in invoke_padata_reorder()
366 squeue = container_of(serial_work, struct padata_serial_queue, work); in padata_serial_worker()
554 INIT_WORK(&squeue->work, padata_serial_worker); in padata_init_squeues()
|
| D | user_namespace.c | 30 static void free_user_ns(struct work_struct *work);
121 INIT_WORK(&ns->work, free_user_ns); in create_user_ns()
175 static void free_user_ns(struct work_struct *work) in free_user_ns() argument
178 container_of(work, struct user_namespace, work); in free_user_ns()
206 schedule_work(&ns->work); in __put_user_ns()
|
| /kernel/locking/ |
| D | test-ww_mutex.c | 20 struct work_struct work; member
31 static void test_mutex_work(struct work_struct *work) in test_mutex_work() argument
33 struct test_mutex *mtx = container_of(work, typeof(*mtx), work); in test_mutex_work()
58 INIT_WORK_ONSTACK(&mtx.work, test_mutex_work); in __test_mutex()
64 schedule_work(&mtx.work); in __test_mutex()
92 flush_work(&mtx.work); in __test_mutex()
93 destroy_work_on_stack(&mtx.work); in __test_mutex()
148 struct work_struct work; member
157 static void test_abba_work(struct work_struct *work) in test_abba_work() argument
159 struct test_abba *abba = container_of(work, typeof(*abba), work); in test_abba_work()
[all …]
|
| /kernel/printk/ |
| D | printk_safe.c | 41 struct irq_work work; /* IRQ work that flushes the buffer */ member
58 irq_work_queue(&s->work); in queue_flush_work()
183 static void __printk_safe_flush(struct irq_work *work) in __printk_safe_flush() argument
186 container_of(work, struct printk_safe_seq_buf, work); in __printk_safe_flush()
251 __printk_safe_flush(&per_cpu(nmi_print_seq, cpu).work); in printk_safe_flush()
253 __printk_safe_flush(&per_cpu(safe_print_seq, cpu).work); in printk_safe_flush()
412 init_irq_work(&s->work, __printk_safe_flush); in printk_safe_init()
416 init_irq_work(&s->work, __printk_safe_flush); in printk_safe_init()
|
| /kernel/rcu/ |
| D | srcutree.c | 45 static void srcu_invoke_callbacks(struct work_struct *work);
47 static void process_srcu(struct work_struct *work);
150 INIT_WORK(&sdp->work, srcu_invoke_callbacks); in init_srcu_struct_nodes()
180 INIT_DELAYED_WORK(&ssp->work, process_srcu); in init_srcu_struct_fields()
381 flush_delayed_work(&ssp->work); in cleanup_srcu_struct()
386 flush_work(&sdp->work); in cleanup_srcu_struct()
465 queue_work_on(sdp->cpu, rcu_gp_wq, &sdp->work); in srcu_delay_timer()
472 queue_work_on(sdp->cpu, rcu_gp_wq, &sdp->work); in srcu_queue_delayed_work_on()
688 queue_delayed_work(rcu_gp_wq, &ssp->work, in srcu_funnel_gp_start()
690 else if (list_empty(&ssp->work.work.entry)) in srcu_funnel_gp_start()
[all …]
|
| /kernel/irq/ |
| D | irq_sim.c | 14 struct irq_work work; member
85 irq_work_queue(&irq_ctx->work_ctx->work); in irq_sim_set_irqchip_state()
104 static void irq_sim_handle_irq(struct irq_work *work) in irq_sim_handle_irq() argument
110 work_ctx = container_of(work, struct irq_sim_work_ctx, work); in irq_sim_handle_irq()
188 init_irq_work(&work_ctx->work, irq_sim_handle_irq); in irq_domain_create_sim()
211 irq_work_sync(&work_ctx->work); in irq_domain_remove_sim()
|
| /kernel/bpf/ |
| D | stackmap.c | 43 struct stack_map_irq_work *work; in do_up_read() local
48 work = container_of(entry, struct stack_map_irq_work, irq_work); in do_up_read()
49 mmap_read_unlock_non_owner(work->mm); in do_up_read()
299 struct stack_map_irq_work *work = NULL; in stack_map_get_build_id_offset() local
303 work = this_cpu_ptr(&up_read_work); in stack_map_get_build_id_offset()
304 if (atomic_read(&work->irq_work.flags) & IRQ_WORK_BUSY) { in stack_map_get_build_id_offset()
352 if (!work) { in stack_map_get_build_id_offset()
355 work->mm = current->mm; in stack_map_get_build_id_offset()
356 irq_work_queue(&work->irq_work); in stack_map_get_build_id_offset()
869 struct stack_map_irq_work *work; in stack_map_init() local
[all …]
|
| D | ringbuf.c | 35 struct irq_work work; member
126 static void bpf_ringbuf_notify(struct irq_work *work) in bpf_ringbuf_notify() argument
128 struct bpf_ringbuf *rb = container_of(work, struct bpf_ringbuf, work); in bpf_ringbuf_notify()
143 init_irq_work(&rb->work, bpf_ringbuf_notify); in bpf_ringbuf_alloc()
412 irq_work_queue(&rb->work); in bpf_ringbuf_commit()
414 irq_work_queue(&rb->work); in bpf_ringbuf_commit()
|
| D | trampoline.c | 170 static void __bpf_tramp_image_put_deferred(struct work_struct *work) in __bpf_tramp_image_put_deferred() argument
174 im = container_of(work, struct bpf_tramp_image, work); in __bpf_tramp_image_put_deferred()
189 INIT_WORK(&im->work, __bpf_tramp_image_put_deferred); in __bpf_tramp_image_put_rcu()
190 schedule_work(&im->work); in __bpf_tramp_image_put_rcu()
|
| /kernel/trace/ |
| D | ring_buffer.c | 31 static void update_pages_handler(struct work_struct *work);
415 struct irq_work work; member
829 static void rb_wake_up_waiters(struct irq_work *work) in rb_wake_up_waiters() argument
831 struct rb_irq_work *rbwork = container_of(work, struct rb_irq_work, work); in rb_wake_up_waiters()
870 irq_work_queue(&rbwork->work); in ring_buffer_wake_waiters()
887 struct rb_irq_work *work; in ring_buffer_wait() local
897 work = &buffer->irq_work; in ring_buffer_wait()
904 work = &cpu_buffer->irq_work; in ring_buffer_wait()
907 wait_index = READ_ONCE(work->wait_index); in ring_buffer_wait()
911 prepare_to_wait(&work->full_waiters, &wait, TASK_INTERRUPTIBLE); in ring_buffer_wait()
[all …]
|
| D | bpf_trace.c | 1058 struct send_signal_irq_work *work; in do_bpf_send_signal() local
1060 work = container_of(entry, struct send_signal_irq_work, irq_work); in do_bpf_send_signal()
1061 group_send_sig_info(work->sig, SEND_SIG_PRIV, work->task, work->type); in do_bpf_send_signal()
1062 put_task_struct(work->task); in do_bpf_send_signal()
1067 struct send_signal_irq_work *work = NULL; in bpf_send_signal_common() local
1091 work = this_cpu_ptr(&send_signal_work); in bpf_send_signal_common()
1092 if (atomic_read(&work->irq_work.flags) & IRQ_WORK_BUSY) in bpf_send_signal_common()
1099 work->task = get_task_struct(current); in bpf_send_signal_common()
1100 work->sig = sig; in bpf_send_signal_common()
1101 work->type = type; in bpf_send_signal_common()
[all …]
|
| /kernel/sched/ |
| D | cpufreq_schedutil.c | 38 struct kthread_work work; member
543 static void sugov_work(struct kthread_work *work) in sugov_work() argument
545 struct sugov_policy *sg_policy = container_of(work, struct sugov_policy, work); in sugov_work()
575 kthread_queue_work(&sg_policy->worker, &sg_policy->work); in sugov_irq_work()
680 kthread_init_work(&sg_policy->work, sugov_work); in sugov_kthread_create()
878 kthread_cancel_work_sync(&sg_policy->work); in sugov_stop()
|
| /kernel/events/ |
| D | ring_buffer.c | 882 static void rb_free_work(struct work_struct *work) in rb_free_work() argument
888 rb = container_of(work, struct perf_buffer, work); in rb_free_work()
902 schedule_work(&rb->work); in rb_free()
918 INIT_WORK(&rb->work, rb_free_work); in rb_alloc()
|
| /kernel/time/ |
| D | posix-cpu-timers.c | 1100 static void posix_cpu_timers_work(struct callback_head *work) in posix_cpu_timers_work() argument
1102 struct posix_cputimers_work *cw = container_of(work, typeof(*cw), work); in posix_cpu_timers_work()
1156 memset(&p->posix_cputimers_work.work, 0, in clear_posix_cputimers_work()
1157 sizeof(p->posix_cputimers_work.work)); in clear_posix_cputimers_work()
1158 init_task_work(&p->posix_cputimers_work.work, in clear_posix_cputimers_work()
1191 task_work_add(tsk, &tsk->posix_cputimers_work.work, TWA_RESUME); in __run_posix_cpu_timers()
|
| /kernel/power/ |
| D | wakelock.c | 85 static void __wakelocks_gc(struct work_struct *work);
100 static void __wakelocks_gc(struct work_struct *work) in __wakelocks_gc() argument
|