| /kernel/locking/ |
| D | osq_lock.c | 44 struct optimistic_spin_node *prev) in osq_wait_next() argument
55 old = prev ? prev->cpu : OSQ_UNLOCKED_VAL; in osq_wait_next()
93 struct optimistic_spin_node *prev, *next; in osq_lock() local
111 prev = decode_cpu(old); in osq_lock()
112 node->prev = prev; in osq_lock()
126 WRITE_ONCE(prev->next, node); in osq_lock()
144 vcpu_is_preempted(node_cpu(node->prev)))) in osq_lock()
161 if (data_race(prev->next) == node && in osq_lock()
162 cmpxchg(&prev->next, node, NULL) == node) in osq_lock()
179 prev = READ_ONCE(node->prev); in osq_lock()
[all …]
|
| D | mcs_spinlock.h | 67 struct mcs_spinlock *prev; in mcs_spin_lock() local
79 prev = xchg(lock, node); in mcs_spin_lock()
80 if (likely(prev == NULL)) { in mcs_spin_lock()
91 WRITE_ONCE(prev->next, node); in mcs_spin_lock()
|
| D | lockdep.c | 1615 __calc_dep_bit(struct held_lock *prev, struct held_lock *next) in __calc_dep_bit() argument
1617 return (prev->read == 0) + ((next->read != 2) << 1); in __calc_dep_bit()
1620 static inline u8 calc_dep(struct held_lock *prev, struct held_lock *next) in calc_dep() argument
1622 return 1U << __calc_dep_bit(prev, next); in calc_dep()
1630 __calc_dep_bitb(struct held_lock *prev, struct held_lock *next) in __calc_dep_bitb() argument
1632 return (next->read != 2) + ((prev->read == 0) << 1); in __calc_dep_bitb()
1635 static inline u8 calc_depb(struct held_lock *prev, struct held_lock *next) in calc_depb() argument
1637 return 1U << __calc_dep_bitb(prev, next); in calc_depb()
2564 struct held_lock *prev, in print_bad_irq_dependency() argument
2588 print_lock(prev); in print_bad_irq_dependency()
[all …]
|
| D | qspinlock.c | 275 struct mcs_spinlock *prev) { } in __pv_wait_node() argument
318 struct mcs_spinlock *prev, *next, *node; in queued_spin_lock_slowpath() local
471 prev = decode_tail(old); in queued_spin_lock_slowpath()
474 WRITE_ONCE(prev->next, node); in queued_spin_lock_slowpath()
476 pv_wait_node(node, prev); in queued_spin_lock_slowpath()
|
| D | qspinlock_paravirt.h | 267 pv_wait_early(struct pv_node *prev, int loop) in pv_wait_early() argument
272 return READ_ONCE(prev->state) != vcpu_running; in pv_wait_early()
293 static void pv_wait_node(struct mcs_spinlock *node, struct mcs_spinlock *prev) in pv_wait_node() argument
296 struct pv_node *pp = (struct pv_node *)prev; in pv_wait_node()
|
| /kernel/sched/ |
| D | cputime.c | 444 void vtime_task_switch(struct task_struct *prev) in vtime_task_switch() argument
446 if (is_idle_task(prev)) in vtime_task_switch()
447 vtime_account_idle(prev); in vtime_task_switch()
449 vtime_account_kernel(prev); in vtime_task_switch()
451 vtime_flush(prev); in vtime_task_switch()
452 arch_vtime_task_switch(prev); in vtime_task_switch()
472 void cputime_adjust(struct task_cputime *curr, struct prev_cputime *prev, in cputime_adjust() argument
580 void cputime_adjust(struct task_cputime *curr, struct prev_cputime *prev, in cputime_adjust() argument
587 raw_spin_lock_irqsave(&prev->lock, flags); in cputime_adjust()
598 if (prev->stime + prev->utime >= rtime) in cputime_adjust()
[all …]
|
| D | stats.h | 111 void psi_task_switch(struct task_struct *prev, struct task_struct *next,
114 void psi_account_irqtime(struct rq *rq, struct task_struct *curr, struct task_struct *prev);
117 struct task_struct *prev) {} in psi_account_irqtime() argument
182 static inline void psi_sched_switch(struct task_struct *prev, in psi_sched_switch() argument
189 psi_task_switch(prev, next, sleep); in psi_sched_switch()
196 static inline void psi_sched_switch(struct task_struct *prev, in psi_sched_switch() argument
200 struct task_struct *prev) {} in psi_account_irqtime() argument
281 sched_info_switch(struct rq *rq, struct task_struct *prev, struct task_struct *next) in sched_info_switch() argument
288 if (prev != rq->idle) in sched_info_switch()
289 sched_info_depart(rq, prev); in sched_info_switch()
|
| D | core.c | 5119 static inline void finish_task(struct task_struct *prev) in finish_task() argument
5133 smp_store_release(&prev->on_cpu, 0); in finish_task()
5310 prepare_task_switch(struct rq *rq, struct task_struct *prev, in prepare_task_switch() argument
5313 kcov_prepare_switch(prev); in prepare_task_switch()
5314 sched_info_switch(rq, prev, next); in prepare_task_switch()
5315 perf_event_task_sched_out(prev, next); in prepare_task_switch()
5316 rseq_preempt(prev); in prepare_task_switch()
5317 fire_sched_out_preempt_notifiers(prev, next); in prepare_task_switch()
5342 static struct rq *finish_task_switch(struct task_struct *prev) in finish_task_switch() argument
5378 prev_state = READ_ONCE(prev->__state); in finish_task_switch()
[all …]
|
| D | autogroup.c | 160 struct autogroup *prev; in autogroup_move_group() local
167 prev = p->signal->autogroup; in autogroup_move_group()
168 if (prev == ag) { in autogroup_move_group()
189 autogroup_kref_put(prev); in autogroup_move_group()
|
| D | psi.c | 934 void psi_task_switch(struct task_struct *prev, struct task_struct *next, in psi_task_switch() argument
938 int cpu = task_cpu(prev); in psi_task_switch()
960 if (prev->pid) { in psi_task_switch()
972 if (prev->in_memstall) in psi_task_switch()
974 if (prev->in_iowait) in psi_task_switch()
983 if (unlikely((prev->flags & PF_WQ_WORKER) && in psi_task_switch()
984 wq_worker_last_func(prev) == psi_avgs_work)) in psi_task_switch()
988 psi_flags_change(prev, clear, set); in psi_task_switch()
990 group = task_psi_group(prev); in psi_task_switch()
1003 if ((prev->psi_flags ^ next->psi_flags) & ~TSK_ONCPU) { in psi_task_switch()
[all …]
|
| D | stop_task.c | 19 balance_stop(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) in balance_stop() argument
71 static void put_prev_task_stop(struct rq *rq, struct task_struct *prev) in put_prev_task_stop() argument
|
| D | sched.h | 500 struct cfs_rq *prev, struct cfs_rq *next);
503 struct cfs_rq *prev, struct cfs_rq *next) { } in set_task_rq_fair() argument
2301 int (*balance)(struct rq *rq, struct task_struct *prev, struct rq_flags *rf);
2346 static inline void put_prev_task(struct rq *rq, struct task_struct *prev) in put_prev_task() argument
2348 WARN_ON_ONCE(rq->curr != prev); in put_prev_task()
2349 prev->sched_class->put_prev_task(rq, prev); in put_prev_task()
2411 extern struct task_struct *pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct rq_f…
3529 struct task_struct *prev, in switch_mm_cid() argument
3544 if (prev->mm) // from user in switch_mm_cid()
3556 if (!prev->mm) { // from kernel in switch_mm_cid()
[all …]
|
| /kernel/ |
| D | scs.c | 133 unsigned long *p, prev, curr = highest, used = 0; in scs_check_usage() local
145 prev = cmpxchg_relaxed(&highest, curr, used); in scs_check_usage()
147 if (prev == curr) { in scs_check_usage()
153 curr = prev; in scs_check_usage()
|
| D | smp.c | 448 struct llist_node *entry, *prev; in __flush_smp_call_function_queue() local
497 prev = NULL; in __flush_smp_call_function_queue()
504 if (prev) { in __flush_smp_call_function_queue()
505 prev->next = &csd_next->node.llist; in __flush_smp_call_function_queue()
515 prev = &csd->node.llist; in __flush_smp_call_function_queue()
525 prev = NULL; in __flush_smp_call_function_queue()
530 if (prev) { in __flush_smp_call_function_queue()
531 prev->next = &csd_next->node.llist; in __flush_smp_call_function_queue()
549 prev = &csd->node.llist; in __flush_smp_call_function_queue()
|
| D | static_call_inline.c | 409 struct static_call_mod *site_mod, **prev; in static_call_del_module() local
419 for (prev = &key->mods, site_mod = key->mods; in static_call_del_module()
421 prev = &site_mod->next, site_mod = site_mod->next) in static_call_del_module()
427 *prev = site_mod->next; in static_call_del_module()
|
| D | seccomp.c | 232 struct seccomp_filter *prev; member
425 for (; f; f = f->prev) { in seccomp_run_filters()
475 for (; child; child = child->prev) in is_ancestor()
536 orig = orig->prev; in __seccomp_filter_orphan()
545 orig = orig->prev; in __put_seccomp_filter()
848 sfilter->prev ? &sfilter->prev->cache : NULL; in seccomp_cache_prepare()
886 for (walker = current->seccomp.filter; walker; walker = walker->prev) in seccomp_attach_filter()
916 filter->prev = current->seccomp.filter; in seccomp_attach_filter()
1880 for (cur = current->seccomp.filter; cur; cur = cur->prev) { in has_duplicate_listener()
2134 for (filter = orig; filter; filter = filter->prev) in get_nth_filter()
[all …]
|
| D | jump_label.c | 704 struct static_key_mod *jlm, **prev; in jump_label_del_module() local
719 prev = &key->next; in jump_label_del_module()
723 prev = &jlm->next; in jump_label_del_module()
731 if (prev == &key->next) in jump_label_del_module()
734 *prev = jlm->next; in jump_label_del_module()
|
| /kernel/gcov/ |
| D | base.c | 108 struct gcov_info *prev = NULL; in gcov_module_notifier() local
117 gcov_info_unlink(prev, info); in gcov_module_notifier()
121 prev = info; in gcov_module_notifier()
|
| D | gcc_4_7.c | 158 void gcov_info_unlink(struct gcov_info *prev, struct gcov_info *info) in gcov_info_unlink() argument
160 if (prev) in gcov_info_unlink()
161 prev->next = info->next; in gcov_info_unlink()
|
| /kernel/dma/ |
| D | pool.c | 224 static inline struct gen_pool *dma_guess_pool(struct gen_pool *prev, gfp_t gfp) in dma_guess_pool() argument
226 if (prev == NULL) { in dma_guess_pool()
233 if (prev == atomic_pool_kernel) in dma_guess_pool()
235 if (prev == atomic_pool_dma32) in dma_guess_pool()
|
| /kernel/trace/ |
| D | trace_sched_wakeup.c | 375 struct task_struct *prev, in tracing_sched_switch_trace() argument
389 entry->prev_pid = prev->pid; in tracing_sched_switch_trace()
390 entry->prev_prio = prev->prio; in tracing_sched_switch_trace()
391 entry->prev_state = task_state_index(prev); in tracing_sched_switch_trace()
431 struct task_struct *prev, struct task_struct *next, in probe_wakeup_sched_switch() argument
441 tracing_record_cmdline(prev); in probe_wakeup_sched_switch()
477 tracing_sched_switch_trace(wakeup_trace, prev, next, trace_ctx); in probe_wakeup_sched_switch()
|
| D | trace_sched_switch.c | 25 struct task_struct *prev, struct task_struct *next, in probe_sched_switch() argument
35 tracing_record_taskinfo_sched_switch(prev, next, flags); in probe_sched_switch()
|
| /kernel/time/ |
| D | tick-sched.c | 424 int prev; in tick_nohz_dep_set_all() local
426 prev = atomic_fetch_or(BIT(bit), dep); in tick_nohz_dep_set_all()
427 if (!prev) in tick_nohz_dep_set_all()
451 int prev; in tick_nohz_dep_set_cpu() local
456 prev = atomic_fetch_or(BIT(bit), &ts->tick_dep_mask); in tick_nohz_dep_set_cpu()
457 if (!prev) { in tick_nohz_dep_set_cpu()
504 int prev; in tick_nohz_dep_set_signal() local
507 prev = atomic_fetch_or(BIT(bit), &sig->tick_dep_mask); in tick_nohz_dep_set_signal()
508 if (!prev) { in tick_nohz_dep_set_signal()
|
| /kernel/events/ |
| D | uprobes.c | 968 struct map_info *prev = NULL; in build_map_info() local
978 if (!prev && !more) { in build_map_info()
983 prev = kmalloc(sizeof(struct map_info), in build_map_info()
985 if (prev) in build_map_info()
986 prev->next = NULL; in build_map_info()
988 if (!prev) { in build_map_info()
996 info = prev; in build_map_info()
997 prev = prev->next; in build_map_info()
1009 prev = curr; in build_map_info()
1021 info->next = prev; in build_map_info()
[all …]
|
| /kernel/livepatch/ |
| D | transition.c | 241 struct klp_func *prev; in klp_check_stack_func() local
243 prev = list_next_entry(func, stack_node); in klp_check_stack_func()
244 func_addr = (unsigned long)prev->new_func; in klp_check_stack_func()
245 func_size = prev->new_size; in klp_check_stack_func()
|