| /kernel/gcov/ |
| D | fs.c | 255 static struct gcov_info *get_node_info(struct gcov_node *node) in get_node_info() argument
257 if (node->num_loaded > 0) in get_node_info()
258 return node->loaded_info[0]; in get_node_info()
260 return node->unloaded_info; in get_node_info()
267 static struct gcov_info *get_accumulated_info(struct gcov_node *node) in get_accumulated_info() argument
272 if (node->unloaded_info) in get_accumulated_info()
273 info = gcov_info_dup(node->unloaded_info); in get_accumulated_info()
275 info = gcov_info_dup(node->loaded_info[i++]); in get_accumulated_info()
278 for (; i < node->num_loaded; i++) in get_accumulated_info()
279 gcov_info_add(info, node->loaded_info[i]); in get_accumulated_info()
[all …]
|
| /kernel/bpf/ |
| D | bpf_lru_list.c | 42 static bool bpf_lru_node_is_ref(const struct bpf_lru_node *node) in bpf_lru_node_is_ref() argument
44 return READ_ONCE(node->ref); in bpf_lru_node_is_ref()
47 static void bpf_lru_node_clear_ref(struct bpf_lru_node *node) in bpf_lru_node_clear_ref() argument
49 WRITE_ONCE(node->ref, 0); in bpf_lru_node_clear_ref()
67 struct bpf_lru_node *node, in __bpf_lru_node_move_to_free() argument
71 if (WARN_ON_ONCE(IS_LOCAL_LIST_TYPE(node->type))) in __bpf_lru_node_move_to_free()
77 if (&node->list == l->next_inactive_rotation) in __bpf_lru_node_move_to_free()
80 bpf_lru_list_count_dec(l, node->type); in __bpf_lru_node_move_to_free()
82 node->type = tgt_free_type; in __bpf_lru_node_move_to_free()
83 list_move(&node->list, free_list); in __bpf_lru_node_move_to_free()
[all …]
|
| D | lpm_trie.c | 166 const struct lpm_trie_node *node, in longest_prefix_match() argument
169 u32 limit = min(node->prefixlen, key->prefixlen); in longest_prefix_match()
181 u64 diff = be64_to_cpu(*(__be64 *)node->data ^ in longest_prefix_match()
194 u32 diff = be32_to_cpu(*(__be32 *)&node->data[i] ^ in longest_prefix_match()
206 u16 diff = be16_to_cpu(*(__be16 *)&node->data[i] ^ in longest_prefix_match()
218 prefixlen += 8 - fls(node->data[i] ^ key->data[i]); in longest_prefix_match()
231 struct lpm_trie_node *node, *found = NULL; in trie_lookup_elem() local
239 for (node = rcu_dereference_check(trie->root, rcu_read_lock_bh_held()); in trie_lookup_elem()
240 node;) { in trie_lookup_elem()
248 matchlen = longest_prefix_match(trie, node, key); in trie_lookup_elem()
[all …]
|
| D | percpu_freelist.c | 31 struct pcpu_freelist_node *node) in pcpu_freelist_push_node() argument
33 node->next = head->first; in pcpu_freelist_push_node()
34 WRITE_ONCE(head->first, node); in pcpu_freelist_push_node()
38 struct pcpu_freelist_node *node) in ___pcpu_freelist_push() argument
41 pcpu_freelist_push_node(head, node); in ___pcpu_freelist_push()
46 struct pcpu_freelist_node *node) in pcpu_freelist_try_push_extra() argument
51 pcpu_freelist_push_node(&s->extralist, node); in pcpu_freelist_try_push_extra()
57 struct pcpu_freelist_node *node) in ___pcpu_freelist_push_nmi() argument
68 pcpu_freelist_push_node(head, node); in ___pcpu_freelist_push_nmi()
75 if (pcpu_freelist_try_push_extra(s, node)) in ___pcpu_freelist_push_nmi()
[all …]
|
| D | bpf_lru_list.h | 51 typedef bool (*del_from_htab_func)(void *arg, struct bpf_lru_node *node);
65 static inline void bpf_lru_node_set_ref(struct bpf_lru_node *node) in bpf_lru_node_set_ref() argument
67 if (!READ_ONCE(node->ref)) in bpf_lru_node_set_ref()
68 WRITE_ONCE(node->ref, 1); in bpf_lru_node_set_ref()
77 void bpf_lru_push_free(struct bpf_lru *lru, struct bpf_lru_node *node);
|
| D | local_storage.c | 71 struct rb_node *node; in cgroup_storage_lookup() local
76 node = root->rb_node; in cgroup_storage_lookup()
77 while (node) { in cgroup_storage_lookup()
80 storage = container_of(node, struct bpf_cgroup_storage, node); in cgroup_storage_lookup()
84 node = node->rb_left; in cgroup_storage_lookup()
87 node = node->rb_right; in cgroup_storage_lookup()
111 this = container_of(*new, struct bpf_cgroup_storage, node); in cgroup_storage_insert()
126 rb_link_node(&storage->node, parent, new); in cgroup_storage_insert()
127 rb_insert_color(&storage->node, root); in cgroup_storage_insert()
607 rb_erase(&storage->node, root); in bpf_cgroup_storage_unlink()
|
| /kernel/trace/ |
| D | rethook.c | 24 struct llist_node *node; in rethook_flush_task() local
26 node = __llist_del_all(&tk->rethooks); in rethook_flush_task()
27 while (node) { in rethook_flush_task()
28 rhn = container_of(node, struct rethook_node, llist); in rethook_flush_task()
29 node = node->next; in rethook_flush_task()
40 struct freelist_node *node; in rethook_free_rcu() local
43 node = rh->pool.head; in rethook_free_rcu()
44 while (node) { in rethook_free_rcu()
45 rhn = container_of(node, struct rethook_node, freelist); in rethook_free_rcu()
46 node = node->next; in rethook_free_rcu()
[all …]
|
| D | trace_boot.c | 25 trace_boot_set_instance_options(struct trace_array *tr, struct xbc_node *node) in trace_boot_set_instance_options() argument
33 xbc_node_for_each_array_value(node, "options", anode, p) { in trace_boot_set_instance_options()
43 p = xbc_node_find_value(node, "tracing_on", NULL); in trace_boot_set_instance_options()
53 p = xbc_node_find_value(node, "trace_clock", NULL); in trace_boot_set_instance_options()
59 p = xbc_node_find_value(node, "buffer_size", NULL); in trace_boot_set_instance_options()
68 p = xbc_node_find_value(node, "cpumask", NULL); in trace_boot_set_instance_options()
83 trace_boot_enable_events(struct trace_array *tr, struct xbc_node *node) in trace_boot_enable_events() argument
89 xbc_node_for_each_array_value(node, "events", anode, p) { in trace_boot_enable_events()
102 trace_boot_add_kprobe_event(struct xbc_node *node, const char *event) in trace_boot_add_kprobe_event() argument
110 xbc_node_for_each_array_value(node, "probes", anode, val) { in trace_boot_add_kprobe_event()
[all …]
|
| D | trace_stat.c | 27 struct rb_node node; member
51 rbtree_postorder_for_each_entry_safe(snode, n, &session->stat_root, node) { in __reset_stat_session()
93 this = container_of(*new, struct stat_node, node); in insert_stat()
103 rb_link_node(&data->node, parent, new); in insert_stat()
104 rb_insert_color(&data->node, root); in insert_stat()
174 struct rb_node *node; in stat_seq_start() local
188 node = rb_first(&session->stat_root); in stat_seq_start()
189 for (i = 0; node && i < n; i++) in stat_seq_start()
190 node = rb_next(node); in stat_seq_start()
192 return node; in stat_seq_start()
[all …]
|
| D | fprobe.c | 18 struct rethook_node node; member
41 fpr = container_of(rh, struct fprobe_rethook_node, node); in __fprobe_handler()
132 fpr = container_of(rh, struct fprobe_rethook_node, node); in fprobe_exit_handler()
212 struct fprobe_rethook_node *node; in fprobe_init_rethook() local
214 node = kzalloc(sizeof(*node) + fp->entry_data_size, GFP_KERNEL); in fprobe_init_rethook()
215 if (!node) { in fprobe_init_rethook()
220 rethook_add_node(fp->rethook, &node->node); in fprobe_init_rethook()
|
| /kernel/locking/ |
| D | osq_lock.c | 25 static inline int node_cpu(struct optimistic_spin_node *node) in node_cpu() argument
27 return node->cpu - 1; in node_cpu()
43 struct optimistic_spin_node *node, in osq_wait_next() argument
78 if (node->next) { in osq_wait_next()
79 next = xchg(&node->next, NULL); in osq_wait_next()
92 struct optimistic_spin_node *node = this_cpu_ptr(&osq_node); in osq_lock() local
97 node->locked = 0; in osq_lock()
98 node->next = NULL; in osq_lock()
99 node->cpu = curr; in osq_lock()
112 node->prev = prev; in osq_lock()
[all …]
|
| D | qspinlock_paravirt.h | 173 struct pv_node *node; member
212 static struct qspinlock **pv_hash(struct qspinlock *lock, struct pv_node *node) in pv_hash() argument
221 WRITE_ONCE(he->node, node); in pv_hash()
243 struct pv_node *node; in pv_unhash() local
247 node = READ_ONCE(he->node); in pv_unhash()
249 return node; in pv_unhash()
278 static void pv_init_node(struct mcs_spinlock *node) in pv_init_node() argument
280 struct pv_node *pn = (struct pv_node *)node; in pv_init_node()
293 static void pv_wait_node(struct mcs_spinlock *node, struct mcs_spinlock *prev) in pv_wait_node() argument
295 struct pv_node *pn = (struct pv_node *)node; in pv_wait_node()
[all …]
|
| D | mcs_spinlock.h | 65 void mcs_spin_lock(struct mcs_spinlock **lock, struct mcs_spinlock *node) in mcs_spin_lock() argument
70 node->locked = 0; in mcs_spin_lock()
71 node->next = NULL; in mcs_spin_lock()
79 prev = xchg(lock, node); in mcs_spin_lock()
91 WRITE_ONCE(prev->next, node); in mcs_spin_lock()
94 arch_mcs_spin_lock_contended(&node->locked); in mcs_spin_lock()
102 void mcs_spin_unlock(struct mcs_spinlock **lock, struct mcs_spinlock *node) in mcs_spin_unlock() argument
104 struct mcs_spinlock *next = READ_ONCE(node->next); in mcs_spin_unlock()
110 if (likely(cmpxchg_release(lock, node, NULL) == node)) in mcs_spin_unlock()
113 while (!(next = READ_ONCE(node->next))) in mcs_spin_unlock()
|
| D | qspinlock.c | 273 static __always_inline void __pv_init_node(struct mcs_spinlock *node) { } in __pv_init_node() argument
274 static __always_inline void __pv_wait_node(struct mcs_spinlock *node, in __pv_wait_node() argument
277 struct mcs_spinlock *node) { } in __pv_kick_node() argument
279 struct mcs_spinlock *node) in __pv_wait_head_or_lock() argument
318 struct mcs_spinlock *prev, *next, *node; in queued_spin_lock_slowpath() local
401 node = this_cpu_ptr(&qnodes[0].mcs); in queued_spin_lock_slowpath()
402 idx = node->count++; in queued_spin_lock_slowpath()
423 node = grab_mcs_node(node, idx); in queued_spin_lock_slowpath()
437 node->locked = 0; in queued_spin_lock_slowpath()
438 node->next = NULL; in queued_spin_lock_slowpath()
[all …]
|
| /kernel/power/ |
| D | wakelock.c | 29 struct rb_node node; member
40 struct rb_node *node; in pm_show_wakelocks() local
46 for (node = rb_first(&wakelocks_tree); node; node = rb_next(node)) { in pm_show_wakelocks()
47 wl = rb_entry(node, struct wakelock, node); in pm_show_wakelocks()
122 rb_erase(&wl->node, &wakelocks_tree); in __wakelocks_gc()
150 struct rb_node **node = &wakelocks_tree.rb_node; in wakelock_lookup_add() local
151 struct rb_node *parent = *node; in wakelock_lookup_add()
154 while (*node) { in wakelock_lookup_add()
157 parent = *node; in wakelock_lookup_add()
158 wl = rb_entry(*node, struct wakelock, node); in wakelock_lookup_add()
[all …]
|
| D | snapshot.c | 405 struct rtree_node *node; member
444 struct rtree_node *node; in alloc_rtree_node() local
446 node = chain_alloc(ca, sizeof(struct rtree_node)); in alloc_rtree_node()
447 if (!node) in alloc_rtree_node()
450 node->data = get_image_page(gfp_mask, safe_needed); in alloc_rtree_node()
451 if (!node->data) in alloc_rtree_node()
454 list_add_tail(&node->list, list); in alloc_rtree_node()
456 return node; in alloc_rtree_node()
469 struct rtree_node *node, *block, **dst; in add_rtree_block() local
484 node = alloc_rtree_node(gfp_mask, safe_needed, ca, in add_rtree_block()
[all …]
|
| /kernel/ |
| D | smp.c | 37 #define CSD_TYPE(_csd) ((_csd)->node.u_flags & CSD_FLAG_TYPE_MASK)
206 return csd->node.dst; /* Other CSD_TYPE_ values might not have ->dst. */ in csd_lock_wait_getcpu()
222 unsigned int flags = READ_ONCE(csd->node.u_flags); in csd_lock_wait_toolong()
311 smp_cond_load_acquire(&csd->node.u_flags, !(VAL & CSD_FLAG_LOCK)); in csd_lock_wait()
320 smp_cond_load_acquire(&csd->node.u_flags, !(VAL & CSD_FLAG_LOCK)); in csd_lock_wait()
327 csd->node.u_flags |= CSD_FLAG_LOCK; in csd_lock()
339 WARN_ON(!(csd->node.u_flags & CSD_FLAG_LOCK)); in csd_unlock()
344 smp_store_release(&csd->node.u_flags, 0); in csd_unlock()
349 void __smp_call_single_queue(int cpu, struct llist_node *node) in __smp_call_single_queue() argument
362 csd = container_of(node, call_single_data_t, node.llist); in __smp_call_single_queue()
[all …]
|
| D | irq_work.c | 61 oflags = atomic_fetch_or(IRQ_WORK_CLAIMED | CSD_TYPE_IRQ_WORK, &work->node.a_flags); in irq_work_claim()
95 work_flags = atomic_read(&work->node.a_flags); in __irq_work_queue_local()
107 if (!llist_add(&work->node.llist, list)) in __irq_work_queue_local()
163 !(atomic_read(&work->node.a_flags) & IRQ_WORK_HARD_IRQ)) { in irq_work_queue_on()
165 if (!llist_add(&work->node.llist, &per_cpu(lazy_list, cpu))) in irq_work_queue_on()
173 __smp_call_single_queue(cpu, &work->node.llist); in irq_work_queue_on()
212 flags = atomic_read(&work->node.a_flags); in irq_work_single()
214 atomic_set(&work->node.a_flags, flags); in irq_work_single()
229 (void)atomic_cmpxchg(&work->node.a_flags, flags, flags & ~IRQ_WORK_BUSY); in irq_work_single()
252 llist_for_each_entry_safe(work, tmp, llnode, node.llist) in irq_work_run_list()
|
| D | kthread.c | 44 int node; member
408 current->pref_node_fork = create->node; in create_kthread()
429 void *data, int node, in __kthread_create_on_node() argument
442 create->node = node; in __kthread_create_on_node()
504 void *data, int node, in kthread_create_on_node() argument
512 task = __kthread_create_on_node(threadfn, data, node, namefmt, args); in kthread_create_on_node()
833 struct kthread_work, node); in kthread_worker_fn()
834 list_del_init(&work->node); in kthread_worker_fn()
864 int node = NUMA_NO_NODE; in __kthread_create_worker() local
873 node = cpu_to_node(cpu); in __kthread_create_worker()
[all …]
|
| D | scs.c | 31 static void *__scs_alloc(int node) in __scs_alloc() argument
47 GFP_SCS, PAGE_KERNEL, 0, node, in __scs_alloc()
54 void *scs_alloc(int node) in scs_alloc() argument
58 s = __scs_alloc(node); in scs_alloc()
114 int scs_prepare(struct task_struct *tsk, int node) in scs_prepare() argument
121 s = scs_alloc(node); in scs_prepare()
|
| D | cpu.c | 77 struct hlist_node *node; member
131 struct hlist_node *node);
136 struct hlist_node *node);
171 bool bringup, struct hlist_node *node, in cpuhp_invoke_callback() argument
176 int (*cbm)(unsigned int cpu, struct hlist_node *node); in cpuhp_invoke_callback()
202 if (node) { in cpuhp_invoke_callback()
204 trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node); in cpuhp_invoke_callback()
205 ret = cbm(cpu, node); in cpuhp_invoke_callback()
212 hlist_for_each(node, &step->list) { in cpuhp_invoke_callback()
213 if (lastp && node == *lastp) in cpuhp_invoke_callback()
[all …]
|
| /kernel/module/ |
| D | tree_lookup.c | 24 struct module_memory *mod_mem = container_of(n, struct module_memory, mtn.node); in __mod_tree_val()
31 struct module_memory *mod_mem = container_of(n, struct module_memory, mtn.node); in __mod_tree_size()
64 static noinline void __mod_tree_insert(struct mod_tree_node *node, struct mod_tree_root *tree) in __mod_tree_insert() argument
66 latch_tree_insert(&node->node, &tree->root, &mod_tree_ops); in __mod_tree_insert()
69 static void __mod_tree_remove(struct mod_tree_node *node, struct mod_tree_root *tree) in __mod_tree_remove() argument
71 latch_tree_erase(&node->node, &tree->root, &mod_tree_ops); in __mod_tree_remove()
111 return container_of(ltn, struct mod_tree_node, node)->mod; in mod_find()
|
| /kernel/irq/ |
| D | irqdesc.c | 54 static int alloc_masks(struct irq_desc *desc, int node) in alloc_masks() argument
57 GFP_KERNEL, node)) in alloc_masks()
62 GFP_KERNEL, node)) { in alloc_masks()
69 if (!zalloc_cpumask_var_node(&desc->pending_mask, GFP_KERNEL, node)) { in alloc_masks()
80 static void desc_smp_init(struct irq_desc *desc, int node, in desc_smp_init() argument
91 desc->irq_common_data.node = node; in desc_smp_init()
97 alloc_masks(struct irq_desc *desc, int node) { return 0; } in alloc_masks() argument
99 desc_smp_init(struct irq_desc *desc, int node, const struct cpumask *affinity) { } in desc_smp_init() argument
102 static void desc_set_defaults(unsigned int irq, struct irq_desc *desc, int node, in desc_set_defaults() argument
126 desc_smp_init(desc, node, affinity); in desc_set_defaults()
[all …]
|
| /kernel/time/ |
| D | alarmtimer.c | 162 timerqueue_del(&base->timerqueue, &alarm->node); in alarmtimer_enqueue()
164 timerqueue_add(&base->timerqueue, &alarm->node); in alarmtimer_enqueue()
182 timerqueue_del(&base->timerqueue, &alarm->node); in alarmtimer_dequeue()
213 hrtimer_set_expires(&alarm->timer, alarm->node.expires); in alarmtimer_fired()
227 return ktime_sub(alarm->node.expires, base->get_ktime()); in alarm_expires_remaining()
328 timerqueue_init(&alarm->node); in __alarm_init()
361 alarm->node.expires = start; in alarm_start()
363 hrtimer_start(&alarm->timer, alarm->node.expires, HRTIMER_MODE_ABS); in alarm_start()
390 hrtimer_set_expires(&alarm->timer, alarm->node.expires); in alarm_restart()
445 delta = ktime_sub(now, alarm->node.expires); in alarm_forward()
[all …]
|
| /kernel/livepatch/ |
| D | shadow.c | 55 struct hlist_node node; member
89 hash_for_each_possible_rcu(klp_shadow_hash, shadow, node, in klp_shadow_get()
157 hash_add_rcu(klp_shadow_hash, &new_shadow->node, in __klp_shadow_get_or_alloc()
237 hash_del_rcu(&shadow->node); in klp_shadow_free_struct()
261 hash_for_each_possible(klp_shadow_hash, shadow, node, in klp_shadow_free()
292 hash_for_each(klp_shadow_hash, i, shadow, node) { in klp_shadow_free_all()
|