| /kernel/locking/ |
| D | qspinlock.c | 118 u32 tail; in encode_tail() local
120 tail = (cpu + 1) << _Q_TAIL_CPU_OFFSET; in encode_tail()
121 tail |= idx << _Q_TAIL_IDX_OFFSET; /* assume < 4 */ in encode_tail()
123 return tail; in encode_tail()
126 static inline __pure struct mcs_spinlock *decode_tail(u32 tail) in decode_tail() argument
128 int cpu = (tail >> _Q_TAIL_CPU_OFFSET) - 1; in decode_tail()
129 int idx = (tail & _Q_TAIL_IDX_MASK) >> _Q_TAIL_IDX_OFFSET; in decode_tail()
177 static __always_inline u32 xchg_tail(struct qspinlock *lock, u32 tail) in xchg_tail() argument
183 return (u32)xchg_relaxed(&lock->tail, in xchg_tail()
184 tail >> _Q_TAIL_OFFSET) << _Q_TAIL_OFFSET; in xchg_tail()
[all …]
|
| D | osq_lock.c | 58 if (atomic_read(&lock->tail) == curr && in osq_wait_next()
59 atomic_cmpxchg_acquire(&lock->tail, curr, old) == curr) { in osq_wait_next()
107 old = atomic_xchg(&lock->tail, curr); in osq_lock()
215 if (likely(atomic_cmpxchg_release(&lock->tail, curr, in osq_unlock()
|
| /kernel/rcu/ |
| D | rcu_segcblist.c | 21 rclp->tail = &rclp->head; in rcu_cblist_init()
30 *rclp->tail = rhp; in rcu_cblist_enqueue()
31 rclp->tail = &rhp->next; in rcu_cblist_enqueue()
49 drclp->tail = srclp->tail; in rcu_cblist_flush_enqueue()
51 drclp->tail = &drclp->head; in rcu_cblist_flush_enqueue()
58 srclp->tail = &rhp->next; in rcu_cblist_flush_enqueue()
77 rclp->tail = &rclp->head; in rcu_cblist_dequeue()
393 *rclp->tail = rsclp->head; in rcu_segcblist_extract_done_cbs()
396 rclp->tail = rsclp->tails[RCU_DONE_TAIL]; in rcu_segcblist_extract_done_cbs()
418 *rclp->tail = *rsclp->tails[RCU_DONE_TAIL]; in rcu_segcblist_extract_pend_cbs()
[all …]
|
| /kernel/ |
| D | softirq.c | 777 struct tasklet_struct **tail; member
793 *head->tail = t; in __tasklet_schedule_common()
794 head->tail = &(t->next); in __tasklet_schedule_common()
836 tl_head->tail = &tl_head->head; in tasklet_action_common()
865 *tl_head->tail = t; in tasklet_action_common()
866 tl_head->tail = &t->next; in tasklet_action_common()
967 per_cpu(tasklet_vec, cpu).tail = in softirq_init()
969 per_cpu(tasklet_hi_vec, cpu).tail = in softirq_init()
1005 if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) { in takeover_tasklets()
1006 *__this_cpu_read(tasklet_vec.tail) = per_cpu(tasklet_vec, cpu).head; in takeover_tasklets()
[all …]
|
| D | watch_queue.c | 105 unsigned int head, tail, mask, note, offset, len; in post_one_notification() local
112 tail = pipe->tail; in post_one_notification()
113 if (pipe_full(head, tail, pipe->ring_size)) in post_one_notification()
|
| D | kallsyms.c | 80 goto tail; in kallsyms_expand_symbol()
90 tail: in kallsyms_expand_symbol()
|
| /kernel/bpf/ |
| D | queue_stack_maps.c | 20 u32 head, tail; member
33 return qs->head == qs->tail; in queue_stack_map_is_empty()
43 return head == qs->tail; in queue_stack_map_is_full()
118 ptr = &qs->elements[qs->tail * qs->map.value_size]; in __queue_map_get()
122 if (unlikely(++qs->tail >= qs->size)) in __queue_map_get()
123 qs->tail = 0; in __queue_map_get()
223 if (unlikely(++qs->tail >= qs->size)) in queue_stack_map_push_elem()
224 qs->tail = 0; in queue_stack_map_push_elem()
|
| /kernel/trace/ |
| D | ring_buffer.c | 2597 unsigned long tail, struct rb_event_info *info) in rb_reset_tail() argument
2607 if (tail >= BUF_PAGE_SIZE) { in rb_reset_tail()
2613 if (tail == BUF_PAGE_SIZE) in rb_reset_tail()
2620 event = __rb_page_index(tail_page, tail); in rb_reset_tail()
2627 tail_page->real_end = tail; in rb_reset_tail()
2641 if (tail > (BUF_PAGE_SIZE - RB_EVNT_MIN_SIZE)) { in rb_reset_tail()
2656 event->array[0] = (BUF_PAGE_SIZE - tail) - RB_EVNT_HDR_SIZE; in rb_reset_tail()
2662 local_add(BUF_PAGE_SIZE - tail, &cpu_buffer->entries_bytes); in rb_reset_tail()
2668 length = (tail + length) - BUF_PAGE_SIZE; in rb_reset_tail()
2679 unsigned long tail, struct rb_event_info *info) in rb_move_tail() argument
[all …]
|
| D | trace_probe.c | 537 char *tail; in __parse_bitfield_probe_arg() local
542 bw = simple_strtoul(bf + 1, &tail, 0); /* Use simple one */ in __parse_bitfield_probe_arg()
544 if (bw == 0 || *tail != '@') in __parse_bitfield_probe_arg()
547 bf = tail + 1; in __parse_bitfield_probe_arg()
548 bo = simple_strtoul(bf, &tail, 0); in __parse_bitfield_probe_arg()
550 if (tail == bf || *tail != '/') in __parse_bitfield_probe_arg()
|
| D | trace_uprobe.c | 658 char *tail; in __trace_uprobe_create() local
661 tail = kstrdup(kbasename(filename), GFP_KERNEL); in __trace_uprobe_create()
662 if (!tail) { in __trace_uprobe_create()
667 ptr = strpbrk(tail, ".-_"); in __trace_uprobe_create()
671 snprintf(buf, MAX_EVENT_NAME_LEN, "%c_%s_0x%lx", 'p', tail, offset); in __trace_uprobe_create()
673 kfree(tail); in __trace_uprobe_create()
|
| D | trace_events.c | 233 struct ftrace_event_field *tail; in trace_event_get_offsets() local
241 tail = list_first_entry(head, struct ftrace_event_field, link); in trace_event_get_offsets()
242 return tail->offset + tail->size; in trace_event_get_offsets()
|
| D | trace.c | 174 struct trace_eval_map_tail tail; member
6029 if (ptr->tail.next) { in update_eval_map()
6030 ptr = ptr->tail.next; in update_eval_map()
6154 if (!ptr->tail.next) in trace_insert_eval_map_file()
6156 ptr = ptr->tail.next; in trace_insert_eval_map_file()
6159 ptr->tail.next = map_array; in trace_insert_eval_map_file()
9848 last = &map->tail.next; in trace_module_remove_evals()
9849 map = map->tail.next; in trace_module_remove_evals()
9854 *last = trace_eval_jmp_to_tail(map)->tail.next; in trace_module_remove_evals()
|
| D | trace_functions_graph.c | 60 { TRACER_OPT(funcgraph-tail, TRACE_GRAPH_PRINT_TAIL) },
|
| /kernel/irq/ |
| D | irqdomain.c | 1285 struct irq_data *tail, *irqd, *irq_data; in irq_domain_trim_hierarchy() local
1288 tail = NULL; in irq_domain_trim_hierarchy()
1300 if (irqd->chip && tail) in irq_domain_trim_hierarchy()
1304 if (!irqd->chip && !tail) in irq_domain_trim_hierarchy()
1312 tail = irq_data; in irq_domain_trim_hierarchy()
1317 if (!tail) in irq_domain_trim_hierarchy()
1321 virq, tail->parent_data->domain->name); in irq_domain_trim_hierarchy()
1324 irqd = tail; in irq_domain_trim_hierarchy()
1325 tail = tail->parent_data; in irq_domain_trim_hierarchy()
1327 __irq_domain_free_hierarchy(tail); in irq_domain_trim_hierarchy()
|
| /kernel/events/ |
| D | ring_buffer.c | 138 ring_buffer_has_space(unsigned long head, unsigned long tail, in ring_buffer_has_space() argument
143 return CIRC_SPACE(head, tail, data_size) >= size; in ring_buffer_has_space()
145 return CIRC_SPACE(tail, head, data_size) >= size; in ring_buffer_has_space()
155 unsigned long tail, offset, head; in __perf_output_begin() local
195 tail = READ_ONCE(rb->user_page->data_tail); in __perf_output_begin()
198 if (unlikely(!ring_buffer_has_space(head, tail, in __perf_output_begin()
|
| /kernel/futex/ |
| D | core.c | 226 struct page *page, *tail; in get_futex_key() local
304 tail = page; in get_futex_key()
399 key->shared.pgoff = page_to_pgoff(tail); in get_futex_key()
|