| /kernel/ |
| D | jump_label.c | 72 jea->key = jeb->key - delta; in jump_label_swap()
76 jeb->key = tmp.key + delta; in jump_label_swap()
93 static void jump_label_update(struct static_key *key);
104 int static_key_count(struct static_key *key) in static_key_count() argument
110 int n = atomic_read(&key->enabled); in static_key_count()
116 void static_key_slow_inc_cpuslocked(struct static_key *key) in static_key_slow_inc_cpuslocked() argument
120 STATIC_KEY_CHECK_USE(key); in static_key_slow_inc_cpuslocked()
135 for (v = atomic_read(&key->enabled); v > 0; v = v1) { in static_key_slow_inc_cpuslocked()
136 v1 = atomic_cmpxchg(&key->enabled, v, v + 1); in static_key_slow_inc_cpuslocked()
142 if (atomic_read(&key->enabled) == 0) { in static_key_slow_inc_cpuslocked()
[all …]
|
| D | static_call_inline.c | 40 return (long)site->key + (long)&site->key; in __static_call_key()
61 site->key = (__static_call_key(site) | STATIC_CALL_SITE_INIT) - in static_call_set_init()
62 (long)&site->key; in static_call_set_init()
89 a->key = b->key - delta; in static_call_site_swap()
92 b->key = tmp.key + delta; in static_call_site_swap()
102 static inline bool static_call_key_has_mods(struct static_call_key *key) in static_call_key_has_mods() argument
104 return !(key->type & 1); in static_call_key_has_mods()
107 static inline struct static_call_mod *static_call_key_next(struct static_call_key *key) in static_call_key_next() argument
109 if (!static_call_key_has_mods(key)) in static_call_key_next()
112 return key->mods; in static_call_key_next()
[all …]
|
| /kernel/sched/ |
| D | wait_bit.c | 23 struct wait_bit_key *key = arg; in wake_bit_function() local
26 if (wait_bit->key.flags != key->flags || in wake_bit_function()
27 wait_bit->key.bit_nr != key->bit_nr || in wake_bit_function()
28 test_bit(key->bit_nr, key->flags)) in wake_bit_function()
31 return autoremove_wake_function(wq_entry, mode, sync, key); in wake_bit_function()
48 if (test_bit(wbq_entry->key.bit_nr, wbq_entry->key.flags)) in __wait_on_bit()
49 ret = (*action)(&wbq_entry->key, mode); in __wait_on_bit()
50 } while (test_bit_acquire(wbq_entry->key.bit_nr, wbq_entry->key.flags) && !ret); in __wait_on_bit()
75 wq_entry.key.timeout = jiffies + timeout; in out_of_line_wait_on_bit_timeout()
89 if (test_bit(wbq_entry->key.bit_nr, wbq_entry->key.flags)) { in __wait_on_bit_lock()
[all …]
|
| D | wait.c | 9 …_init_waitqueue_head(struct wait_queue_head *wq_head, const char *name, struct lock_class_key *key) in __init_waitqueue_head() argument
12 lockdep_set_class_and_name(&wq_head->lock, key, name); in __init_waitqueue_head()
82 int nr_exclusive, int wake_flags, void *key, in __wake_up_common() argument
108 ret = curr->func(curr, mode, wake_flags, key); in __wake_up_common()
126 int nr_exclusive, int wake_flags, void *key) in __wake_up_common_lock() argument
140 wake_flags, key, &bookmark); in __wake_up_common_lock()
159 int nr_exclusive, void *key) in __wake_up() argument
161 return __wake_up_common_lock(wq_head, mode, nr_exclusive, 0, key); in __wake_up()
174 void __wake_up_locked_key(struct wait_queue_head *wq_head, unsigned int mode, void *key) in __wake_up_locked_key() argument
176 __wake_up_common(wq_head, mode, 1, 0, key, NULL); in __wake_up_locked_key()
[all …]
|
| /kernel/bpf/ |
| D | hashtab.c | 128 char key[] __aligned(8); member
200 *(void __percpu **)(l->key + key_size) = pptr; in htab_elem_set_ptr()
205 return *(void __percpu **)(l->key + key_size); in htab_elem_get_ptr()
210 return *(void **)(l->key + roundup(map->key_size, 8)); in fd_htab_map_get_ptr()
237 bpf_timer_cancel_and_free(elem->key + in htab_free_prealloced_timers()
258 bpf_map_free_kptrs(&htab->map, elem->key + round_up(htab->map.key_size, 8)); in htab_free_prealloced_kptrs()
293 static struct htab_elem *prealloc_lru_pop(struct bpf_htab *htab, void *key, in prealloc_lru_pop() argument
301 memcpy(l->key, key, htab->map.key_size); in prealloc_lru_pop()
603 static inline u32 htab_map_hash(const void *key, u32 key_len, u32 hashrnd) in htab_map_hash() argument
605 return jhash(key, key_len, hashrnd); in htab_map_hash()
[all …]
|
| D | lpm_trie.c | 167 const struct bpf_lpm_trie_key *key) in longest_prefix_match() argument
169 u32 limit = min(node->prefixlen, key->prefixlen); in longest_prefix_match()
182 *(__be64 *)key->data); in longest_prefix_match()
195 *(__be32 *)&key->data[i]); in longest_prefix_match()
207 *(__be16 *)&key->data[i]); in longest_prefix_match()
218 prefixlen += 8 - fls(node->data[i] ^ key->data[i]); in longest_prefix_match()
232 struct bpf_lpm_trie_key *key = _key; in trie_lookup_elem() local
234 if (key->prefixlen > trie->max_prefixlen) in trie_lookup_elem()
248 matchlen = longest_prefix_match(trie, node, key); in trie_lookup_elem()
271 next_bit = extract_bit(key->data, node->prefixlen); in trie_lookup_elem()
[all …]
|
| D | local_storage.c | 68 void *key, bool locked) in cgroup_storage_lookup() argument
82 switch (bpf_cgroup_storage_key_cmp(map, key, &storage->key)) { in cgroup_storage_lookup()
114 switch (bpf_cgroup_storage_key_cmp(map, &storage->key, &this->key)) { in cgroup_storage_insert()
132 static void *cgroup_storage_lookup_elem(struct bpf_map *_map, void *key) in cgroup_storage_lookup_elem() argument
137 storage = cgroup_storage_lookup(map, key, false); in cgroup_storage_lookup_elem()
144 static int cgroup_storage_update_elem(struct bpf_map *map, void *key, in cgroup_storage_update_elem() argument
158 key, false); in cgroup_storage_update_elem()
182 int bpf_percpu_cgroup_storage_copy(struct bpf_map *_map, void *key, in bpf_percpu_cgroup_storage_copy() argument
191 storage = cgroup_storage_lookup(map, key, false); in bpf_percpu_cgroup_storage_copy()
211 int bpf_percpu_cgroup_storage_update(struct bpf_map *_map, void *key, in bpf_percpu_cgroup_storage_update() argument
[all …]
|
| D | arraymap.c | 163 static void *array_map_lookup_elem(struct bpf_map *map, void *key) in array_map_lookup_elem() argument
166 u32 index = *(u32 *)key; in array_map_lookup_elem()
238 static void *percpu_array_map_lookup_elem(struct bpf_map *map, void *key) in percpu_array_map_lookup_elem() argument
241 u32 index = *(u32 *)key; in percpu_array_map_lookup_elem()
249 static void *percpu_array_map_lookup_percpu_elem(struct bpf_map *map, void *key, u32 cpu) in percpu_array_map_lookup_percpu_elem() argument
252 u32 index = *(u32 *)key; in percpu_array_map_lookup_percpu_elem()
263 int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value) in bpf_percpu_array_copy() argument
266 u32 index = *(u32 *)key; in bpf_percpu_array_copy()
291 static int array_map_get_next_key(struct bpf_map *map, void *key, void *next_key) in array_map_get_next_key() argument
294 u32 index = key ? *(u32 *)key : U32_MAX; in array_map_get_next_key()
[all …]
|
| D | reuseport_array.c | 50 static void *reuseport_array_lookup_elem(struct bpf_map *map, void *key) in reuseport_array_lookup_elem() argument
53 u32 index = *(u32 *)key; in reuseport_array_lookup_elem()
62 static int reuseport_array_delete_elem(struct bpf_map *map, void *key) in reuseport_array_delete_elem() argument
65 u32 index = *(u32 *)key; in reuseport_array_delete_elem()
168 int bpf_fd_reuseport_array_lookup_elem(struct bpf_map *map, void *key, in bpf_fd_reuseport_array_lookup_elem() argument
178 sk = reuseport_array_lookup_elem(map, key); in bpf_fd_reuseport_array_lookup_elem()
235 int bpf_fd_reuseport_array_update_elem(struct bpf_map *map, void *key, in bpf_fd_reuseport_array_update_elem() argument
241 u32 index = *(u32 *)key; in bpf_fd_reuseport_array_update_elem()
319 static int reuseport_array_get_next_key(struct bpf_map *map, void *key, in reuseport_array_get_next_key() argument
323 u32 index = key ? *(u32 *)key : U32_MAX; in reuseport_array_get_next_key()
|
| D | devmap.c | 246 static int dev_map_get_next_key(struct bpf_map *map, void *key, void *next_key) in dev_map_get_next_key() argument
249 u32 index = key ? *(u32 *)key : U32_MAX; in dev_map_get_next_key()
267 static void *__dev_map_hash_lookup_elem(struct bpf_map *map, u32 key) in __dev_map_hash_lookup_elem() argument
270 struct hlist_head *head = dev_map_index_hash(dtab, key); in __dev_map_hash_lookup_elem()
275 if (dev->idx == key) in __dev_map_hash_lookup_elem()
281 static int dev_map_hash_get_next_key(struct bpf_map *map, void *key, in dev_map_hash_get_next_key() argument
290 if (!key) in dev_map_hash_get_next_key()
293 idx = *(u32 *)key; in dev_map_hash_get_next_key()
429 static void *__dev_map_lookup_elem(struct bpf_map *map, u32 key) in __dev_map_lookup_elem() argument
434 if (key >= map->max_entries) in __dev_map_lookup_elem()
[all …]
|
| D | syscall.c | 180 static int bpf_map_update_value(struct bpf_map *map, struct fd f, void *key, in bpf_map_update_value() argument
187 return bpf_map_offload_update_elem(map, key, value, flags); in bpf_map_update_value()
190 return map->ops->map_update_elem(map, key, value, flags); in bpf_map_update_value()
193 return sock_map_update_elem_sys(map, key, value, flags); in bpf_map_update_value()
195 return bpf_fd_array_map_update_elem(map, f.file, key, value, in bpf_map_update_value()
202 err = bpf_percpu_hash_update(map, key, value, flags); in bpf_map_update_value()
204 err = bpf_percpu_array_update(map, key, value, flags); in bpf_map_update_value()
206 err = bpf_percpu_cgroup_storage_update(map, key, value, in bpf_map_update_value()
210 err = bpf_fd_array_map_update_elem(map, f.file, key, value, in bpf_map_update_value()
215 err = bpf_fd_htab_map_update_elem(map, f.file, key, value, in bpf_map_update_value()
[all …]
|
| D | bpf_inode_storage.c | 105 static void *bpf_fd_inode_storage_lookup_elem(struct bpf_map *map, void *key) in bpf_fd_inode_storage_lookup_elem() argument
111 fd = *(int *)key; in bpf_fd_inode_storage_lookup_elem()
121 static int bpf_fd_inode_storage_update_elem(struct bpf_map *map, void *key, in bpf_fd_inode_storage_update_elem() argument
128 fd = *(int *)key; in bpf_fd_inode_storage_update_elem()
157 static int bpf_fd_inode_storage_delete_elem(struct bpf_map *map, void *key) in bpf_fd_inode_storage_delete_elem() argument
162 fd = *(int *)key; in bpf_fd_inode_storage_delete_elem()
221 static int notsupp_get_next_key(struct bpf_map *map, void *key, in notsupp_get_next_key() argument
|
| D | inode.c | 175 void *key; member
192 kfree(iter->key); in map_iter_free()
205 iter->key = kzalloc(map->key_size, GFP_KERNEL | __GFP_NOWARN); in map_iter_alloc()
206 if (!iter->key) in map_iter_alloc()
219 void *key = map_iter(m)->key; in map_seq_next() local
229 prev_key = key; in map_seq_next()
232 if (map->ops->map_get_next_key(map, prev_key, key)) { in map_seq_next()
234 key = NULL; in map_seq_next()
237 return key; in map_seq_next()
245 return *pos ? map_iter(m)->key : SEQ_START_TOKEN; in map_seq_start()
[all …]
|
| D | bpf_task_storage.c | 118 static void *bpf_pid_task_storage_lookup_elem(struct bpf_map *map, void *key) in bpf_pid_task_storage_lookup_elem() argument
126 fd = *(int *)key; in bpf_pid_task_storage_lookup_elem()
151 static int bpf_pid_task_storage_update_elem(struct bpf_map *map, void *key, in bpf_pid_task_storage_update_elem() argument
160 fd = *(int *)key; in bpf_pid_task_storage_update_elem()
200 static int bpf_pid_task_storage_delete_elem(struct bpf_map *map, void *key) in bpf_pid_task_storage_delete_elem() argument
207 fd = *(int *)key; in bpf_pid_task_storage_delete_elem()
284 static int notsupp_get_next_key(struct bpf_map *map, void *key, void *next_key) in notsupp_get_next_key() argument
|
| D | trampoline.c | 135 static struct bpf_trampoline *bpf_trampoline_lookup(u64 key) in bpf_trampoline_lookup() argument
142 head = &trampoline_table[hash_64(key, TRAMPOLINE_HASH_BITS)]; in bpf_trampoline_lookup()
144 if (tr->key == key) { in bpf_trampoline_lookup()
163 tr->key = key; in bpf_trampoline_lookup()
380 static struct bpf_tramp_image *bpf_tramp_image_alloc(u64 key, u32 idx) in bpf_tramp_image_alloc() argument
407 snprintf(ksym->name, KSYM_NAME_LEN, "bpf_trampoline_%llu_%u", key, idx); in bpf_tramp_image_alloc()
441 im = bpf_tramp_image_alloc(tr->key, tr->selector); in bpf_trampoline_update()
718 u64 key; in bpf_trampoline_link_cgroup_shim() local
727 key = bpf_trampoline_compute_key(NULL, prog->aux->attach_btf, in bpf_trampoline_link_cgroup_shim()
731 tr = bpf_trampoline_get(key, &tgt_info); in bpf_trampoline_link_cgroup_shim()
[all …]
|
| D | cpumap.c | 559 static int cpu_map_delete_elem(struct bpf_map *map, void *key) in cpu_map_delete_elem() argument
562 u32 key_cpu = *(u32 *)key; in cpu_map_delete_elem()
572 static int cpu_map_update_elem(struct bpf_map *map, void *key, void *value, in cpu_map_update_elem() argument
579 u32 key_cpu = *(u32 *)key; in cpu_map_update_elem()
648 static void *__cpu_map_lookup_elem(struct bpf_map *map, u32 key) in __cpu_map_lookup_elem() argument
653 if (key >= map->max_entries) in __cpu_map_lookup_elem()
656 rcpu = rcu_dereference_check(cmap->cpu_map[key], in __cpu_map_lookup_elem()
661 static void *cpu_map_lookup_elem(struct bpf_map *map, void *key) in cpu_map_lookup_elem() argument
664 __cpu_map_lookup_elem(map, *(u32 *)key); in cpu_map_lookup_elem()
669 static int cpu_map_get_next_key(struct bpf_map *map, void *key, void *next_key) in cpu_map_get_next_key() argument
[all …]
|
| D | bpf_struct_ops.c | 236 static int bpf_struct_ops_map_get_next_key(struct bpf_map *map, void *key, in bpf_struct_ops_map_get_next_key() argument
239 if (key && *(u32 *)key == 0) in bpf_struct_ops_map_get_next_key()
246 int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map *map, void *key, in bpf_struct_ops_map_sys_lookup_elem() argument
253 if (unlikely(*(u32 *)key != 0)) in bpf_struct_ops_map_sys_lookup_elem()
275 static void *bpf_struct_ops_map_lookup_elem(struct bpf_map *map, void *key) in bpf_struct_ops_map_lookup_elem() argument
352 static int bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, in bpf_struct_ops_map_update_elem() argument
369 if (*(u32 *)key != 0) in bpf_struct_ops_map_update_elem()
528 static int bpf_struct_ops_map_delete_elem(struct bpf_map *map, void *key) in bpf_struct_ops_map_delete_elem() argument
554 static void bpf_struct_ops_map_seq_show_elem(struct bpf_map *map, void *key, in bpf_struct_ops_map_seq_show_elem() argument
564 err = bpf_struct_ops_map_sys_lookup_elem(map, key, value); in bpf_struct_ops_map_seq_show_elem()
|
| /kernel/trace/ |
| D | tracing_map.c | 399 kfree(elt->key); in tracing_map_elt_free()
414 elt->key = kzalloc(map->key_size, GFP_KERNEL); in tracing_map_elt_alloc()
415 if (!elt->key) { in tracing_map_elt_alloc()
505 static inline bool keys_match(void *key, void *test_key, unsigned key_size) in keys_match() argument
509 if (memcmp(key, test_key, key_size)) in keys_match()
516 __tracing_map_insert(struct tracing_map *map, void *key, bool lookup_only) in __tracing_map_insert() argument
523 key_hash = jhash(key, map->key_size, 0); in __tracing_map_insert()
531 test_key = entry->key; in __tracing_map_insert()
536 keys_match(key, val->key, map->key_size)) { in __tracing_map_insert()
566 if (!cmpxchg(&entry->key, 0, key_hash)) { in __tracing_map_insert()
[all …]
|
| D | tracing_map.h | 143 void *key; member
148 u32 key; member
158 void *key; member
257 tracing_map_insert(struct tracing_map *map, void *key);
259 tracing_map_lookup(struct tracing_map *map, void *key);
|
| /kernel/futex/ |
| D | core.c | 116 struct futex_hash_bucket *futex_hash(union futex_key *key) in futex_hash() argument
118 u32 hash = jhash2((u32 *)key, offsetof(typeof(*key), both.offset) / 4, in futex_hash()
119 key->both.offset); in futex_hash()
221 int get_futex_key(u32 __user *uaddr, bool fshared, union futex_key *key, in get_futex_key() argument
233 key->both.offset = address % PAGE_SIZE; in get_futex_key()
236 address -= key->both.offset; in get_futex_key()
259 key->private.mm = mm; in get_futex_key()
261 key->private.mm = NULL; in get_futex_key()
263 key->private.address = address; in get_futex_key()
362 key->both.offset |= FUT_OFF_MMSHARED; /* ref taken on mm */ in get_futex_key()
[all …]
|
| D | pi.c | 28 pi_state->key = FUTEX_KEY_INIT; in refill_pi_state_cache()
376 static void __attach_to_pi_owner(struct task_struct *p, union futex_key *key, in __attach_to_pi_owner() argument
394 pi_state->key = *key; in __attach_to_pi_owner()
410 static int attach_to_pi_owner(u32 __user *uaddr, u32 uval, union futex_key *key, in attach_to_pi_owner() argument
466 __attach_to_pi_owner(p, key, ps); in attach_to_pi_owner()
515 union futex_key *key, in futex_lock_pi_atomic() argument
548 top_waiter = futex_top_waiter(hb, key); in futex_lock_pi_atomic()
587 __attach_to_pi_owner(task, key, ps); in futex_lock_pi_atomic()
607 return attach_to_pi_owner(uaddr, newval, key, ps, exiting); in futex_lock_pi_atomic()
948 ret = get_futex_key(uaddr, flags & FLAGS_SHARED, &q.key, FUTEX_WRITE); in futex_lock_pi()
[all …]
|
| D | waitwake.c | 148 union futex_key key = FUTEX_KEY_INIT; in futex_wake() local
156 ret = get_futex_key(uaddr, flags & FLAGS_SHARED, &key, FUTEX_READ); in futex_wake()
160 hb = futex_hash(&key); in futex_wake()
168 trace_android_vh_futex_wake_traverse_plist(&hb->chain, &target_nr, key, bitset); in futex_wake()
170 if (futex_match (&this->key, &key)) { in futex_wake()
292 if (futex_match (&this->key, &key1)) { in futex_wake_op()
306 if (futex_match (&this->key, &key2)) { in futex_wake_op()
434 &vs[i].q.key, FUTEX_READ); in futex_wait_multiple_setup()
609 ret = get_futex_key(uaddr, flags & FLAGS_SHARED, &q->key, FUTEX_READ); in futex_wait_setup()
|
| D | futex.h | 69 union futex_key key; member
101 union futex_key key; member
119 extern int get_futex_key(u32 __user *uaddr, bool fshared, union futex_key *key,
126 extern struct futex_hash_bucket *futex_hash(union futex_key *key);
152 extern struct futex_q *futex_top_waiter(struct futex_hash_bucket *hb, union futex_key *key);
224 union futex_key *key,
|
| /kernel/debug/kdb/ |
| D | kdb_io.c | 133 int key; in kdb_getchar() local
142 key = (*f)(); in kdb_getchar()
143 if (key == -1) { in kdb_getchar()
163 *pbuf++ = key; in kdb_getchar()
164 key = kdb_handle_escape(buf, pbuf - buf); in kdb_getchar()
165 if (key < 0) /* no escape sequence; return best character */ in kdb_getchar()
167 if (key > 0) in kdb_getchar()
168 return key; in kdb_getchar()
210 int key, buf_size, ret; in kdb_read() local
227 key = kdb_getchar(); in kdb_read()
[all …]
|
| /kernel/locking/ |
| D | lockdep.c | 407 #define __classhashfn(key) hash_long((unsigned long)key, CLASSHASH_BITS) argument
408 #define classhashentry(key) (classhash_table + __classhashfn((key))) argument
444 static inline u64 iterate_chain_key(u64 key, u32 idx) in iterate_chain_key() argument
446 u32 k0 = key, k1 = key >> 32; in iterate_chain_key()
663 const char *__get_key_name(const struct lockdep_subclass_key *key, char *str) in __get_key_name() argument
665 return kallsyms_lookup((unsigned long)key, NULL, NULL, NULL, str); in __get_key_name()
719 name = __get_key_name(class->key, str); in __print_lock_name()
750 name = __get_key_name(lock->key->subkeys, str); in print_lockdep_cache()
869 if (new_class->key - new_class->subclass == class->key) in count_matching_names()
882 struct lockdep_subclass_key *key; in look_up_lock_class() local
[all …]
|