Lines Matching refs:map
93 struct bpf_map map; member
132 return !(htab->map.map_flags & BPF_F_NO_PREALLOC); in htab_is_prealloc()
213 return htab->map.map_type == BPF_MAP_TYPE_LRU_HASH || in htab_is_lru()
214 htab->map.map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH; in htab_is_lru()
219 return htab->map.map_type == BPF_MAP_TYPE_PERCPU_HASH || in htab_is_percpu()
220 htab->map.map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH; in htab_is_percpu()
234 static void *fd_htab_map_get_ptr(const struct bpf_map *map, struct htab_elem *l) in fd_htab_map_get_ptr() argument
236 return *(void **)(l->key + roundup(map->key_size, 8)); in fd_htab_map_get_ptr()
251 u32 num_entries = htab->map.max_entries; in htab_free_prealloced_timers()
254 if (likely(!map_value_has_timer(&htab->map))) in htab_free_prealloced_timers()
264 round_up(htab->map.key_size, 8) + in htab_free_prealloced_timers()
265 htab->map.timer_off); in htab_free_prealloced_timers()
277 for (i = 0; i < htab->map.max_entries; i++) { in htab_free_elems()
281 htab->map.key_size); in htab_free_elems()
308 memcpy(l->key, key, htab->map.key_size); in prealloc_lru_pop()
317 u32 num_entries = htab->map.max_entries; in prealloc_init()
324 htab->map.numa_node); in prealloc_init()
332 u32 size = round_up(htab->map.value_size, 8); in prealloc_init()
335 pptr = bpf_map_alloc_percpu(&htab->map, size, 8, in prealloc_init()
339 htab_elem_set_ptr(get_htab_elem(htab, i), htab->map.key_size, in prealloc_init()
347 htab->map.map_flags & BPF_F_NO_COMMON_LRU, in prealloc_init()
390 pptr = bpf_map_alloc_percpu(&htab->map, sizeof(struct htab_elem *), 8, in alloc_extra_elems()
493 bpf_map_init_from_attr(&htab->map, attr); in htab_map_alloc()
500 htab->map.max_entries = roundup(attr->max_entries, in htab_map_alloc()
502 if (htab->map.max_entries < attr->max_entries) in htab_map_alloc()
503 htab->map.max_entries = rounddown(attr->max_entries, in htab_map_alloc()
508 htab->n_buckets = roundup_pow_of_two(htab->map.max_entries); in htab_map_alloc()
511 round_up(htab->map.key_size, 8); in htab_map_alloc()
515 htab->elem_size += round_up(htab->map.value_size, 8); in htab_map_alloc()
526 htab->map.numa_node); in htab_map_alloc()
531 htab->map_locked[i] = bpf_map_alloc_percpu(&htab->map, in htab_map_alloc()
539 if (htab->map.map_flags & BPF_F_ZERO_SEED) in htab_map_alloc()
561 return &htab->map; in htab_map_alloc()
631 static void *__htab_map_lookup_elem(struct bpf_map *map, void *key) in __htab_map_lookup_elem() argument
633 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in __htab_map_lookup_elem()
641 key_size = map->key_size; in __htab_map_lookup_elem()
652 static void *htab_map_lookup_elem(struct bpf_map *map, void *key) in htab_map_lookup_elem() argument
654 struct htab_elem *l = __htab_map_lookup_elem(map, key); in htab_map_lookup_elem()
657 return l->key + round_up(map->key_size, 8); in htab_map_lookup_elem()
673 static int htab_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf) in htab_map_gen_lookup() argument
679 (void *(*)(struct bpf_map *map, void *key))NULL)); in htab_map_gen_lookup()
684 round_up(map->key_size, 8)); in htab_map_gen_lookup()
688 static __always_inline void *__htab_lru_map_lookup_elem(struct bpf_map *map, in __htab_lru_map_lookup_elem() argument
691 struct htab_elem *l = __htab_map_lookup_elem(map, key); in __htab_lru_map_lookup_elem()
696 return l->key + round_up(map->key_size, 8); in __htab_lru_map_lookup_elem()
702 static void *htab_lru_map_lookup_elem(struct bpf_map *map, void *key) in htab_lru_map_lookup_elem() argument
704 return __htab_lru_map_lookup_elem(map, key, true); in htab_lru_map_lookup_elem()
707 static void *htab_lru_map_lookup_elem_sys(struct bpf_map *map, void *key) in htab_lru_map_lookup_elem_sys() argument
709 return __htab_lru_map_lookup_elem(map, key, false); in htab_lru_map_lookup_elem_sys()
712 static int htab_lru_map_gen_lookup(struct bpf_map *map, in htab_lru_map_gen_lookup() argument
720 (void *(*)(struct bpf_map *map, void *key))NULL)); in htab_lru_map_gen_lookup()
733 round_up(map->key_size, 8)); in htab_lru_map_gen_lookup()
739 if (unlikely(map_value_has_timer(&htab->map))) in check_and_free_timer()
741 round_up(htab->map.key_size, 8) + in check_and_free_timer()
742 htab->map.timer_off); in check_and_free_timer()
779 static int htab_map_get_next_key(struct bpf_map *map, void *key, void *next_key) in htab_map_get_next_key() argument
781 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in htab_map_get_next_key()
789 key_size = map->key_size; in htab_map_get_next_key()
839 if (htab->map.map_type == BPF_MAP_TYPE_PERCPU_HASH) in htab_elem_free()
840 free_percpu(htab_elem_get_ptr(l, htab->map.key_size)); in htab_elem_free()
855 struct bpf_map *map = &htab->map; in htab_put_fd_value() local
858 if (map->ops->map_fd_put_ptr) { in htab_put_fd_value()
859 ptr = fd_htab_map_get_ptr(map, l); in htab_put_fd_value()
860 map->ops->map_fd_put_ptr(ptr); in htab_put_fd_value()
883 memcpy(this_cpu_ptr(pptr), value, htab->map.value_size); in pcpu_copy_value()
885 u32 size = round_up(htab->map.value_size, 8); in pcpu_copy_value()
906 u32 size = round_up(htab->map.value_size, 8); in pcpu_init_value()
924 return htab->map.map_type == BPF_MAP_TYPE_HASH_OF_MAPS && in fd_htab_map_needs_adjust()
933 u32 size = htab->map.value_size; in alloc_htab_elem()
956 if (atomic_inc_return(&htab->count) > htab->map.max_entries) in alloc_htab_elem()
966 l_new = bpf_map_kmalloc_node(&htab->map, htab->elem_size, in alloc_htab_elem()
968 htab->map.numa_node); in alloc_htab_elem()
973 check_and_init_map_value(&htab->map, in alloc_htab_elem()
984 pptr = bpf_map_alloc_percpu(&htab->map, size, 8, in alloc_htab_elem()
1001 copy_map_value(&htab->map, in alloc_htab_elem()
1028 static int htab_map_update_elem(struct bpf_map *map, void *key, void *value, in htab_map_update_elem() argument
1031 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in htab_map_update_elem()
1046 key_size = map->key_size; in htab_map_update_elem()
1054 if (unlikely(!map_value_has_spin_lock(map))) in htab_map_update_elem()
1064 copy_map_value_locked(map, in htab_map_update_elem()
1092 copy_map_value_locked(map, in htab_map_update_elem()
1130 static int htab_lru_map_update_elem(struct bpf_map *map, void *key, void *value, in htab_lru_map_update_elem() argument
1133 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in htab_lru_map_update_elem()
1148 key_size = map->key_size; in htab_lru_map_update_elem()
1163 copy_map_value(&htab->map, in htab_lru_map_update_elem()
1164 l_new->key + round_up(map->key_size, 8), value); in htab_lru_map_update_elem()
1198 static int __htab_percpu_map_update_elem(struct bpf_map *map, void *key, in __htab_percpu_map_update_elem() argument
1202 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in __htab_percpu_map_update_elem()
1217 key_size = map->key_size; in __htab_percpu_map_update_elem()
1253 static int __htab_lru_percpu_map_update_elem(struct bpf_map *map, void *key, in __htab_lru_percpu_map_update_elem() argument
1257 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in __htab_lru_percpu_map_update_elem()
1272 key_size = map->key_size; in __htab_lru_percpu_map_update_elem()
1321 static int htab_percpu_map_update_elem(struct bpf_map *map, void *key, in htab_percpu_map_update_elem() argument
1324 return __htab_percpu_map_update_elem(map, key, value, map_flags, false); in htab_percpu_map_update_elem()
1327 static int htab_lru_percpu_map_update_elem(struct bpf_map *map, void *key, in htab_lru_percpu_map_update_elem() argument
1330 return __htab_lru_percpu_map_update_elem(map, key, value, map_flags, in htab_lru_percpu_map_update_elem()
1335 static int htab_map_delete_elem(struct bpf_map *map, void *key) in htab_map_delete_elem() argument
1337 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in htab_map_delete_elem()
1348 key_size = map->key_size; in htab_map_delete_elem()
1371 static int htab_lru_map_delete_elem(struct bpf_map *map, void *key) in htab_lru_map_delete_elem() argument
1373 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in htab_lru_map_delete_elem()
1384 key_size = map->key_size; in htab_lru_map_delete_elem()
1440 static void htab_map_free_timers(struct bpf_map *map) in htab_map_free_timers() argument
1442 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in htab_map_free_timers()
1444 if (likely(!map_value_has_timer(&htab->map))) in htab_map_free_timers()
1453 static void htab_map_free(struct bpf_map *map) in htab_map_free() argument
1455 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in htab_map_free()
1480 static void htab_map_seq_show_elem(struct bpf_map *map, void *key, in htab_map_seq_show_elem() argument
1487 value = htab_map_lookup_elem(map, key); in htab_map_seq_show_elem()
1493 btf_type_seq_show(map->btf, map->btf_key_type_id, key, m); in htab_map_seq_show_elem()
1495 btf_type_seq_show(map->btf, map->btf_value_type_id, value, m); in htab_map_seq_show_elem()
1501 static int __htab_map_lookup_and_delete_elem(struct bpf_map *map, void *key, in __htab_map_lookup_and_delete_elem() argument
1505 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in __htab_map_lookup_and_delete_elem()
1513 key_size = map->key_size; in __htab_map_lookup_and_delete_elem()
1528 u32 roundup_value_size = round_up(map->value_size, 8); in __htab_map_lookup_and_delete_elem()
1540 u32 roundup_key_size = round_up(map->key_size, 8); in __htab_map_lookup_and_delete_elem()
1543 copy_map_value_locked(map, value, l->key + in __htab_map_lookup_and_delete_elem()
1547 copy_map_value(map, value, l->key + in __htab_map_lookup_and_delete_elem()
1549 check_and_init_map_value(map, value); in __htab_map_lookup_and_delete_elem()
1565 static int htab_map_lookup_and_delete_elem(struct bpf_map *map, void *key, in htab_map_lookup_and_delete_elem() argument
1568 return __htab_map_lookup_and_delete_elem(map, key, value, false, false, in htab_map_lookup_and_delete_elem()
1572 static int htab_percpu_map_lookup_and_delete_elem(struct bpf_map *map, in htab_percpu_map_lookup_and_delete_elem() argument
1576 return __htab_map_lookup_and_delete_elem(map, key, value, false, true, in htab_percpu_map_lookup_and_delete_elem()
1580 static int htab_lru_map_lookup_and_delete_elem(struct bpf_map *map, void *key, in htab_lru_map_lookup_and_delete_elem() argument
1583 return __htab_map_lookup_and_delete_elem(map, key, value, true, false, in htab_lru_map_lookup_and_delete_elem()
1587 static int htab_lru_percpu_map_lookup_and_delete_elem(struct bpf_map *map, in htab_lru_percpu_map_lookup_and_delete_elem() argument
1591 return __htab_map_lookup_and_delete_elem(map, key, value, true, true, in htab_lru_percpu_map_lookup_and_delete_elem()
1596 __htab_map_lookup_and_delete_batch(struct bpf_map *map, in __htab_map_lookup_and_delete_batch() argument
1602 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in __htab_map_lookup_and_delete_batch()
1621 ((elem_map_flags & BPF_F_LOCK) && !map_value_has_spin_lock(map))) in __htab_map_lookup_and_delete_batch()
1642 key_size = htab->map.key_size; in __htab_map_lookup_and_delete_batch()
1643 roundup_key_size = round_up(htab->map.key_size, 8); in __htab_map_lookup_and_delete_batch()
1644 value_size = htab->map.value_size; in __htab_map_lookup_and_delete_batch()
1728 pptr = htab_elem_get_ptr(l, map->key_size); in __htab_map_lookup_and_delete_batch()
1737 copy_map_value_locked(map, dst_val, value, in __htab_map_lookup_and_delete_batch()
1740 copy_map_value(map, dst_val, value); in __htab_map_lookup_and_delete_batch()
1741 check_and_init_map_value(map, dst_val); in __htab_map_lookup_and_delete_batch()
1815 htab_percpu_map_lookup_batch(struct bpf_map *map, const union bpf_attr *attr, in htab_percpu_map_lookup_batch() argument
1818 return __htab_map_lookup_and_delete_batch(map, attr, uattr, false, in htab_percpu_map_lookup_batch()
1823 htab_percpu_map_lookup_and_delete_batch(struct bpf_map *map, in htab_percpu_map_lookup_and_delete_batch() argument
1827 return __htab_map_lookup_and_delete_batch(map, attr, uattr, true, in htab_percpu_map_lookup_and_delete_batch()
1832 htab_map_lookup_batch(struct bpf_map *map, const union bpf_attr *attr, in htab_map_lookup_batch() argument
1835 return __htab_map_lookup_and_delete_batch(map, attr, uattr, false, in htab_map_lookup_batch()
1840 htab_map_lookup_and_delete_batch(struct bpf_map *map, in htab_map_lookup_and_delete_batch() argument
1844 return __htab_map_lookup_and_delete_batch(map, attr, uattr, true, in htab_map_lookup_and_delete_batch()
1849 htab_lru_percpu_map_lookup_batch(struct bpf_map *map, in htab_lru_percpu_map_lookup_batch() argument
1853 return __htab_map_lookup_and_delete_batch(map, attr, uattr, false, in htab_lru_percpu_map_lookup_batch()
1858 htab_lru_percpu_map_lookup_and_delete_batch(struct bpf_map *map, in htab_lru_percpu_map_lookup_and_delete_batch() argument
1862 return __htab_map_lookup_and_delete_batch(map, attr, uattr, true, in htab_lru_percpu_map_lookup_and_delete_batch()
1867 htab_lru_map_lookup_batch(struct bpf_map *map, const union bpf_attr *attr, in htab_lru_map_lookup_batch() argument
1870 return __htab_map_lookup_and_delete_batch(map, attr, uattr, false, in htab_lru_map_lookup_batch()
1875 htab_lru_map_lookup_and_delete_batch(struct bpf_map *map, in htab_lru_map_lookup_and_delete_batch() argument
1879 return __htab_map_lookup_and_delete_batch(map, attr, uattr, true, in htab_lru_map_lookup_and_delete_batch()
1884 struct bpf_map *map; member
1975 struct bpf_map *map = info->map; in __bpf_hash_map_seq_show() local
1985 ctx.map = info->map; in __bpf_hash_map_seq_show()
1987 roundup_key_size = round_up(map->key_size, 8); in __bpf_hash_map_seq_show()
1992 roundup_value_size = round_up(map->value_size, 8); in __bpf_hash_map_seq_show()
1993 pptr = htab_elem_get_ptr(elem, map->key_size); in __bpf_hash_map_seq_show()
2026 struct bpf_map *map = aux->map; in bpf_iter_init_hash_map() local
2030 if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || in bpf_iter_init_hash_map()
2031 map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) { in bpf_iter_init_hash_map()
2032 buf_size = round_up(map->value_size, 8) * num_possible_cpus(); in bpf_iter_init_hash_map()
2040 bpf_map_inc_with_uref(map); in bpf_iter_init_hash_map()
2041 seq_info->map = map; in bpf_iter_init_hash_map()
2042 seq_info->htab = container_of(map, struct bpf_htab, map); in bpf_iter_init_hash_map()
2050 bpf_map_put_with_uref(seq_info->map); in bpf_iter_fini_hash_map()
2068 static int bpf_for_each_hash_elem(struct bpf_map *map, void *callback_fn, in bpf_for_each_hash_elem() argument
2071 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in bpf_for_each_hash_elem()
2088 roundup_key_size = round_up(map->key_size, 8); in bpf_for_each_hash_elem()
2102 pptr = htab_elem_get_ptr(elem, map->key_size); in bpf_for_each_hash_elem()
2108 ret = BPF_CAST_CALL(callback_fn)((u64)(long)map, in bpf_for_each_hash_elem()
2171 static void *htab_percpu_map_lookup_elem(struct bpf_map *map, void *key) in htab_percpu_map_lookup_elem() argument
2173 struct htab_elem *l = __htab_map_lookup_elem(map, key); in htab_percpu_map_lookup_elem()
2176 return this_cpu_ptr(htab_elem_get_ptr(l, map->key_size)); in htab_percpu_map_lookup_elem()
2181 static void *htab_lru_percpu_map_lookup_elem(struct bpf_map *map, void *key) in htab_lru_percpu_map_lookup_elem() argument
2183 struct htab_elem *l = __htab_map_lookup_elem(map, key); in htab_lru_percpu_map_lookup_elem()
2187 return this_cpu_ptr(htab_elem_get_ptr(l, map->key_size)); in htab_lru_percpu_map_lookup_elem()
2193 int bpf_percpu_hash_copy(struct bpf_map *map, void *key, void *value) in bpf_percpu_hash_copy() argument
2205 size = round_up(map->value_size, 8); in bpf_percpu_hash_copy()
2207 l = __htab_map_lookup_elem(map, key); in bpf_percpu_hash_copy()
2213 pptr = htab_elem_get_ptr(l, map->key_size); in bpf_percpu_hash_copy()
2225 int bpf_percpu_hash_update(struct bpf_map *map, void *key, void *value, in bpf_percpu_hash_update() argument
2228 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in bpf_percpu_hash_update()
2233 ret = __htab_lru_percpu_map_update_elem(map, key, value, in bpf_percpu_hash_update()
2236 ret = __htab_percpu_map_update_elem(map, key, value, map_flags, in bpf_percpu_hash_update()
2243 static void htab_percpu_map_seq_show_elem(struct bpf_map *map, void *key, in htab_percpu_map_seq_show_elem() argument
2252 l = __htab_map_lookup_elem(map, key); in htab_percpu_map_seq_show_elem()
2258 btf_type_seq_show(map->btf, map->btf_key_type_id, key, m); in htab_percpu_map_seq_show_elem()
2260 pptr = htab_elem_get_ptr(l, map->key_size); in htab_percpu_map_seq_show_elem()
2263 btf_type_seq_show(map->btf, map->btf_value_type_id, in htab_percpu_map_seq_show_elem()
2319 static void fd_htab_map_free(struct bpf_map *map) in fd_htab_map_free() argument
2321 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in fd_htab_map_free()
2331 void *ptr = fd_htab_map_get_ptr(map, l); in fd_htab_map_free()
2333 map->ops->map_fd_put_ptr(ptr); in fd_htab_map_free()
2337 htab_map_free(map); in fd_htab_map_free()
2341 int bpf_fd_htab_map_lookup_elem(struct bpf_map *map, void *key, u32 *value) in bpf_fd_htab_map_lookup_elem() argument
2346 if (!map->ops->map_fd_sys_lookup_elem) in bpf_fd_htab_map_lookup_elem()
2350 ptr = htab_map_lookup_elem(map, key); in bpf_fd_htab_map_lookup_elem()
2352 *value = map->ops->map_fd_sys_lookup_elem(READ_ONCE(*ptr)); in bpf_fd_htab_map_lookup_elem()
2361 int bpf_fd_htab_map_update_elem(struct bpf_map *map, struct file *map_file, in bpf_fd_htab_map_update_elem() argument
2368 ptr = map->ops->map_fd_get_ptr(map, map_file, ufd); in bpf_fd_htab_map_update_elem()
2372 ret = htab_map_update_elem(map, key, &ptr, map_flags); in bpf_fd_htab_map_update_elem()
2374 map->ops->map_fd_put_ptr(ptr); in bpf_fd_htab_map_update_elem()
2381 struct bpf_map *map, *inner_map_meta; in htab_of_map_alloc() local
2387 map = htab_map_alloc(attr); in htab_of_map_alloc()
2388 if (IS_ERR(map)) { in htab_of_map_alloc()
2390 return map; in htab_of_map_alloc()
2393 map->inner_map_meta = inner_map_meta; in htab_of_map_alloc()
2395 return map; in htab_of_map_alloc()
2398 static void *htab_of_map_lookup_elem(struct bpf_map *map, void *key) in htab_of_map_lookup_elem() argument
2400 struct bpf_map **inner_map = htab_map_lookup_elem(map, key); in htab_of_map_lookup_elem()
2408 static int htab_of_map_gen_lookup(struct bpf_map *map, in htab_of_map_gen_lookup() argument
2415 (void *(*)(struct bpf_map *map, void *key))NULL)); in htab_of_map_gen_lookup()
2420 round_up(map->key_size, 8)); in htab_of_map_gen_lookup()
2426 static void htab_of_map_free(struct bpf_map *map) in htab_of_map_free() argument
2428 bpf_map_meta_free(map->inner_map_meta); in htab_of_map_free()
2429 fd_htab_map_free(map); in htab_of_map_free()