Lines Matching refs:map
87 struct bpf_map map; member
133 return !(htab->map.map_flags & BPF_F_NO_PREALLOC); in htab_is_prealloc()
187 return htab->map.map_type == BPF_MAP_TYPE_LRU_HASH || in htab_is_lru()
188 htab->map.map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH; in htab_is_lru()
193 return htab->map.map_type == BPF_MAP_TYPE_PERCPU_HASH || in htab_is_percpu()
194 htab->map.map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH; in htab_is_percpu()
208 static void *fd_htab_map_get_ptr(const struct bpf_map *map, struct htab_elem *l) in fd_htab_map_get_ptr() argument
210 return *(void **)(l->key + roundup(map->key_size, 8)); in fd_htab_map_get_ptr()
225 u32 num_entries = htab->map.max_entries; in htab_free_prealloced_timers()
228 if (!map_value_has_timer(&htab->map)) in htab_free_prealloced_timers()
238 round_up(htab->map.key_size, 8) + in htab_free_prealloced_timers()
239 htab->map.timer_off); in htab_free_prealloced_timers()
246 u32 num_entries = htab->map.max_entries; in htab_free_prealloced_kptrs()
249 if (!map_value_has_kptrs(&htab->map)) in htab_free_prealloced_kptrs()
258 bpf_map_free_kptrs(&htab->map, elem->key + round_up(htab->map.key_size, 8)); in htab_free_prealloced_kptrs()
270 for (i = 0; i < htab->map.max_entries; i++) { in htab_free_elems()
274 htab->map.key_size); in htab_free_elems()
301 memcpy(l->key, key, htab->map.key_size); in prealloc_lru_pop()
310 u32 num_entries = htab->map.max_entries; in prealloc_init()
317 htab->map.numa_node); in prealloc_init()
325 u32 size = round_up(htab->map.value_size, 8); in prealloc_init()
328 pptr = bpf_map_alloc_percpu(&htab->map, size, 8, in prealloc_init()
332 htab_elem_set_ptr(get_htab_elem(htab, i), htab->map.key_size, in prealloc_init()
340 htab->map.map_flags & BPF_F_NO_COMMON_LRU, in prealloc_init()
383 pptr = bpf_map_alloc_percpu(&htab->map, sizeof(struct htab_elem *), 8, in alloc_extra_elems()
484 bpf_map_init_from_attr(&htab->map, attr); in htab_map_alloc()
491 htab->map.max_entries = roundup(attr->max_entries, in htab_map_alloc()
493 if (htab->map.max_entries < attr->max_entries) in htab_map_alloc()
494 htab->map.max_entries = rounddown(attr->max_entries, in htab_map_alloc()
499 htab->n_buckets = roundup_pow_of_two(htab->map.max_entries); in htab_map_alloc()
502 round_up(htab->map.key_size, 8); in htab_map_alloc()
506 htab->elem_size += round_up(htab->map.value_size, 8); in htab_map_alloc()
517 htab->map.numa_node); in htab_map_alloc()
522 htab->map_locked[i] = bpf_map_alloc_percpu(&htab->map, in htab_map_alloc()
530 if (htab->map.map_flags & BPF_F_ZERO_SEED) in htab_map_alloc()
579 round_up(htab->map.value_size, 8), true); in htab_map_alloc()
585 return &htab->map; in htab_map_alloc()
659 static void *__htab_map_lookup_elem(struct bpf_map *map, void *key) in __htab_map_lookup_elem() argument
661 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in __htab_map_lookup_elem()
669 key_size = map->key_size; in __htab_map_lookup_elem()
680 static void *htab_map_lookup_elem(struct bpf_map *map, void *key) in htab_map_lookup_elem() argument
682 struct htab_elem *l = __htab_map_lookup_elem(map, key); in htab_map_lookup_elem()
685 return l->key + round_up(map->key_size, 8); in htab_map_lookup_elem()
701 static int htab_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf) in htab_map_gen_lookup() argument
707 (void *(*)(struct bpf_map *map, void *key))NULL)); in htab_map_gen_lookup()
712 round_up(map->key_size, 8)); in htab_map_gen_lookup()
716 static __always_inline void *__htab_lru_map_lookup_elem(struct bpf_map *map, in __htab_lru_map_lookup_elem() argument
719 struct htab_elem *l = __htab_map_lookup_elem(map, key); in __htab_lru_map_lookup_elem()
724 return l->key + round_up(map->key_size, 8); in __htab_lru_map_lookup_elem()
730 static void *htab_lru_map_lookup_elem(struct bpf_map *map, void *key) in htab_lru_map_lookup_elem() argument
732 return __htab_lru_map_lookup_elem(map, key, true); in htab_lru_map_lookup_elem()
735 static void *htab_lru_map_lookup_elem_sys(struct bpf_map *map, void *key) in htab_lru_map_lookup_elem_sys() argument
737 return __htab_lru_map_lookup_elem(map, key, false); in htab_lru_map_lookup_elem_sys()
740 static int htab_lru_map_gen_lookup(struct bpf_map *map, in htab_lru_map_gen_lookup() argument
748 (void *(*)(struct bpf_map *map, void *key))NULL)); in htab_lru_map_gen_lookup()
761 round_up(map->key_size, 8)); in htab_lru_map_gen_lookup()
768 void *map_value = elem->key + round_up(htab->map.key_size, 8); in check_and_free_fields()
770 if (map_value_has_timer(&htab->map)) in check_and_free_fields()
771 bpf_timer_cancel_and_free(map_value + htab->map.timer_off); in check_and_free_fields()
772 if (map_value_has_kptrs(&htab->map)) in check_and_free_fields()
773 bpf_map_free_kptrs(&htab->map, map_value); in check_and_free_fields()
810 static int htab_map_get_next_key(struct bpf_map *map, void *key, void *next_key) in htab_map_get_next_key() argument
812 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in htab_map_get_next_key()
820 key_size = map->key_size; in htab_map_get_next_key()
870 if (htab->map.map_type == BPF_MAP_TYPE_PERCPU_HASH) in htab_elem_free()
878 struct bpf_map *map = &htab->map; in htab_put_fd_value() local
881 if (map->ops->map_fd_put_ptr) { in htab_put_fd_value()
882 ptr = fd_htab_map_get_ptr(map, l); in htab_put_fd_value()
883 map->ops->map_fd_put_ptr(ptr); in htab_put_fd_value()
890 return __percpu_counter_compare(&htab->pcount, htab->map.max_entries, in is_map_full()
892 return atomic_read(&htab->count) >= htab->map.max_entries; in is_map_full()
930 memcpy(this_cpu_ptr(pptr), value, htab->map.value_size); in pcpu_copy_value()
932 u32 size = round_up(htab->map.value_size, 8); in pcpu_copy_value()
952 u32 size = round_up(htab->map.value_size, 8); in pcpu_init_value()
970 return htab->map.map_type == BPF_MAP_TYPE_HASH_OF_MAPS && in fd_htab_map_needs_adjust()
979 u32 size = htab->map.value_size; in alloc_htab_elem()
1042 copy_map_value(&htab->map, in alloc_htab_elem()
1069 static int htab_map_update_elem(struct bpf_map *map, void *key, void *value, in htab_map_update_elem() argument
1072 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in htab_map_update_elem()
1087 key_size = map->key_size; in htab_map_update_elem()
1095 if (unlikely(!map_value_has_spin_lock(map))) in htab_map_update_elem()
1105 copy_map_value_locked(map, in htab_map_update_elem()
1133 copy_map_value_locked(map, in htab_map_update_elem()
1171 static int htab_lru_map_update_elem(struct bpf_map *map, void *key, void *value, in htab_lru_map_update_elem() argument
1174 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in htab_lru_map_update_elem()
1189 key_size = map->key_size; in htab_lru_map_update_elem()
1204 copy_map_value(&htab->map, in htab_lru_map_update_elem()
1205 l_new->key + round_up(map->key_size, 8), value); in htab_lru_map_update_elem()
1239 static int __htab_percpu_map_update_elem(struct bpf_map *map, void *key, in __htab_percpu_map_update_elem() argument
1243 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in __htab_percpu_map_update_elem()
1258 key_size = map->key_size; in __htab_percpu_map_update_elem()
1294 static int __htab_lru_percpu_map_update_elem(struct bpf_map *map, void *key, in __htab_lru_percpu_map_update_elem() argument
1298 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in __htab_lru_percpu_map_update_elem()
1313 key_size = map->key_size; in __htab_lru_percpu_map_update_elem()
1362 static int htab_percpu_map_update_elem(struct bpf_map *map, void *key, in htab_percpu_map_update_elem() argument
1365 return __htab_percpu_map_update_elem(map, key, value, map_flags, false); in htab_percpu_map_update_elem()
1368 static int htab_lru_percpu_map_update_elem(struct bpf_map *map, void *key, in htab_lru_percpu_map_update_elem() argument
1371 return __htab_lru_percpu_map_update_elem(map, key, value, map_flags, in htab_lru_percpu_map_update_elem()
1376 static int htab_map_delete_elem(struct bpf_map *map, void *key) in htab_map_delete_elem() argument
1378 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in htab_map_delete_elem()
1389 key_size = map->key_size; in htab_map_delete_elem()
1412 static int htab_lru_map_delete_elem(struct bpf_map *map, void *key) in htab_lru_map_delete_elem() argument
1414 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in htab_lru_map_delete_elem()
1425 key_size = map->key_size; in htab_lru_map_delete_elem()
1484 round_up(htab->map.key_size, 8) + in htab_free_malloced_timers()
1485 htab->map.timer_off); in htab_free_malloced_timers()
1492 static void htab_map_free_timers(struct bpf_map *map) in htab_map_free_timers() argument
1494 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in htab_map_free_timers()
1497 if (!map_value_has_timer(&htab->map)) in htab_map_free_timers()
1506 static void htab_map_free(struct bpf_map *map) in htab_map_free() argument
1508 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in htab_map_free()
1527 bpf_map_free_kptr_off_tab(map); in htab_map_free()
1540 static void htab_map_seq_show_elem(struct bpf_map *map, void *key, in htab_map_seq_show_elem() argument
1547 value = htab_map_lookup_elem(map, key); in htab_map_seq_show_elem()
1553 btf_type_seq_show(map->btf, map->btf_key_type_id, key, m); in htab_map_seq_show_elem()
1555 btf_type_seq_show(map->btf, map->btf_value_type_id, value, m); in htab_map_seq_show_elem()
1561 static int __htab_map_lookup_and_delete_elem(struct bpf_map *map, void *key, in __htab_map_lookup_and_delete_elem() argument
1565 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in __htab_map_lookup_and_delete_elem()
1573 key_size = map->key_size; in __htab_map_lookup_and_delete_elem()
1588 u32 roundup_value_size = round_up(map->value_size, 8); in __htab_map_lookup_and_delete_elem()
1600 u32 roundup_key_size = round_up(map->key_size, 8); in __htab_map_lookup_and_delete_elem()
1603 copy_map_value_locked(map, value, l->key + in __htab_map_lookup_and_delete_elem()
1607 copy_map_value(map, value, l->key + in __htab_map_lookup_and_delete_elem()
1610 check_and_init_map_value(map, value); in __htab_map_lookup_and_delete_elem()
1626 static int htab_map_lookup_and_delete_elem(struct bpf_map *map, void *key, in htab_map_lookup_and_delete_elem() argument
1629 return __htab_map_lookup_and_delete_elem(map, key, value, false, false, in htab_map_lookup_and_delete_elem()
1633 static int htab_percpu_map_lookup_and_delete_elem(struct bpf_map *map, in htab_percpu_map_lookup_and_delete_elem() argument
1637 return __htab_map_lookup_and_delete_elem(map, key, value, false, true, in htab_percpu_map_lookup_and_delete_elem()
1641 static int htab_lru_map_lookup_and_delete_elem(struct bpf_map *map, void *key, in htab_lru_map_lookup_and_delete_elem() argument
1644 return __htab_map_lookup_and_delete_elem(map, key, value, true, false, in htab_lru_map_lookup_and_delete_elem()
1648 static int htab_lru_percpu_map_lookup_and_delete_elem(struct bpf_map *map, in htab_lru_percpu_map_lookup_and_delete_elem() argument
1652 return __htab_map_lookup_and_delete_elem(map, key, value, true, true, in htab_lru_percpu_map_lookup_and_delete_elem()
1657 __htab_map_lookup_and_delete_batch(struct bpf_map *map, in __htab_map_lookup_and_delete_batch() argument
1663 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in __htab_map_lookup_and_delete_batch()
1682 ((elem_map_flags & BPF_F_LOCK) && !map_value_has_spin_lock(map))) in __htab_map_lookup_and_delete_batch()
1703 key_size = htab->map.key_size; in __htab_map_lookup_and_delete_batch()
1704 roundup_key_size = round_up(htab->map.key_size, 8); in __htab_map_lookup_and_delete_batch()
1705 value_size = htab->map.value_size; in __htab_map_lookup_and_delete_batch()
1789 pptr = htab_elem_get_ptr(l, map->key_size); in __htab_map_lookup_and_delete_batch()
1797 if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS) { in __htab_map_lookup_and_delete_batch()
1801 map_id = map->ops->map_fd_sys_lookup_elem(*inner_map); in __htab_map_lookup_and_delete_batch()
1806 copy_map_value_locked(map, dst_val, value, in __htab_map_lookup_and_delete_batch()
1809 copy_map_value(map, dst_val, value); in __htab_map_lookup_and_delete_batch()
1811 check_and_init_map_value(map, dst_val); in __htab_map_lookup_and_delete_batch()
1885 htab_percpu_map_lookup_batch(struct bpf_map *map, const union bpf_attr *attr, in htab_percpu_map_lookup_batch() argument
1888 return __htab_map_lookup_and_delete_batch(map, attr, uattr, false, in htab_percpu_map_lookup_batch()
1893 htab_percpu_map_lookup_and_delete_batch(struct bpf_map *map, in htab_percpu_map_lookup_and_delete_batch() argument
1897 return __htab_map_lookup_and_delete_batch(map, attr, uattr, true, in htab_percpu_map_lookup_and_delete_batch()
1902 htab_map_lookup_batch(struct bpf_map *map, const union bpf_attr *attr, in htab_map_lookup_batch() argument
1905 return __htab_map_lookup_and_delete_batch(map, attr, uattr, false, in htab_map_lookup_batch()
1910 htab_map_lookup_and_delete_batch(struct bpf_map *map, in htab_map_lookup_and_delete_batch() argument
1914 return __htab_map_lookup_and_delete_batch(map, attr, uattr, true, in htab_map_lookup_and_delete_batch()
1919 htab_lru_percpu_map_lookup_batch(struct bpf_map *map, in htab_lru_percpu_map_lookup_batch() argument
1923 return __htab_map_lookup_and_delete_batch(map, attr, uattr, false, in htab_lru_percpu_map_lookup_batch()
1928 htab_lru_percpu_map_lookup_and_delete_batch(struct bpf_map *map, in htab_lru_percpu_map_lookup_and_delete_batch() argument
1932 return __htab_map_lookup_and_delete_batch(map, attr, uattr, true, in htab_lru_percpu_map_lookup_and_delete_batch()
1937 htab_lru_map_lookup_batch(struct bpf_map *map, const union bpf_attr *attr, in htab_lru_map_lookup_batch() argument
1940 return __htab_map_lookup_and_delete_batch(map, attr, uattr, false, in htab_lru_map_lookup_batch()
1945 htab_lru_map_lookup_and_delete_batch(struct bpf_map *map, in htab_lru_map_lookup_and_delete_batch() argument
1949 return __htab_map_lookup_and_delete_batch(map, attr, uattr, true, in htab_lru_map_lookup_and_delete_batch()
1954 struct bpf_map *map; member
2045 struct bpf_map *map = info->map; in __bpf_hash_map_seq_show() local
2055 ctx.map = info->map; in __bpf_hash_map_seq_show()
2057 roundup_key_size = round_up(map->key_size, 8); in __bpf_hash_map_seq_show()
2062 roundup_value_size = round_up(map->value_size, 8); in __bpf_hash_map_seq_show()
2063 pptr = htab_elem_get_ptr(elem, map->key_size); in __bpf_hash_map_seq_show()
2096 struct bpf_map *map = aux->map; in bpf_iter_init_hash_map() local
2100 if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || in bpf_iter_init_hash_map()
2101 map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) { in bpf_iter_init_hash_map()
2102 buf_size = round_up(map->value_size, 8) * num_possible_cpus(); in bpf_iter_init_hash_map()
2110 bpf_map_inc_with_uref(map); in bpf_iter_init_hash_map()
2111 seq_info->map = map; in bpf_iter_init_hash_map()
2112 seq_info->htab = container_of(map, struct bpf_htab, map); in bpf_iter_init_hash_map()
2120 bpf_map_put_with_uref(seq_info->map); in bpf_iter_fini_hash_map()
2138 static int bpf_for_each_hash_elem(struct bpf_map *map, bpf_callback_t callback_fn, in bpf_for_each_hash_elem() argument
2141 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in bpf_for_each_hash_elem()
2158 roundup_key_size = round_up(map->key_size, 8); in bpf_for_each_hash_elem()
2172 pptr = htab_elem_get_ptr(elem, map->key_size); in bpf_for_each_hash_elem()
2178 ret = callback_fn((u64)(long)map, (u64)(long)key, in bpf_for_each_hash_elem()
2237 static void *htab_percpu_map_lookup_elem(struct bpf_map *map, void *key) in htab_percpu_map_lookup_elem() argument
2239 struct htab_elem *l = __htab_map_lookup_elem(map, key); in htab_percpu_map_lookup_elem()
2242 return this_cpu_ptr(htab_elem_get_ptr(l, map->key_size)); in htab_percpu_map_lookup_elem()
2247 static void *htab_percpu_map_lookup_percpu_elem(struct bpf_map *map, void *key, u32 cpu) in htab_percpu_map_lookup_percpu_elem() argument
2254 l = __htab_map_lookup_elem(map, key); in htab_percpu_map_lookup_percpu_elem()
2256 return per_cpu_ptr(htab_elem_get_ptr(l, map->key_size), cpu); in htab_percpu_map_lookup_percpu_elem()
2261 static void *htab_lru_percpu_map_lookup_elem(struct bpf_map *map, void *key) in htab_lru_percpu_map_lookup_elem() argument
2263 struct htab_elem *l = __htab_map_lookup_elem(map, key); in htab_lru_percpu_map_lookup_elem()
2267 return this_cpu_ptr(htab_elem_get_ptr(l, map->key_size)); in htab_lru_percpu_map_lookup_elem()
2273 static void *htab_lru_percpu_map_lookup_percpu_elem(struct bpf_map *map, void *key, u32 cpu) in htab_lru_percpu_map_lookup_percpu_elem() argument
2280 l = __htab_map_lookup_elem(map, key); in htab_lru_percpu_map_lookup_percpu_elem()
2283 return per_cpu_ptr(htab_elem_get_ptr(l, map->key_size), cpu); in htab_lru_percpu_map_lookup_percpu_elem()
2289 int bpf_percpu_hash_copy(struct bpf_map *map, void *key, void *value) in bpf_percpu_hash_copy() argument
2301 size = round_up(map->value_size, 8); in bpf_percpu_hash_copy()
2303 l = __htab_map_lookup_elem(map, key); in bpf_percpu_hash_copy()
2309 pptr = htab_elem_get_ptr(l, map->key_size); in bpf_percpu_hash_copy()
2321 int bpf_percpu_hash_update(struct bpf_map *map, void *key, void *value, in bpf_percpu_hash_update() argument
2324 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in bpf_percpu_hash_update()
2329 ret = __htab_lru_percpu_map_update_elem(map, key, value, in bpf_percpu_hash_update()
2332 ret = __htab_percpu_map_update_elem(map, key, value, map_flags, in bpf_percpu_hash_update()
2339 static void htab_percpu_map_seq_show_elem(struct bpf_map *map, void *key, in htab_percpu_map_seq_show_elem() argument
2348 l = __htab_map_lookup_elem(map, key); in htab_percpu_map_seq_show_elem()
2354 btf_type_seq_show(map->btf, map->btf_key_type_id, key, m); in htab_percpu_map_seq_show_elem()
2356 pptr = htab_elem_get_ptr(l, map->key_size); in htab_percpu_map_seq_show_elem()
2359 btf_type_seq_show(map->btf, map->btf_value_type_id, in htab_percpu_map_seq_show_elem()
2413 static void fd_htab_map_free(struct bpf_map *map) in fd_htab_map_free() argument
2415 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in fd_htab_map_free()
2425 void *ptr = fd_htab_map_get_ptr(map, l); in fd_htab_map_free()
2427 map->ops->map_fd_put_ptr(ptr); in fd_htab_map_free()
2431 htab_map_free(map); in fd_htab_map_free()
2435 int bpf_fd_htab_map_lookup_elem(struct bpf_map *map, void *key, u32 *value) in bpf_fd_htab_map_lookup_elem() argument
2440 if (!map->ops->map_fd_sys_lookup_elem) in bpf_fd_htab_map_lookup_elem()
2444 ptr = htab_map_lookup_elem(map, key); in bpf_fd_htab_map_lookup_elem()
2446 *value = map->ops->map_fd_sys_lookup_elem(READ_ONCE(*ptr)); in bpf_fd_htab_map_lookup_elem()
2455 int bpf_fd_htab_map_update_elem(struct bpf_map *map, struct file *map_file, in bpf_fd_htab_map_update_elem() argument
2462 ptr = map->ops->map_fd_get_ptr(map, map_file, ufd); in bpf_fd_htab_map_update_elem()
2466 ret = htab_map_update_elem(map, key, &ptr, map_flags); in bpf_fd_htab_map_update_elem()
2468 map->ops->map_fd_put_ptr(ptr); in bpf_fd_htab_map_update_elem()
2475 struct bpf_map *map, *inner_map_meta; in htab_of_map_alloc() local
2481 map = htab_map_alloc(attr); in htab_of_map_alloc()
2482 if (IS_ERR(map)) { in htab_of_map_alloc()
2484 return map; in htab_of_map_alloc()
2487 map->inner_map_meta = inner_map_meta; in htab_of_map_alloc()
2489 return map; in htab_of_map_alloc()
2492 static void *htab_of_map_lookup_elem(struct bpf_map *map, void *key) in htab_of_map_lookup_elem() argument
2494 struct bpf_map **inner_map = htab_map_lookup_elem(map, key); in htab_of_map_lookup_elem()
2502 static int htab_of_map_gen_lookup(struct bpf_map *map, in htab_of_map_gen_lookup() argument
2509 (void *(*)(struct bpf_map *map, void *key))NULL)); in htab_of_map_gen_lookup()
2514 round_up(map->key_size, 8)); in htab_of_map_gen_lookup()
2520 static void htab_of_map_free(struct bpf_map *map) in htab_of_map_free() argument
2522 bpf_map_meta_free(map->inner_map_meta); in htab_of_map_free()
2523 fd_htab_map_free(map); in htab_of_map_free()