Lines Matching refs:map
26 struct bpf_map map; member
64 return htab->map.map_type == BPF_MAP_TYPE_LRU_HASH || in htab_is_lru()
65 htab->map.map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH; in htab_is_lru()
70 return htab->map.map_type == BPF_MAP_TYPE_PERCPU_HASH || in htab_is_percpu()
71 htab->map.map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH; in htab_is_percpu()
76 return !(htab->map.map_flags & BPF_F_NO_PREALLOC); in htab_is_prealloc()
90 static void *fd_htab_map_get_ptr(const struct bpf_map *map, struct htab_elem *l) in fd_htab_map_get_ptr() argument
92 return *(void **)(l->key + roundup(map->key_size, 8)); in fd_htab_map_get_ptr()
107 for (i = 0; i < htab->map.max_entries; i++) { in htab_free_elems()
111 htab->map.key_size); in htab_free_elems()
127 memcpy(l->key, key, htab->map.key_size); in prealloc_lru_pop()
136 u32 num_entries = htab->map.max_entries; in prealloc_init()
143 htab->map.numa_node); in prealloc_init()
151 u32 size = round_up(htab->map.value_size, 8); in prealloc_init()
157 htab_elem_set_ptr(get_htab_elem(htab, i), htab->map.key_size, in prealloc_init()
165 htab->map.map_flags & BPF_F_NO_COMMON_LRU, in prealloc_init()
316 bpf_map_init_from_attr(&htab->map, attr); in htab_map_alloc()
323 htab->map.max_entries = roundup(attr->max_entries, in htab_map_alloc()
325 if (htab->map.max_entries < attr->max_entries) in htab_map_alloc()
326 htab->map.max_entries = rounddown(attr->max_entries, in htab_map_alloc()
331 htab->n_buckets = roundup_pow_of_two(htab->map.max_entries); in htab_map_alloc()
334 round_up(htab->map.key_size, 8); in htab_map_alloc()
338 htab->elem_size += round_up(htab->map.value_size, 8); in htab_map_alloc()
347 (u64) htab->elem_size * htab->map.max_entries; in htab_map_alloc()
350 cost += (u64) round_up(htab->map.value_size, 8) * in htab_map_alloc()
351 num_possible_cpus() * htab->map.max_entries; in htab_map_alloc()
356 err = bpf_map_charge_init(&htab->map.memory, cost); in htab_map_alloc()
363 htab->map.numa_node); in htab_map_alloc()
367 if (htab->map.map_flags & BPF_F_ZERO_SEED) in htab_map_alloc()
392 return &htab->map; in htab_map_alloc()
399 bpf_map_charge_finish(&htab->map.memory); in htab_map_alloc()
461 static void *__htab_map_lookup_elem(struct bpf_map *map, void *key) in __htab_map_lookup_elem() argument
463 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in __htab_map_lookup_elem()
471 key_size = map->key_size; in __htab_map_lookup_elem()
482 static void *htab_map_lookup_elem(struct bpf_map *map, void *key) in htab_map_lookup_elem() argument
484 struct htab_elem *l = __htab_map_lookup_elem(map, key); in htab_map_lookup_elem()
487 return l->key + round_up(map->key_size, 8); in htab_map_lookup_elem()
503 static u32 htab_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf) in htab_map_gen_lookup() argument
509 (void *(*)(struct bpf_map *map, void *key))NULL)); in htab_map_gen_lookup()
514 round_up(map->key_size, 8)); in htab_map_gen_lookup()
518 static __always_inline void *__htab_lru_map_lookup_elem(struct bpf_map *map, in __htab_lru_map_lookup_elem() argument
521 struct htab_elem *l = __htab_map_lookup_elem(map, key); in __htab_lru_map_lookup_elem()
526 return l->key + round_up(map->key_size, 8); in __htab_lru_map_lookup_elem()
532 static void *htab_lru_map_lookup_elem(struct bpf_map *map, void *key) in htab_lru_map_lookup_elem() argument
534 return __htab_lru_map_lookup_elem(map, key, true); in htab_lru_map_lookup_elem()
537 static void *htab_lru_map_lookup_elem_sys(struct bpf_map *map, void *key) in htab_lru_map_lookup_elem_sys() argument
539 return __htab_lru_map_lookup_elem(map, key, false); in htab_lru_map_lookup_elem_sys()
542 static u32 htab_lru_map_gen_lookup(struct bpf_map *map, in htab_lru_map_gen_lookup() argument
550 (void *(*)(struct bpf_map *map, void *key))NULL)); in htab_lru_map_gen_lookup()
563 round_up(map->key_size, 8)); in htab_lru_map_gen_lookup()
597 static int htab_map_get_next_key(struct bpf_map *map, void *key, void *next_key) in htab_map_get_next_key() argument
599 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in htab_map_get_next_key()
607 key_size = map->key_size; in htab_map_get_next_key()
657 if (htab->map.map_type == BPF_MAP_TYPE_PERCPU_HASH) in htab_elem_free()
658 free_percpu(htab_elem_get_ptr(l, htab->map.key_size)); in htab_elem_free()
680 struct bpf_map *map = &htab->map; in free_htab_elem() local
682 if (map->ops->map_fd_put_ptr) { in free_htab_elem()
683 void *ptr = fd_htab_map_get_ptr(map, l); in free_htab_elem()
685 map->ops->map_fd_put_ptr(ptr); in free_htab_elem()
702 memcpy(this_cpu_ptr(pptr), value, htab->map.value_size); in pcpu_copy_value()
704 u32 size = round_up(htab->map.value_size, 8); in pcpu_copy_value()
717 return htab->map.map_type == BPF_MAP_TYPE_HASH_OF_MAPS && in fd_htab_map_needs_adjust()
726 u32 size = htab->map.value_size; in alloc_htab_elem()
748 if (atomic_inc_return(&htab->count) > htab->map.max_entries) in alloc_htab_elem()
759 htab->map.numa_node); in alloc_htab_elem()
764 check_and_init_map_lock(&htab->map, in alloc_htab_elem()
792 copy_map_value(&htab->map, in alloc_htab_elem()
819 static int htab_map_update_elem(struct bpf_map *map, void *key, void *value, in htab_map_update_elem() argument
822 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in htab_map_update_elem()
836 key_size = map->key_size; in htab_map_update_elem()
844 if (unlikely(!map_value_has_spin_lock(map))) in htab_map_update_elem()
854 copy_map_value_locked(map, in htab_map_update_elem()
881 copy_map_value_locked(map, in htab_map_update_elem()
911 static int htab_lru_map_update_elem(struct bpf_map *map, void *key, void *value, in htab_lru_map_update_elem() argument
914 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in htab_lru_map_update_elem()
928 key_size = map->key_size; in htab_lru_map_update_elem()
943 memcpy(l_new->key + round_up(map->key_size, 8), value, map->value_size); in htab_lru_map_update_elem()
975 static int __htab_percpu_map_update_elem(struct bpf_map *map, void *key, in __htab_percpu_map_update_elem() argument
979 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in __htab_percpu_map_update_elem()
993 key_size = map->key_size; in __htab_percpu_map_update_elem()
1028 static int __htab_lru_percpu_map_update_elem(struct bpf_map *map, void *key, in __htab_lru_percpu_map_update_elem() argument
1032 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in __htab_lru_percpu_map_update_elem()
1046 key_size = map->key_size; in __htab_lru_percpu_map_update_elem()
1093 static int htab_percpu_map_update_elem(struct bpf_map *map, void *key, in htab_percpu_map_update_elem() argument
1096 return __htab_percpu_map_update_elem(map, key, value, map_flags, false); in htab_percpu_map_update_elem()
1099 static int htab_lru_percpu_map_update_elem(struct bpf_map *map, void *key, in htab_lru_percpu_map_update_elem() argument
1102 return __htab_lru_percpu_map_update_elem(map, key, value, map_flags, in htab_lru_percpu_map_update_elem()
1107 static int htab_map_delete_elem(struct bpf_map *map, void *key) in htab_map_delete_elem() argument
1109 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in htab_map_delete_elem()
1119 key_size = map->key_size; in htab_map_delete_elem()
1139 static int htab_lru_map_delete_elem(struct bpf_map *map, void *key) in htab_lru_map_delete_elem() argument
1141 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in htab_lru_map_delete_elem()
1151 key_size = map->key_size; in htab_lru_map_delete_elem()
1189 static void htab_map_free(struct bpf_map *map) in htab_map_free() argument
1191 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in htab_map_free()
1214 static void htab_map_seq_show_elem(struct bpf_map *map, void *key, in htab_map_seq_show_elem() argument
1221 value = htab_map_lookup_elem(map, key); in htab_map_seq_show_elem()
1227 btf_type_seq_show(map->btf, map->btf_key_type_id, key, m); in htab_map_seq_show_elem()
1229 btf_type_seq_show(map->btf, map->btf_value_type_id, value, m); in htab_map_seq_show_elem()
1261 static void *htab_percpu_map_lookup_elem(struct bpf_map *map, void *key) in htab_percpu_map_lookup_elem() argument
1263 struct htab_elem *l = __htab_map_lookup_elem(map, key); in htab_percpu_map_lookup_elem()
1266 return this_cpu_ptr(htab_elem_get_ptr(l, map->key_size)); in htab_percpu_map_lookup_elem()
1271 static void *htab_lru_percpu_map_lookup_elem(struct bpf_map *map, void *key) in htab_lru_percpu_map_lookup_elem() argument
1273 struct htab_elem *l = __htab_map_lookup_elem(map, key); in htab_lru_percpu_map_lookup_elem()
1277 return this_cpu_ptr(htab_elem_get_ptr(l, map->key_size)); in htab_lru_percpu_map_lookup_elem()
1283 int bpf_percpu_hash_copy(struct bpf_map *map, void *key, void *value) in bpf_percpu_hash_copy() argument
1295 size = round_up(map->value_size, 8); in bpf_percpu_hash_copy()
1297 l = __htab_map_lookup_elem(map, key); in bpf_percpu_hash_copy()
1303 pptr = htab_elem_get_ptr(l, map->key_size); in bpf_percpu_hash_copy()
1315 int bpf_percpu_hash_update(struct bpf_map *map, void *key, void *value, in bpf_percpu_hash_update() argument
1318 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in bpf_percpu_hash_update()
1323 ret = __htab_lru_percpu_map_update_elem(map, key, value, in bpf_percpu_hash_update()
1326 ret = __htab_percpu_map_update_elem(map, key, value, map_flags, in bpf_percpu_hash_update()
1333 static void htab_percpu_map_seq_show_elem(struct bpf_map *map, void *key, in htab_percpu_map_seq_show_elem() argument
1342 l = __htab_map_lookup_elem(map, key); in htab_percpu_map_seq_show_elem()
1348 btf_type_seq_show(map->btf, map->btf_key_type_id, key, m); in htab_percpu_map_seq_show_elem()
1350 pptr = htab_elem_get_ptr(l, map->key_size); in htab_percpu_map_seq_show_elem()
1353 btf_type_seq_show(map->btf, map->btf_value_type_id, in htab_percpu_map_seq_show_elem()
1391 static void fd_htab_map_free(struct bpf_map *map) in fd_htab_map_free() argument
1393 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in fd_htab_map_free()
1403 void *ptr = fd_htab_map_get_ptr(map, l); in fd_htab_map_free()
1405 map->ops->map_fd_put_ptr(ptr); in fd_htab_map_free()
1409 htab_map_free(map); in fd_htab_map_free()
1413 int bpf_fd_htab_map_lookup_elem(struct bpf_map *map, void *key, u32 *value) in bpf_fd_htab_map_lookup_elem() argument
1418 if (!map->ops->map_fd_sys_lookup_elem) in bpf_fd_htab_map_lookup_elem()
1422 ptr = htab_map_lookup_elem(map, key); in bpf_fd_htab_map_lookup_elem()
1424 *value = map->ops->map_fd_sys_lookup_elem(READ_ONCE(*ptr)); in bpf_fd_htab_map_lookup_elem()
1433 int bpf_fd_htab_map_update_elem(struct bpf_map *map, struct file *map_file, in bpf_fd_htab_map_update_elem() argument
1440 ptr = map->ops->map_fd_get_ptr(map, map_file, ufd); in bpf_fd_htab_map_update_elem()
1444 ret = htab_map_update_elem(map, key, &ptr, map_flags); in bpf_fd_htab_map_update_elem()
1446 map->ops->map_fd_put_ptr(ptr); in bpf_fd_htab_map_update_elem()
1453 struct bpf_map *map, *inner_map_meta; in htab_of_map_alloc() local
1459 map = htab_map_alloc(attr); in htab_of_map_alloc()
1460 if (IS_ERR(map)) { in htab_of_map_alloc()
1462 return map; in htab_of_map_alloc()
1465 map->inner_map_meta = inner_map_meta; in htab_of_map_alloc()
1467 return map; in htab_of_map_alloc()
1470 static void *htab_of_map_lookup_elem(struct bpf_map *map, void *key) in htab_of_map_lookup_elem() argument
1472 struct bpf_map **inner_map = htab_map_lookup_elem(map, key); in htab_of_map_lookup_elem()
1480 static u32 htab_of_map_gen_lookup(struct bpf_map *map, in htab_of_map_gen_lookup() argument
1487 (void *(*)(struct bpf_map *map, void *key))NULL)); in htab_of_map_gen_lookup()
1492 round_up(map->key_size, 8)); in htab_of_map_gen_lookup()
1498 static void htab_of_map_free(struct bpf_map *map) in htab_of_map_free() argument
1500 bpf_map_meta_free(map->inner_map_meta); in htab_of_map_free()
1501 fd_htab_map_free(map); in htab_of_map_free()