Lines Matching refs:htab
116 struct bpf_htab *htab; member
130 static inline bool htab_is_prealloc(const struct bpf_htab *htab) in htab_is_prealloc() argument
132 return !(htab->map.map_flags & BPF_F_NO_PREALLOC); in htab_is_prealloc()
135 static inline bool htab_use_raw_lock(const struct bpf_htab *htab) in htab_use_raw_lock() argument
137 return (!IS_ENABLED(CONFIG_PREEMPT_RT) || htab_is_prealloc(htab)); in htab_use_raw_lock()
140 static void htab_init_buckets(struct bpf_htab *htab) in htab_init_buckets() argument
144 for (i = 0; i < htab->n_buckets; i++) { in htab_init_buckets()
145 INIT_HLIST_NULLS_HEAD(&htab->buckets[i].head, i); in htab_init_buckets()
146 if (htab_use_raw_lock(htab)) { in htab_init_buckets()
147 raw_spin_lock_init(&htab->buckets[i].raw_lock); in htab_init_buckets()
148 lockdep_set_class(&htab->buckets[i].raw_lock, in htab_init_buckets()
149 &htab->lockdep_key); in htab_init_buckets()
151 spin_lock_init(&htab->buckets[i].lock); in htab_init_buckets()
152 lockdep_set_class(&htab->buckets[i].lock, in htab_init_buckets()
153 &htab->lockdep_key); in htab_init_buckets()
159 static inline int htab_lock_bucket(const struct bpf_htab *htab, in htab_lock_bucket() argument
166 hash = hash & min_t(u32, HASHTAB_MAP_LOCK_MASK, htab->n_buckets - 1); in htab_lock_bucket()
168 use_raw_lock = htab_use_raw_lock(htab); in htab_lock_bucket()
173 if (unlikely(__this_cpu_inc_return(*(htab->map_locked[hash])) != 1)) { in htab_lock_bucket()
174 __this_cpu_dec(*(htab->map_locked[hash])); in htab_lock_bucket()
191 static inline void htab_unlock_bucket(const struct bpf_htab *htab, in htab_unlock_bucket() argument
195 bool use_raw_lock = htab_use_raw_lock(htab); in htab_unlock_bucket()
197 hash = hash & min_t(u32, HASHTAB_MAP_LOCK_MASK, htab->n_buckets - 1); in htab_unlock_bucket()
202 __this_cpu_dec(*(htab->map_locked[hash])); in htab_unlock_bucket()
211 static bool htab_is_lru(const struct bpf_htab *htab) in htab_is_lru() argument
213 return htab->map.map_type == BPF_MAP_TYPE_LRU_HASH || in htab_is_lru()
214 htab->map.map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH; in htab_is_lru()
217 static bool htab_is_percpu(const struct bpf_htab *htab) in htab_is_percpu() argument
219 return htab->map.map_type == BPF_MAP_TYPE_PERCPU_HASH || in htab_is_percpu()
220 htab->map.map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH; in htab_is_percpu()
239 static struct htab_elem *get_htab_elem(struct bpf_htab *htab, int i) in get_htab_elem() argument
241 return (struct htab_elem *) (htab->elems + i * (u64)htab->elem_size); in get_htab_elem()
244 static bool htab_has_extra_elems(struct bpf_htab *htab) in htab_has_extra_elems() argument
246 return !htab_is_percpu(htab) && !htab_is_lru(htab); in htab_has_extra_elems()
249 static void htab_free_prealloced_timers(struct bpf_htab *htab) in htab_free_prealloced_timers() argument
251 u32 num_entries = htab->map.max_entries; in htab_free_prealloced_timers()
254 if (likely(!map_value_has_timer(&htab->map))) in htab_free_prealloced_timers()
256 if (htab_has_extra_elems(htab)) in htab_free_prealloced_timers()
262 elem = get_htab_elem(htab, i); in htab_free_prealloced_timers()
264 round_up(htab->map.key_size, 8) + in htab_free_prealloced_timers()
265 htab->map.timer_off); in htab_free_prealloced_timers()
270 static void htab_free_elems(struct bpf_htab *htab) in htab_free_elems() argument
274 if (!htab_is_percpu(htab)) in htab_free_elems()
277 for (i = 0; i < htab->map.max_entries; i++) { in htab_free_elems()
280 pptr = htab_elem_get_ptr(get_htab_elem(htab, i), in htab_free_elems()
281 htab->map.key_size); in htab_free_elems()
286 bpf_map_area_free(htab->elems); in htab_free_elems()
300 static struct htab_elem *prealloc_lru_pop(struct bpf_htab *htab, void *key, in prealloc_lru_pop() argument
303 struct bpf_lru_node *node = bpf_lru_pop_free(&htab->lru, hash); in prealloc_lru_pop()
308 memcpy(l->key, key, htab->map.key_size); in prealloc_lru_pop()
315 static int prealloc_init(struct bpf_htab *htab) in prealloc_init() argument
317 u32 num_entries = htab->map.max_entries; in prealloc_init()
320 if (htab_has_extra_elems(htab)) in prealloc_init()
323 htab->elems = bpf_map_area_alloc((u64)htab->elem_size * num_entries, in prealloc_init()
324 htab->map.numa_node); in prealloc_init()
325 if (!htab->elems) in prealloc_init()
328 if (!htab_is_percpu(htab)) in prealloc_init()
332 u32 size = round_up(htab->map.value_size, 8); in prealloc_init()
335 pptr = bpf_map_alloc_percpu(&htab->map, size, 8, in prealloc_init()
339 htab_elem_set_ptr(get_htab_elem(htab, i), htab->map.key_size, in prealloc_init()
345 if (htab_is_lru(htab)) in prealloc_init()
346 err = bpf_lru_init(&htab->lru, in prealloc_init()
347 htab->map.map_flags & BPF_F_NO_COMMON_LRU, in prealloc_init()
351 htab); in prealloc_init()
353 err = pcpu_freelist_init(&htab->freelist); in prealloc_init()
358 if (htab_is_lru(htab)) in prealloc_init()
359 bpf_lru_populate(&htab->lru, htab->elems, in prealloc_init()
361 htab->elem_size, num_entries); in prealloc_init()
363 pcpu_freelist_populate(&htab->freelist, in prealloc_init()
364 htab->elems + offsetof(struct htab_elem, fnode), in prealloc_init()
365 htab->elem_size, num_entries); in prealloc_init()
370 htab_free_elems(htab); in prealloc_init()
374 static void prealloc_destroy(struct bpf_htab *htab) in prealloc_destroy() argument
376 htab_free_elems(htab); in prealloc_destroy()
378 if (htab_is_lru(htab)) in prealloc_destroy()
379 bpf_lru_destroy(&htab->lru); in prealloc_destroy()
381 pcpu_freelist_destroy(&htab->freelist); in prealloc_destroy()
384 static int alloc_extra_elems(struct bpf_htab *htab) in alloc_extra_elems() argument
390 pptr = bpf_map_alloc_percpu(&htab->map, sizeof(struct htab_elem *), 8, in alloc_extra_elems()
396 l = pcpu_freelist_pop(&htab->freelist); in alloc_extra_elems()
403 htab->extra_elems = pptr; in alloc_extra_elems()
424 BUILD_BUG_ON(offsetof(struct htab_elem, htab) != in htab_map_alloc_check()
484 struct bpf_htab *htab; in htab_map_alloc() local
487 htab = kzalloc(sizeof(*htab), GFP_USER | __GFP_ACCOUNT); in htab_map_alloc()
488 if (!htab) in htab_map_alloc()
491 lockdep_register_key(&htab->lockdep_key); in htab_map_alloc()
493 bpf_map_init_from_attr(&htab->map, attr); in htab_map_alloc()
500 htab->map.max_entries = roundup(attr->max_entries, in htab_map_alloc()
502 if (htab->map.max_entries < attr->max_entries) in htab_map_alloc()
503 htab->map.max_entries = rounddown(attr->max_entries, in htab_map_alloc()
508 htab->n_buckets = roundup_pow_of_two(htab->map.max_entries); in htab_map_alloc()
510 htab->elem_size = sizeof(struct htab_elem) + in htab_map_alloc()
511 round_up(htab->map.key_size, 8); in htab_map_alloc()
513 htab->elem_size += sizeof(void *); in htab_map_alloc()
515 htab->elem_size += round_up(htab->map.value_size, 8); in htab_map_alloc()
519 if (htab->n_buckets == 0 || in htab_map_alloc()
520 htab->n_buckets > U32_MAX / sizeof(struct bucket)) in htab_map_alloc()
524 htab->buckets = bpf_map_area_alloc(htab->n_buckets * in htab_map_alloc()
526 htab->map.numa_node); in htab_map_alloc()
527 if (!htab->buckets) in htab_map_alloc()
531 htab->map_locked[i] = bpf_map_alloc_percpu(&htab->map, in htab_map_alloc()
535 if (!htab->map_locked[i]) in htab_map_alloc()
539 if (htab->map.map_flags & BPF_F_ZERO_SEED) in htab_map_alloc()
540 htab->hashrnd = 0; in htab_map_alloc()
542 htab->hashrnd = get_random_int(); in htab_map_alloc()
544 htab_init_buckets(htab); in htab_map_alloc()
547 err = prealloc_init(htab); in htab_map_alloc()
555 err = alloc_extra_elems(htab); in htab_map_alloc()
561 return &htab->map; in htab_map_alloc()
564 prealloc_destroy(htab); in htab_map_alloc()
567 free_percpu(htab->map_locked[i]); in htab_map_alloc()
568 bpf_map_area_free(htab->buckets); in htab_map_alloc()
570 lockdep_unregister_key(&htab->lockdep_key); in htab_map_alloc()
571 kfree(htab); in htab_map_alloc()
580 static inline struct bucket *__select_bucket(struct bpf_htab *htab, u32 hash) in __select_bucket() argument
582 return &htab->buckets[hash & (htab->n_buckets - 1)]; in __select_bucket()
585 static inline struct hlist_nulls_head *select_bucket(struct bpf_htab *htab, u32 hash) in select_bucket() argument
587 return &__select_bucket(htab, hash)->head; in select_bucket()
633 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in __htab_map_lookup_elem() local
643 hash = htab_map_hash(key, key_size, htab->hashrnd); in __htab_map_lookup_elem()
645 head = select_bucket(htab, hash); in __htab_map_lookup_elem()
647 l = lookup_nulls_elem_raw(head, hash, key, key_size, htab->n_buckets); in __htab_map_lookup_elem()
737 static void check_and_free_timer(struct bpf_htab *htab, struct htab_elem *elem) in check_and_free_timer() argument
739 if (unlikely(map_value_has_timer(&htab->map))) in check_and_free_timer()
741 round_up(htab->map.key_size, 8) + in check_and_free_timer()
742 htab->map.timer_off); in check_and_free_timer()
750 struct bpf_htab *htab = (struct bpf_htab *)arg; in htab_lru_map_delete_node() local
759 b = __select_bucket(htab, tgt_l->hash); in htab_lru_map_delete_node()
762 ret = htab_lock_bucket(htab, b, tgt_l->hash, &flags); in htab_lru_map_delete_node()
769 check_and_free_timer(htab, l); in htab_lru_map_delete_node()
773 htab_unlock_bucket(htab, b, tgt_l->hash, flags); in htab_lru_map_delete_node()
781 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in htab_map_get_next_key() local
794 hash = htab_map_hash(key, key_size, htab->hashrnd); in htab_map_get_next_key()
796 head = select_bucket(htab, hash); in htab_map_get_next_key()
799 l = lookup_nulls_elem_raw(head, hash, key, key_size, htab->n_buckets); in htab_map_get_next_key()
815 i = hash & (htab->n_buckets - 1); in htab_map_get_next_key()
820 for (; i < htab->n_buckets; i++) { in htab_map_get_next_key()
821 head = select_bucket(htab, i); in htab_map_get_next_key()
837 static void htab_elem_free(struct bpf_htab *htab, struct htab_elem *l) in htab_elem_free() argument
839 if (htab->map.map_type == BPF_MAP_TYPE_PERCPU_HASH) in htab_elem_free()
840 free_percpu(htab_elem_get_ptr(l, htab->map.key_size)); in htab_elem_free()
841 check_and_free_timer(htab, l); in htab_elem_free()
848 struct bpf_htab *htab = l->htab; in htab_elem_free_rcu() local
850 htab_elem_free(htab, l); in htab_elem_free_rcu()
853 static void htab_put_fd_value(struct bpf_htab *htab, struct htab_elem *l) in htab_put_fd_value() argument
855 struct bpf_map *map = &htab->map; in htab_put_fd_value()
864 static void free_htab_elem(struct bpf_htab *htab, struct htab_elem *l) in free_htab_elem() argument
866 htab_put_fd_value(htab, l); in free_htab_elem()
868 if (htab_is_prealloc(htab)) { in free_htab_elem()
869 check_and_free_timer(htab, l); in free_htab_elem()
870 __pcpu_freelist_push(&htab->freelist, &l->fnode); in free_htab_elem()
872 atomic_dec(&htab->count); in free_htab_elem()
873 l->htab = htab; in free_htab_elem()
878 static void pcpu_copy_value(struct bpf_htab *htab, void __percpu *pptr, in pcpu_copy_value() argument
883 memcpy(this_cpu_ptr(pptr), value, htab->map.value_size); in pcpu_copy_value()
885 u32 size = round_up(htab->map.value_size, 8); in pcpu_copy_value()
896 static void pcpu_init_value(struct bpf_htab *htab, void __percpu *pptr, in pcpu_init_value() argument
905 if (htab_is_prealloc(htab) && !onallcpus) { in pcpu_init_value()
906 u32 size = round_up(htab->map.value_size, 8); in pcpu_init_value()
918 pcpu_copy_value(htab, pptr, value, onallcpus); in pcpu_init_value()
922 static bool fd_htab_map_needs_adjust(const struct bpf_htab *htab) in fd_htab_map_needs_adjust() argument
924 return htab->map.map_type == BPF_MAP_TYPE_HASH_OF_MAPS && in fd_htab_map_needs_adjust()
928 static struct htab_elem *alloc_htab_elem(struct bpf_htab *htab, void *key, in alloc_htab_elem() argument
933 u32 size = htab->map.value_size; in alloc_htab_elem()
934 bool prealloc = htab_is_prealloc(htab); in alloc_htab_elem()
943 pl_new = this_cpu_ptr(htab->extra_elems); in alloc_htab_elem()
945 htab_put_fd_value(htab, old_elem); in alloc_htab_elem()
950 l = __pcpu_freelist_pop(&htab->freelist); in alloc_htab_elem()
956 if (atomic_inc_return(&htab->count) > htab->map.max_entries) in alloc_htab_elem()
966 l_new = bpf_map_kmalloc_node(&htab->map, htab->elem_size, in alloc_htab_elem()
968 htab->map.numa_node); in alloc_htab_elem()
973 check_and_init_map_value(&htab->map, in alloc_htab_elem()
984 pptr = bpf_map_alloc_percpu(&htab->map, size, 8, in alloc_htab_elem()
993 pcpu_init_value(htab, pptr, value, onallcpus); in alloc_htab_elem()
997 } else if (fd_htab_map_needs_adjust(htab)) { in alloc_htab_elem()
1001 copy_map_value(&htab->map, in alloc_htab_elem()
1009 atomic_dec(&htab->count); in alloc_htab_elem()
1013 static int check_flags(struct bpf_htab *htab, struct htab_elem *l_old, in check_flags() argument
1031 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in htab_map_update_elem() local
1048 hash = htab_map_hash(key, key_size, htab->hashrnd); in htab_map_update_elem()
1050 b = __select_bucket(htab, hash); in htab_map_update_elem()
1058 htab->n_buckets); in htab_map_update_elem()
1059 ret = check_flags(htab, l_old, map_flags); in htab_map_update_elem()
1075 ret = htab_lock_bucket(htab, b, hash, &flags); in htab_map_update_elem()
1081 ret = check_flags(htab, l_old, map_flags); in htab_map_update_elem()
1099 l_new = alloc_htab_elem(htab, key, value, key_size, hash, false, false, in htab_map_update_elem()
1113 if (!htab_is_prealloc(htab)) in htab_map_update_elem()
1114 free_htab_elem(htab, l_old); in htab_map_update_elem()
1116 check_and_free_timer(htab, l_old); in htab_map_update_elem()
1120 htab_unlock_bucket(htab, b, hash, flags); in htab_map_update_elem()
1124 static void htab_lru_push_free(struct bpf_htab *htab, struct htab_elem *elem) in htab_lru_push_free() argument
1126 check_and_free_timer(htab, elem); in htab_lru_push_free()
1127 bpf_lru_push_free(&htab->lru, &elem->lru_node); in htab_lru_push_free()
1133 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in htab_lru_map_update_elem() local
1150 hash = htab_map_hash(key, key_size, htab->hashrnd); in htab_lru_map_update_elem()
1152 b = __select_bucket(htab, hash); in htab_lru_map_update_elem()
1160 l_new = prealloc_lru_pop(htab, key, hash); in htab_lru_map_update_elem()
1163 copy_map_value(&htab->map, in htab_lru_map_update_elem()
1166 ret = htab_lock_bucket(htab, b, hash, &flags); in htab_lru_map_update_elem()
1172 ret = check_flags(htab, l_old, map_flags); in htab_lru_map_update_elem()
1187 htab_unlock_bucket(htab, b, hash, flags); in htab_lru_map_update_elem()
1191 htab_lru_push_free(htab, l_new); in htab_lru_map_update_elem()
1193 htab_lru_push_free(htab, l_old); in htab_lru_map_update_elem()
1202 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in __htab_percpu_map_update_elem() local
1219 hash = htab_map_hash(key, key_size, htab->hashrnd); in __htab_percpu_map_update_elem()
1221 b = __select_bucket(htab, hash); in __htab_percpu_map_update_elem()
1224 ret = htab_lock_bucket(htab, b, hash, &flags); in __htab_percpu_map_update_elem()
1230 ret = check_flags(htab, l_old, map_flags); in __htab_percpu_map_update_elem()
1236 pcpu_copy_value(htab, htab_elem_get_ptr(l_old, key_size), in __htab_percpu_map_update_elem()
1239 l_new = alloc_htab_elem(htab, key, value, key_size, in __htab_percpu_map_update_elem()
1249 htab_unlock_bucket(htab, b, hash, flags); in __htab_percpu_map_update_elem()
1257 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in __htab_lru_percpu_map_update_elem() local
1274 hash = htab_map_hash(key, key_size, htab->hashrnd); in __htab_lru_percpu_map_update_elem()
1276 b = __select_bucket(htab, hash); in __htab_lru_percpu_map_update_elem()
1285 l_new = prealloc_lru_pop(htab, key, hash); in __htab_lru_percpu_map_update_elem()
1290 ret = htab_lock_bucket(htab, b, hash, &flags); in __htab_lru_percpu_map_update_elem()
1296 ret = check_flags(htab, l_old, map_flags); in __htab_lru_percpu_map_update_elem()
1304 pcpu_copy_value(htab, htab_elem_get_ptr(l_old, key_size), in __htab_lru_percpu_map_update_elem()
1307 pcpu_init_value(htab, htab_elem_get_ptr(l_new, key_size), in __htab_lru_percpu_map_update_elem()
1314 htab_unlock_bucket(htab, b, hash, flags); in __htab_lru_percpu_map_update_elem()
1317 bpf_lru_push_free(&htab->lru, &l_new->lru_node); in __htab_lru_percpu_map_update_elem()
1337 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in htab_map_delete_elem() local
1350 hash = htab_map_hash(key, key_size, htab->hashrnd); in htab_map_delete_elem()
1351 b = __select_bucket(htab, hash); in htab_map_delete_elem()
1354 ret = htab_lock_bucket(htab, b, hash, &flags); in htab_map_delete_elem()
1362 free_htab_elem(htab, l); in htab_map_delete_elem()
1367 htab_unlock_bucket(htab, b, hash, flags); in htab_map_delete_elem()
1373 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in htab_lru_map_delete_elem() local
1386 hash = htab_map_hash(key, key_size, htab->hashrnd); in htab_lru_map_delete_elem()
1387 b = __select_bucket(htab, hash); in htab_lru_map_delete_elem()
1390 ret = htab_lock_bucket(htab, b, hash, &flags); in htab_lru_map_delete_elem()
1401 htab_unlock_bucket(htab, b, hash, flags); in htab_lru_map_delete_elem()
1403 htab_lru_push_free(htab, l); in htab_lru_map_delete_elem()
1407 static void delete_all_elements(struct bpf_htab *htab) in delete_all_elements() argument
1411 for (i = 0; i < htab->n_buckets; i++) { in delete_all_elements()
1412 struct hlist_nulls_head *head = select_bucket(htab, i); in delete_all_elements()
1418 htab_elem_free(htab, l); in delete_all_elements()
1423 static void htab_free_malloced_timers(struct bpf_htab *htab) in htab_free_malloced_timers() argument
1428 for (i = 0; i < htab->n_buckets; i++) { in htab_free_malloced_timers()
1429 struct hlist_nulls_head *head = select_bucket(htab, i); in htab_free_malloced_timers()
1434 check_and_free_timer(htab, l); in htab_free_malloced_timers()
1442 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in htab_map_free_timers() local
1444 if (likely(!map_value_has_timer(&htab->map))) in htab_map_free_timers()
1446 if (!htab_is_prealloc(htab)) in htab_map_free_timers()
1447 htab_free_malloced_timers(htab); in htab_map_free_timers()
1449 htab_free_prealloced_timers(htab); in htab_map_free_timers()
1455 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in htab_map_free() local
1467 if (!htab_is_prealloc(htab)) in htab_map_free()
1468 delete_all_elements(htab); in htab_map_free()
1470 prealloc_destroy(htab); in htab_map_free()
1472 free_percpu(htab->extra_elems); in htab_map_free()
1473 bpf_map_area_free(htab->buckets); in htab_map_free()
1475 free_percpu(htab->map_locked[i]); in htab_map_free()
1476 lockdep_unregister_key(&htab->lockdep_key); in htab_map_free()
1477 kfree(htab); in htab_map_free()
1505 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in __htab_map_lookup_and_delete_elem() local
1515 hash = htab_map_hash(key, key_size, htab->hashrnd); in __htab_map_lookup_and_delete_elem()
1516 b = __select_bucket(htab, hash); in __htab_map_lookup_and_delete_elem()
1519 ret = htab_lock_bucket(htab, b, hash, &bflags); in __htab_map_lookup_and_delete_elem()
1554 free_htab_elem(htab, l); in __htab_map_lookup_and_delete_elem()
1557 htab_unlock_bucket(htab, b, hash, bflags); in __htab_map_lookup_and_delete_elem()
1560 htab_lru_push_free(htab, l); in __htab_map_lookup_and_delete_elem()
1602 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in __htab_map_lookup_and_delete_batch() local
1639 if (batch >= htab->n_buckets) in __htab_map_lookup_and_delete_batch()
1642 key_size = htab->map.key_size; in __htab_map_lookup_and_delete_batch()
1643 roundup_key_size = round_up(htab->map.key_size, 8); in __htab_map_lookup_and_delete_batch()
1644 value_size = htab->map.value_size; in __htab_map_lookup_and_delete_batch()
1671 b = &htab->buckets[batch]; in __htab_map_lookup_and_delete_batch()
1675 ret = htab_lock_bucket(htab, b, batch, &flags); in __htab_map_lookup_and_delete_batch()
1698 htab_unlock_bucket(htab, b, batch, flags); in __htab_map_lookup_and_delete_batch()
1709 htab_unlock_bucket(htab, b, batch, flags); in __htab_map_lookup_and_delete_batch()
1755 free_htab_elem(htab, l); in __htab_map_lookup_and_delete_batch()
1762 htab_unlock_bucket(htab, b, batch, flags); in __htab_map_lookup_and_delete_batch()
1768 htab_lru_push_free(htab, l); in __htab_map_lookup_and_delete_batch()
1775 if (!bucket_cnt && (batch + 1 < htab->n_buckets)) { in __htab_map_lookup_and_delete_batch()
1792 if (batch >= htab->n_buckets) { in __htab_map_lookup_and_delete_batch()
1885 struct bpf_htab *htab; member
1895 const struct bpf_htab *htab = info->htab; in bpf_hash_map_seq_find_next() local
1904 if (bucket_id >= htab->n_buckets) in bpf_hash_map_seq_find_next()
1918 b = &htab->buckets[bucket_id++]; in bpf_hash_map_seq_find_next()
1923 for (i = bucket_id; i < htab->n_buckets; i++) { in bpf_hash_map_seq_find_next()
1924 b = &htab->buckets[i]; in bpf_hash_map_seq_find_next()
2042 seq_info->htab = container_of(map, struct bpf_htab, map); in bpf_iter_init_hash_map()
2071 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in bpf_for_each_hash_elem() local
2086 is_percpu = htab_is_percpu(htab); in bpf_for_each_hash_elem()
2094 for (i = 0; i < htab->n_buckets; i++) { in bpf_for_each_hash_elem()
2095 b = &htab->buckets[i]; in bpf_for_each_hash_elem()
2141 BATCH_OPS(htab),
2228 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in bpf_percpu_hash_update() local
2232 if (htab_is_lru(htab)) in bpf_percpu_hash_update()
2321 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in fd_htab_map_free() local
2327 for (i = 0; i < htab->n_buckets; i++) { in fd_htab_map_free()
2328 head = select_bucket(htab, i); in fd_htab_map_free()