Lines Matching refs:hash
127 u32 hash; member
150 struct bucket *b, u32 hash, in htab_lock_bucket() argument
155 hash = hash & min_t(u32, HASHTAB_MAP_LOCK_MASK, htab->n_buckets - 1); in htab_lock_bucket()
159 if (unlikely(__this_cpu_inc_return(*(htab->map_locked[hash])) != 1)) { in htab_lock_bucket()
160 __this_cpu_dec(*(htab->map_locked[hash])); in htab_lock_bucket()
173 struct bucket *b, u32 hash, in htab_unlock_bucket() argument
176 hash = hash & min_t(u32, HASHTAB_MAP_LOCK_MASK, htab->n_buckets - 1); in htab_unlock_bucket()
178 __this_cpu_dec(*(htab->map_locked[hash])); in htab_unlock_bucket()
294 u32 hash) in prealloc_lru_pop() argument
296 struct bpf_lru_node *node = bpf_lru_pop_free(&htab->lru, hash); in prealloc_lru_pop()
341 offsetof(struct htab_elem, hash) - in prealloc_init()
608 static inline struct bucket *__select_bucket(struct bpf_htab *htab, u32 hash) in __select_bucket() argument
610 return &htab->buckets[hash & (htab->n_buckets - 1)]; in __select_bucket()
613 static inline struct hlist_nulls_head *select_bucket(struct bpf_htab *htab, u32 hash) in select_bucket() argument
615 return &__select_bucket(htab, hash)->head; in select_bucket()
619 static struct htab_elem *lookup_elem_raw(struct hlist_nulls_head *head, u32 hash, in lookup_elem_raw() argument
626 if (l->hash == hash && !memcmp(&l->key, key, key_size)) in lookup_elem_raw()
637 u32 hash, void *key, in lookup_nulls_elem_raw() argument
645 if (l->hash == hash && !memcmp(&l->key, key, key_size)) in lookup_nulls_elem_raw()
648 if (unlikely(get_nulls_value(n) != (hash & (n_buckets - 1)))) in lookup_nulls_elem_raw()
664 u32 hash, key_size; in __htab_map_lookup_elem() local
671 hash = htab_map_hash(key, key_size, htab->hashrnd); in __htab_map_lookup_elem()
673 head = select_bucket(htab, hash); in __htab_map_lookup_elem()
675 l = lookup_nulls_elem_raw(head, hash, key, key_size, htab->n_buckets); in __htab_map_lookup_elem()
790 b = __select_bucket(htab, tgt_l->hash); in htab_lru_map_delete_node()
793 ret = htab_lock_bucket(htab, b, tgt_l->hash, &flags); in htab_lru_map_delete_node()
804 htab_unlock_bucket(htab, b, tgt_l->hash, flags); in htab_lru_map_delete_node()
815 u32 hash, key_size; in htab_map_get_next_key() local
825 hash = htab_map_hash(key, key_size, htab->hashrnd); in htab_map_get_next_key()
827 head = select_bucket(htab, hash); in htab_map_get_next_key()
830 l = lookup_nulls_elem_raw(head, hash, key, key_size, htab->n_buckets); in htab_map_get_next_key()
846 i = hash & (htab->n_buckets - 1); in htab_map_get_next_key()
975 void *value, u32 key_size, u32 hash, in alloc_htab_elem() argument
1047 l_new->hash = hash; in alloc_htab_elem()
1077 u32 key_size, hash; in htab_map_update_elem() local
1089 hash = htab_map_hash(key, key_size, htab->hashrnd); in htab_map_update_elem()
1091 b = __select_bucket(htab, hash); in htab_map_update_elem()
1098 l_old = lookup_nulls_elem_raw(head, hash, key, key_size, in htab_map_update_elem()
1116 ret = htab_lock_bucket(htab, b, hash, &flags); in htab_map_update_elem()
1120 l_old = lookup_elem_raw(head, hash, key, key_size); in htab_map_update_elem()
1140 l_new = alloc_htab_elem(htab, key, value, key_size, hash, false, false, in htab_map_update_elem()
1161 htab_unlock_bucket(htab, b, hash, flags); in htab_map_update_elem()
1179 u32 key_size, hash; in htab_lru_map_update_elem() local
1191 hash = htab_map_hash(key, key_size, htab->hashrnd); in htab_lru_map_update_elem()
1193 b = __select_bucket(htab, hash); in htab_lru_map_update_elem()
1201 l_new = prealloc_lru_pop(htab, key, hash); in htab_lru_map_update_elem()
1207 ret = htab_lock_bucket(htab, b, hash, &flags); in htab_lru_map_update_elem()
1211 l_old = lookup_elem_raw(head, hash, key, key_size); in htab_lru_map_update_elem()
1228 htab_unlock_bucket(htab, b, hash, flags); in htab_lru_map_update_elem()
1248 u32 key_size, hash; in __htab_percpu_map_update_elem() local
1260 hash = htab_map_hash(key, key_size, htab->hashrnd); in __htab_percpu_map_update_elem()
1262 b = __select_bucket(htab, hash); in __htab_percpu_map_update_elem()
1265 ret = htab_lock_bucket(htab, b, hash, &flags); in __htab_percpu_map_update_elem()
1269 l_old = lookup_elem_raw(head, hash, key, key_size); in __htab_percpu_map_update_elem()
1281 hash, true, onallcpus, NULL); in __htab_percpu_map_update_elem()
1290 htab_unlock_bucket(htab, b, hash, flags); in __htab_percpu_map_update_elem()
1303 u32 key_size, hash; in __htab_lru_percpu_map_update_elem() local
1315 hash = htab_map_hash(key, key_size, htab->hashrnd); in __htab_lru_percpu_map_update_elem()
1317 b = __select_bucket(htab, hash); in __htab_lru_percpu_map_update_elem()
1326 l_new = prealloc_lru_pop(htab, key, hash); in __htab_lru_percpu_map_update_elem()
1331 ret = htab_lock_bucket(htab, b, hash, &flags); in __htab_lru_percpu_map_update_elem()
1335 l_old = lookup_elem_raw(head, hash, key, key_size); in __htab_lru_percpu_map_update_elem()
1355 htab_unlock_bucket(htab, b, hash, flags); in __htab_lru_percpu_map_update_elem()
1383 u32 hash, key_size; in htab_map_delete_elem() local
1391 hash = htab_map_hash(key, key_size, htab->hashrnd); in htab_map_delete_elem()
1392 b = __select_bucket(htab, hash); in htab_map_delete_elem()
1395 ret = htab_lock_bucket(htab, b, hash, &flags); in htab_map_delete_elem()
1399 l = lookup_elem_raw(head, hash, key, key_size); in htab_map_delete_elem()
1408 htab_unlock_bucket(htab, b, hash, flags); in htab_map_delete_elem()
1419 u32 hash, key_size; in htab_lru_map_delete_elem() local
1427 hash = htab_map_hash(key, key_size, htab->hashrnd); in htab_lru_map_delete_elem()
1428 b = __select_bucket(htab, hash); in htab_lru_map_delete_elem()
1431 ret = htab_lock_bucket(htab, b, hash, &flags); in htab_lru_map_delete_elem()
1435 l = lookup_elem_raw(head, hash, key, key_size); in htab_lru_map_delete_elem()
1442 htab_unlock_bucket(htab, b, hash, flags); in htab_lru_map_delete_elem()
1569 u32 hash, key_size; in __htab_map_lookup_and_delete_elem() local
1575 hash = htab_map_hash(key, key_size, htab->hashrnd); in __htab_map_lookup_and_delete_elem()
1576 b = __select_bucket(htab, hash); in __htab_map_lookup_and_delete_elem()
1579 ret = htab_lock_bucket(htab, b, hash, &bflags); in __htab_map_lookup_and_delete_elem()
1583 l = lookup_elem_raw(head, hash, key, key_size); in __htab_map_lookup_and_delete_elem()
1618 htab_unlock_bucket(htab, b, hash, bflags); in __htab_map_lookup_and_delete_elem()