122 	char key[] __aligned(8);  member
187 	*(void __percpu **)(l->key + key_size) = pptr;  in htab_elem_set_ptr()
192 	return *(void __percpu **)(l->key + key_size);  in htab_elem_get_ptr()
197 	return *(void **)(l->key + roundup(map->key_size, 8));  in fd_htab_map_get_ptr()
235 static struct htab_elem *prealloc_lru_pop(struct bpf_htab *htab, void *key,  in prealloc_lru_pop()  argument
243 		memcpy(l->key, key, htab->map.key_size);  in prealloc_lru_pop()
518 static inline u32 htab_map_hash(const void *key, u32 key_len, u32 hashrnd)  in htab_map_hash()  argument
520 	return jhash(key, key_len, hashrnd);  in htab_map_hash()
535 					 void *key, u32 key_size)  in lookup_elem_raw()  argument
541 		if (l->hash == hash && !memcmp(&l->key, key, key_size))  in lookup_elem_raw()
552 					       u32 hash, void *key,  in lookup_nulls_elem_raw()  argument
560 		if (l->hash == hash && !memcmp(&l->key, key, key_size))  in lookup_nulls_elem_raw()
574 static void *__htab_map_lookup_elem(struct bpf_map *map, void *key)  in __htab_map_lookup_elem()  argument
585 	hash = htab_map_hash(key, key_size, htab->hashrnd);  in __htab_map_lookup_elem()
589 	l = lookup_nulls_elem_raw(head, hash, key, key_size, htab->n_buckets);  in __htab_map_lookup_elem()
594 static void *htab_map_lookup_elem(struct bpf_map *map, void *key)  in htab_map_lookup_elem()  argument
596 	struct htab_elem *l = __htab_map_lookup_elem(map, key);  in htab_map_lookup_elem()
599 		return l->key + round_up(map->key_size, 8);  in htab_map_lookup_elem()
621 		     (void *(*)(struct bpf_map *map, void *key))NULL));  in htab_map_gen_lookup()
625 				offsetof(struct htab_elem, key) +  in htab_map_gen_lookup()
631 							void *key, const bool mark)  in __htab_lru_map_lookup_elem()  argument
633 	struct htab_elem *l = __htab_map_lookup_elem(map, key);  in __htab_lru_map_lookup_elem()
638 		return l->key + round_up(map->key_size, 8);  in __htab_lru_map_lookup_elem()
644 static void *htab_lru_map_lookup_elem(struct bpf_map *map, void *key)  in htab_lru_map_lookup_elem()  argument
646 	return __htab_lru_map_lookup_elem(map, key, true);  in htab_lru_map_lookup_elem()
649 static void *htab_lru_map_lookup_elem_sys(struct bpf_map *map, void *key)  in htab_lru_map_lookup_elem_sys()  argument
651 	return __htab_lru_map_lookup_elem(map, key, false);  in htab_lru_map_lookup_elem_sys()
662 		     (void *(*)(struct bpf_map *map, void *key))NULL));  in htab_lru_map_gen_lookup()
674 				offsetof(struct htab_elem, key) +  in htab_lru_map_gen_lookup()
709 static int htab_map_get_next_key(struct bpf_map *map, void *key, void *next_key)  in htab_map_get_next_key()  argument
721 	if (!key)  in htab_map_get_next_key()
724 	hash = htab_map_hash(key, key_size, htab->hashrnd);  in htab_map_get_next_key()
729 	l = lookup_nulls_elem_raw(head, hash, key, key_size, htab->n_buckets);  in htab_map_get_next_key()
740 		memcpy(next_key, next_l->key, key_size);  in htab_map_get_next_key()
758 			memcpy(next_key, next_l->key, key_size);  in htab_map_get_next_key()
856 static struct htab_elem *alloc_htab_elem(struct bpf_htab *htab, void *key,  in alloc_htab_elem()  argument
901 					l_new->key + round_up(key_size, 8));  in alloc_htab_elem()
904 	memcpy(l_new->key, key, key_size);  in alloc_htab_elem()
926 		memcpy(l_new->key + round_up(key_size, 8), value, size);  in alloc_htab_elem()
929 			       l_new->key + round_up(key_size, 8),  in alloc_htab_elem()
955 static int htab_map_update_elem(struct bpf_map *map, void *key, void *value,  in htab_map_update_elem()  argument
974 	hash = htab_map_hash(key, key_size, htab->hashrnd);  in htab_map_update_elem()
983 		l_old = lookup_nulls_elem_raw(head, hash, key, key_size,  in htab_map_update_elem()
991 					      l_old->key + round_up(key_size, 8),  in htab_map_update_elem()
1003 	l_old = lookup_elem_raw(head, hash, key, key_size);  in htab_map_update_elem()
1017 				      l_old->key + round_up(key_size, 8),  in htab_map_update_elem()
1023 	l_new = alloc_htab_elem(htab, key, value, key_size, hash, false, false,  in htab_map_update_elem()
1046 static int htab_lru_map_update_elem(struct bpf_map *map, void *key, void *value,  in htab_lru_map_update_elem()  argument
1065 	hash = htab_map_hash(key, key_size, htab->hashrnd);  in htab_lru_map_update_elem()
1075 	l_new = prealloc_lru_pop(htab, key, hash);  in htab_lru_map_update_elem()
1078 	memcpy(l_new->key + round_up(map->key_size, 8), value, map->value_size);  in htab_lru_map_update_elem()
1082 	l_old = lookup_elem_raw(head, hash, key, key_size);  in htab_lru_map_update_elem()
1109 static int __htab_percpu_map_update_elem(struct bpf_map *map, void *key,  in __htab_percpu_map_update_elem()  argument
1129 	hash = htab_map_hash(key, key_size, htab->hashrnd);  in __htab_percpu_map_update_elem()
1136 	l_old = lookup_elem_raw(head, hash, key, key_size);  in __htab_percpu_map_update_elem()
1147 		l_new = alloc_htab_elem(htab, key, value, key_size,  in __htab_percpu_map_update_elem()
1161 static int __htab_lru_percpu_map_update_elem(struct bpf_map *map, void *key,  in __htab_lru_percpu_map_update_elem()  argument
1181 	hash = htab_map_hash(key, key_size, htab->hashrnd);  in __htab_lru_percpu_map_update_elem()
1192 		l_new = prealloc_lru_pop(htab, key, hash);  in __htab_lru_percpu_map_update_elem()
1199 	l_old = lookup_elem_raw(head, hash, key, key_size);  in __htab_lru_percpu_map_update_elem()
1225 static int htab_percpu_map_update_elem(struct bpf_map *map, void *key,  in htab_percpu_map_update_elem()  argument
1228 	return __htab_percpu_map_update_elem(map, key, value, map_flags, false);  in htab_percpu_map_update_elem()
1231 static int htab_lru_percpu_map_update_elem(struct bpf_map *map, void *key,  in htab_lru_percpu_map_update_elem()  argument
1234 	return __htab_lru_percpu_map_update_elem(map, key, value, map_flags,  in htab_lru_percpu_map_update_elem()
1239 static int htab_map_delete_elem(struct bpf_map *map, void *key)  in htab_map_delete_elem()  argument
1253 	hash = htab_map_hash(key, key_size, htab->hashrnd);  in htab_map_delete_elem()
1259 	l = lookup_elem_raw(head, hash, key, key_size);  in htab_map_delete_elem()
1271 static int htab_lru_map_delete_elem(struct bpf_map *map, void *key)  in htab_lru_map_delete_elem()  argument
1285 	hash = htab_map_hash(key, key_size, htab->hashrnd);  in htab_lru_map_delete_elem()
1291 	l = lookup_elem_raw(head, hash, key, key_size);  in htab_lru_map_delete_elem()
1344 static void htab_map_seq_show_elem(struct bpf_map *map, void *key,  in htab_map_seq_show_elem()  argument
1351 	value = htab_map_lookup_elem(map, key);  in htab_map_seq_show_elem()
1357 	btf_type_seq_show(map->btf, map->btf_key_type_id, key, m);  in htab_map_seq_show_elem()
1486 		memcpy(dst_key, l->key, key_size);  in __htab_map_lookup_and_delete_batch()
1499 			value = l->key + roundup_key_size;  in __htab_map_lookup_and_delete_batch()
1752 			ctx.key = elem->key;  in __bpf_hash_map_seq_show()
1754 				ctx.value = elem->key + roundup_key_size;  in __bpf_hash_map_seq_show()
1870 static void *htab_percpu_map_lookup_elem(struct bpf_map *map, void *key)  in htab_percpu_map_lookup_elem()  argument
1872 	struct htab_elem *l = __htab_map_lookup_elem(map, key);  in htab_percpu_map_lookup_elem()
1880 static void *htab_lru_percpu_map_lookup_elem(struct bpf_map *map, void *key)  in htab_lru_percpu_map_lookup_elem()  argument
1882 	struct htab_elem *l = __htab_map_lookup_elem(map, key);  in htab_lru_percpu_map_lookup_elem()
1892 int bpf_percpu_hash_copy(struct bpf_map *map, void *key, void *value)  in bpf_percpu_hash_copy()  argument
1906 	l = __htab_map_lookup_elem(map, key);  in bpf_percpu_hash_copy()
1924 int bpf_percpu_hash_update(struct bpf_map *map, void *key, void *value,  in bpf_percpu_hash_update()  argument
1932 		ret = __htab_lru_percpu_map_update_elem(map, key, value,  in bpf_percpu_hash_update()
1935 		ret = __htab_percpu_map_update_elem(map, key, value, map_flags,  in bpf_percpu_hash_update()
1942 static void htab_percpu_map_seq_show_elem(struct bpf_map *map, void *key,  in htab_percpu_map_seq_show_elem()  argument
1951 	l = __htab_map_lookup_elem(map, key);  in htab_percpu_map_seq_show_elem()
1957 	btf_type_seq_show(map->btf, map->btf_key_type_id, key, m);  in htab_percpu_map_seq_show_elem()
2034 int bpf_fd_htab_map_lookup_elem(struct bpf_map *map, void *key, u32 *value)  in bpf_fd_htab_map_lookup_elem()  argument
2043 	ptr = htab_map_lookup_elem(map, key);  in bpf_fd_htab_map_lookup_elem()
2055 				void *key, void *value, u64 map_flags)  in bpf_fd_htab_map_update_elem()  argument
2065 	ret = htab_map_update_elem(map, key, &ptr, map_flags);  in bpf_fd_htab_map_update_elem()
2091 static void *htab_of_map_lookup_elem(struct bpf_map *map, void *key)  in htab_of_map_lookup_elem()  argument
2093 	struct bpf_map **inner_map  = htab_map_lookup_elem(map, key);  in htab_of_map_lookup_elem()
2108 		     (void *(*)(struct bpf_map *map, void *key))NULL));  in htab_of_map_gen_lookup()
2112 				offsetof(struct htab_elem, key) +  in htab_of_map_gen_lookup()