128 	char key[] __aligned(8);  member
200 	*(void __percpu **)(l->key + key_size) = pptr;  in htab_elem_set_ptr()
205 	return *(void __percpu **)(l->key + key_size);  in htab_elem_get_ptr()
210 	return *(void **)(l->key + roundup(map->key_size, 8));  in fd_htab_map_get_ptr()
237 		bpf_timer_cancel_and_free(elem->key +  in htab_free_prealloced_timers()
258 		bpf_map_free_kptrs(&htab->map, elem->key + round_up(htab->map.key_size, 8));  in htab_free_prealloced_kptrs()
293 static struct htab_elem *prealloc_lru_pop(struct bpf_htab *htab, void *key,  in prealloc_lru_pop()  argument
301 		memcpy(l->key, key, htab->map.key_size);  in prealloc_lru_pop()
603 static inline u32 htab_map_hash(const void *key, u32 key_len, u32 hashrnd)  in htab_map_hash()  argument
605 	return jhash(key, key_len, hashrnd);  in htab_map_hash()
620 					 void *key, u32 key_size)  in lookup_elem_raw()  argument
626 		if (l->hash == hash && !memcmp(&l->key, key, key_size))  in lookup_elem_raw()
637 					       u32 hash, void *key,  in lookup_nulls_elem_raw()  argument
645 		if (l->hash == hash && !memcmp(&l->key, key, key_size))  in lookup_nulls_elem_raw()
659 static void *__htab_map_lookup_elem(struct bpf_map *map, void *key)  in __htab_map_lookup_elem()  argument
671 	hash = htab_map_hash(key, key_size, htab->hashrnd);  in __htab_map_lookup_elem()
675 	l = lookup_nulls_elem_raw(head, hash, key, key_size, htab->n_buckets);  in __htab_map_lookup_elem()
680 static void *htab_map_lookup_elem(struct bpf_map *map, void *key)  in htab_map_lookup_elem()  argument
682 	struct htab_elem *l = __htab_map_lookup_elem(map, key);  in htab_map_lookup_elem()
685 		return l->key + round_up(map->key_size, 8);  in htab_map_lookup_elem()
707 		     (void *(*)(struct bpf_map *map, void *key))NULL));  in htab_map_gen_lookup()
711 				offsetof(struct htab_elem, key) +  in htab_map_gen_lookup()
717 							void *key, const bool mark)  in __htab_lru_map_lookup_elem()  argument
719 	struct htab_elem *l = __htab_map_lookup_elem(map, key);  in __htab_lru_map_lookup_elem()
724 		return l->key + round_up(map->key_size, 8);  in __htab_lru_map_lookup_elem()
730 static void *htab_lru_map_lookup_elem(struct bpf_map *map, void *key)  in htab_lru_map_lookup_elem()  argument
732 	return __htab_lru_map_lookup_elem(map, key, true);  in htab_lru_map_lookup_elem()
735 static void *htab_lru_map_lookup_elem_sys(struct bpf_map *map, void *key)  in htab_lru_map_lookup_elem_sys()  argument
737 	return __htab_lru_map_lookup_elem(map, key, false);  in htab_lru_map_lookup_elem_sys()
748 		     (void *(*)(struct bpf_map *map, void *key))NULL));  in htab_lru_map_gen_lookup()
760 				offsetof(struct htab_elem, key) +  in htab_lru_map_gen_lookup()
768 	void *map_value = elem->key + round_up(htab->map.key_size, 8);  in check_and_free_fields()
810 static int htab_map_get_next_key(struct bpf_map *map, void *key, void *next_key)  in htab_map_get_next_key()  argument
822 	if (!key)  in htab_map_get_next_key()
825 	hash = htab_map_hash(key, key_size, htab->hashrnd);  in htab_map_get_next_key()
830 	l = lookup_nulls_elem_raw(head, hash, key, key_size, htab->n_buckets);  in htab_map_get_next_key()
841 		memcpy(next_key, next_l->key, key_size);  in htab_map_get_next_key()
859 			memcpy(next_key, next_l->key, key_size);  in htab_map_get_next_key()
974 static struct htab_elem *alloc_htab_elem(struct bpf_htab *htab, void *key,  in alloc_htab_elem()  argument
1018 	memcpy(l_new->key, key, key_size);  in alloc_htab_elem()
1040 		memcpy(l_new->key + round_up(key_size, 8), value, size);  in alloc_htab_elem()
1043 			       l_new->key + round_up(key_size, 8),  in alloc_htab_elem()
1069 static int htab_map_update_elem(struct bpf_map *map, void *key, void *value,  in htab_map_update_elem()  argument
1089 	hash = htab_map_hash(key, key_size, htab->hashrnd);  in htab_map_update_elem()
1098 		l_old = lookup_nulls_elem_raw(head, hash, key, key_size,  in htab_map_update_elem()
1106 					      l_old->key + round_up(key_size, 8),  in htab_map_update_elem()
1120 	l_old = lookup_elem_raw(head, hash, key, key_size);  in htab_map_update_elem()
1134 				      l_old->key + round_up(key_size, 8),  in htab_map_update_elem()
1140 	l_new = alloc_htab_elem(htab, key, value, key_size, hash, false, false,  in htab_map_update_elem()
1171 static int htab_lru_map_update_elem(struct bpf_map *map, void *key, void *value,  in htab_lru_map_update_elem()  argument
1191 	hash = htab_map_hash(key, key_size, htab->hashrnd);  in htab_lru_map_update_elem()
1201 	l_new = prealloc_lru_pop(htab, key, hash);  in htab_lru_map_update_elem()
1205 		       l_new->key + round_up(map->key_size, 8), value);  in htab_lru_map_update_elem()
1211 	l_old = lookup_elem_raw(head, hash, key, key_size);  in htab_lru_map_update_elem()
1239 static int __htab_percpu_map_update_elem(struct bpf_map *map, void *key,  in __htab_percpu_map_update_elem()  argument
1260 	hash = htab_map_hash(key, key_size, htab->hashrnd);  in __htab_percpu_map_update_elem()
1269 	l_old = lookup_elem_raw(head, hash, key, key_size);  in __htab_percpu_map_update_elem()
1280 		l_new = alloc_htab_elem(htab, key, value, key_size,  in __htab_percpu_map_update_elem()
1294 static int __htab_lru_percpu_map_update_elem(struct bpf_map *map, void *key,  in __htab_lru_percpu_map_update_elem()  argument
1315 	hash = htab_map_hash(key, key_size, htab->hashrnd);  in __htab_lru_percpu_map_update_elem()
1326 		l_new = prealloc_lru_pop(htab, key, hash);  in __htab_lru_percpu_map_update_elem()
1335 	l_old = lookup_elem_raw(head, hash, key, key_size);  in __htab_lru_percpu_map_update_elem()
1362 static int htab_percpu_map_update_elem(struct bpf_map *map, void *key,  in htab_percpu_map_update_elem()  argument
1365 	return __htab_percpu_map_update_elem(map, key, value, map_flags, false);  in htab_percpu_map_update_elem()
1368 static int htab_lru_percpu_map_update_elem(struct bpf_map *map, void *key,  in htab_lru_percpu_map_update_elem()  argument
1371 	return __htab_lru_percpu_map_update_elem(map, key, value, map_flags,  in htab_lru_percpu_map_update_elem()
1376 static int htab_map_delete_elem(struct bpf_map *map, void *key)  in htab_map_delete_elem()  argument
1391 	hash = htab_map_hash(key, key_size, htab->hashrnd);  in htab_map_delete_elem()
1399 	l = lookup_elem_raw(head, hash, key, key_size);  in htab_map_delete_elem()
1412 static int htab_lru_map_delete_elem(struct bpf_map *map, void *key)  in htab_lru_map_delete_elem()  argument
1427 	hash = htab_map_hash(key, key_size, htab->hashrnd);  in htab_lru_map_delete_elem()
1435 	l = lookup_elem_raw(head, hash, key, key_size);  in htab_lru_map_delete_elem()
1483 			bpf_timer_cancel_and_free(l->key +  in htab_free_malloced_timers()
1540 static void htab_map_seq_show_elem(struct bpf_map *map, void *key,  in htab_map_seq_show_elem()  argument
1547 	value = htab_map_lookup_elem(map, key);  in htab_map_seq_show_elem()
1553 	btf_type_seq_show(map->btf, map->btf_key_type_id, key, m);  in htab_map_seq_show_elem()
1561 static int __htab_map_lookup_and_delete_elem(struct bpf_map *map, void *key,  in __htab_map_lookup_and_delete_elem()  argument
1575 	hash = htab_map_hash(key, key_size, htab->hashrnd);  in __htab_map_lookup_and_delete_elem()
1583 	l = lookup_elem_raw(head, hash, key, key_size);  in __htab_map_lookup_and_delete_elem()
1603 				copy_map_value_locked(map, value, l->key +  in __htab_map_lookup_and_delete_elem()
1607 				copy_map_value(map, value, l->key +  in __htab_map_lookup_and_delete_elem()
1626 static int htab_map_lookup_and_delete_elem(struct bpf_map *map, void *key,  in htab_map_lookup_and_delete_elem()  argument
1629 	return __htab_map_lookup_and_delete_elem(map, key, value, false, false,  in htab_map_lookup_and_delete_elem()
1634 						  void *key, void *value,  in htab_percpu_map_lookup_and_delete_elem()  argument
1637 	return __htab_map_lookup_and_delete_elem(map, key, value, false, true,  in htab_percpu_map_lookup_and_delete_elem()
1641 static int htab_lru_map_lookup_and_delete_elem(struct bpf_map *map, void *key,  in htab_lru_map_lookup_and_delete_elem()  argument
1644 	return __htab_map_lookup_and_delete_elem(map, key, value, true, false,  in htab_lru_map_lookup_and_delete_elem()
1649 						      void *key, void *value,  in htab_lru_percpu_map_lookup_and_delete_elem()  argument
1652 	return __htab_map_lookup_and_delete_elem(map, key, value, true, true,  in htab_lru_percpu_map_lookup_and_delete_elem()
1783 		memcpy(dst_key, l->key, key_size);  in __htab_map_lookup_and_delete_batch()
1796 			value = l->key + roundup_key_size;  in __htab_map_lookup_and_delete_batch()
2058 			ctx.key = elem->key;  in __bpf_hash_map_seq_show()
2060 				ctx.value = elem->key + roundup_key_size;  in __bpf_hash_map_seq_show()
2149 	void *key, *val;  in bpf_for_each_hash_elem()  local
2169 			key = elem->key;  in bpf_for_each_hash_elem()
2175 				val = elem->key + roundup_key_size;  in bpf_for_each_hash_elem()
2178 			ret = callback_fn((u64)(long)map, (u64)(long)key,  in bpf_for_each_hash_elem()
2237 static void *htab_percpu_map_lookup_elem(struct bpf_map *map, void *key)  in htab_percpu_map_lookup_elem()  argument
2239 	struct htab_elem *l = __htab_map_lookup_elem(map, key);  in htab_percpu_map_lookup_elem()
2247 static void *htab_percpu_map_lookup_percpu_elem(struct bpf_map *map, void *key, u32 cpu)  in htab_percpu_map_lookup_percpu_elem()  argument
2254 	l = __htab_map_lookup_elem(map, key);  in htab_percpu_map_lookup_percpu_elem()
2261 static void *htab_lru_percpu_map_lookup_elem(struct bpf_map *map, void *key)  in htab_lru_percpu_map_lookup_elem()  argument
2263 	struct htab_elem *l = __htab_map_lookup_elem(map, key);  in htab_lru_percpu_map_lookup_elem()
2273 static void *htab_lru_percpu_map_lookup_percpu_elem(struct bpf_map *map, void *key, u32 cpu)  in htab_lru_percpu_map_lookup_percpu_elem()  argument
2280 	l = __htab_map_lookup_elem(map, key);  in htab_lru_percpu_map_lookup_percpu_elem()
2289 int bpf_percpu_hash_copy(struct bpf_map *map, void *key, void *value)  in bpf_percpu_hash_copy()  argument
2303 	l = __htab_map_lookup_elem(map, key);  in bpf_percpu_hash_copy()
2321 int bpf_percpu_hash_update(struct bpf_map *map, void *key, void *value,  in bpf_percpu_hash_update()  argument
2329 		ret = __htab_lru_percpu_map_update_elem(map, key, value,  in bpf_percpu_hash_update()
2332 		ret = __htab_percpu_map_update_elem(map, key, value, map_flags,  in bpf_percpu_hash_update()
2339 static void htab_percpu_map_seq_show_elem(struct bpf_map *map, void *key,  in htab_percpu_map_seq_show_elem()  argument
2348 	l = __htab_map_lookup_elem(map, key);  in htab_percpu_map_seq_show_elem()
2354 	btf_type_seq_show(map->btf, map->btf_key_type_id, key, m);  in htab_percpu_map_seq_show_elem()
2435 int bpf_fd_htab_map_lookup_elem(struct bpf_map *map, void *key, u32 *value)  in bpf_fd_htab_map_lookup_elem()  argument
2444 	ptr = htab_map_lookup_elem(map, key);  in bpf_fd_htab_map_lookup_elem()
2456 				void *key, void *value, u64 map_flags)  in bpf_fd_htab_map_update_elem()  argument
2466 	ret = htab_map_update_elem(map, key, &ptr, map_flags);  in bpf_fd_htab_map_update_elem()
2492 static void *htab_of_map_lookup_elem(struct bpf_map *map, void *key)  in htab_of_map_lookup_elem()  argument
2494 	struct bpf_map **inner_map  = htab_map_lookup_elem(map, key);  in htab_of_map_lookup_elem()
2509 		     (void *(*)(struct bpf_map *map, void *key))NULL));  in htab_of_map_gen_lookup()
2513 				offsetof(struct htab_elem, key) +  in htab_of_map_gen_lookup()