197 static inline void htab_elem_set_ptr(struct htab_elem *l, u32 key_size,  in htab_elem_set_ptr()  argument
200 	*(void __percpu **)(l->key + key_size) = pptr;  in htab_elem_set_ptr()
203 static inline void __percpu *htab_elem_get_ptr(struct htab_elem *l, u32 key_size)  in htab_elem_get_ptr()  argument
205 	return *(void __percpu **)(l->key + key_size);  in htab_elem_get_ptr()
208 static void *fd_htab_map_get_ptr(const struct bpf_map *map, struct htab_elem *l)  in fd_htab_map_get_ptr()  argument
210 	return *(void **)(l->key + roundup(map->key_size, 8));  in fd_htab_map_get_ptr()
297 	struct htab_elem *l;  in prealloc_lru_pop()  local
300 		l = container_of(node, struct htab_elem, lru_node);  in prealloc_lru_pop()
301 		memcpy(l->key, key, htab->map.key_size);  in prealloc_lru_pop()
302 		return l;  in prealloc_lru_pop()
380 	struct pcpu_freelist_node *l;  in alloc_extra_elems()  local
389 		l = pcpu_freelist_pop(&htab->freelist);  in alloc_extra_elems()
393 		l_new = container_of(l, struct htab_elem, fnode);  in alloc_extra_elems()
623 	struct htab_elem *l;  in lookup_elem_raw()  local
625 	hlist_nulls_for_each_entry_rcu(l, n, head, hash_node)  in lookup_elem_raw()
626 		if (l->hash == hash && !memcmp(&l->key, key, key_size))  in lookup_elem_raw()
627 			return l;  in lookup_elem_raw()
641 	struct htab_elem *l;  in lookup_nulls_elem_raw()  local
644 	hlist_nulls_for_each_entry_rcu(l, n, head, hash_node)  in lookup_nulls_elem_raw()
645 		if (l->hash == hash && !memcmp(&l->key, key, key_size))  in lookup_nulls_elem_raw()
646 			return l;  in lookup_nulls_elem_raw()
663 	struct htab_elem *l;  in __htab_map_lookup_elem()  local
675 	l = lookup_nulls_elem_raw(head, hash, key, key_size, htab->n_buckets);  in __htab_map_lookup_elem()
677 	return l;  in __htab_map_lookup_elem()
682 	struct htab_elem *l = __htab_map_lookup_elem(map, key);  in htab_map_lookup_elem()  local
684 	if (l)  in htab_map_lookup_elem()
685 		return l->key + round_up(map->key_size, 8);  in htab_map_lookup_elem()
719 	struct htab_elem *l = __htab_map_lookup_elem(map, key);  in __htab_lru_map_lookup_elem()  local
721 	if (l) {  in __htab_lru_map_lookup_elem()
723 			bpf_lru_node_set_ref(&l->lru_node);  in __htab_lru_map_lookup_elem()
724 		return l->key + round_up(map->key_size, 8);  in __htab_lru_map_lookup_elem()
782 	struct htab_elem *l = NULL, *tgt_l;  in htab_lru_map_delete_node()  local
797 	hlist_nulls_for_each_entry_rcu(l, n, head, hash_node)  in htab_lru_map_delete_node()
798 		if (l == tgt_l) {  in htab_lru_map_delete_node()
799 			hlist_nulls_del_rcu(&l->hash_node);  in htab_lru_map_delete_node()
800 			check_and_free_fields(htab, l);  in htab_lru_map_delete_node()
806 	return l == tgt_l;  in htab_lru_map_delete_node()
814 	struct htab_elem *l, *next_l;  in htab_map_get_next_key()  local
830 	l = lookup_nulls_elem_raw(head, hash, key, key_size, htab->n_buckets);  in htab_map_get_next_key()
832 	if (!l)  in htab_map_get_next_key()
836 	next_l = hlist_nulls_entry_safe(rcu_dereference_raw(hlist_nulls_next_rcu(&l->hash_node)),  in htab_map_get_next_key()
868 static void htab_elem_free(struct bpf_htab *htab, struct htab_elem *l)  in htab_elem_free()  argument
871 		bpf_mem_cache_free(&htab->pcpu_ma, l->ptr_to_pptr);  in htab_elem_free()
872 	check_and_free_fields(htab, l);  in htab_elem_free()
873 	bpf_mem_cache_free(&htab->ma, l);  in htab_elem_free()
876 static void htab_put_fd_value(struct bpf_htab *htab, struct htab_elem *l)  in htab_put_fd_value()  argument
882 		ptr = fd_htab_map_get_ptr(map, l);  in htab_put_fd_value()
912 static void free_htab_elem(struct bpf_htab *htab, struct htab_elem *l)  in free_htab_elem()  argument
914 	htab_put_fd_value(htab, l);  in free_htab_elem()
917 		check_and_free_fields(htab, l);  in free_htab_elem()
918 		__pcpu_freelist_push(&htab->freelist, &l->fnode);  in free_htab_elem()
921 		htab_elem_free(htab, l);  in free_htab_elem()
994 			struct pcpu_freelist_node *l;  in alloc_htab_elem()  local
996 			l = __pcpu_freelist_pop(&htab->freelist);  in alloc_htab_elem()
997 			if (!l)  in alloc_htab_elem()
999 			l_new = container_of(l, struct htab_elem, fnode);  in alloc_htab_elem()
1381 	struct htab_elem *l;  in htab_map_delete_elem()  local
1399 	l = lookup_elem_raw(head, hash, key, key_size);  in htab_map_delete_elem()
1401 	if (l) {  in htab_map_delete_elem()
1402 		hlist_nulls_del_rcu(&l->hash_node);  in htab_map_delete_elem()
1403 		free_htab_elem(htab, l);  in htab_map_delete_elem()
1417 	struct htab_elem *l;  in htab_lru_map_delete_elem()  local
1435 	l = lookup_elem_raw(head, hash, key, key_size);  in htab_lru_map_delete_elem()
1437 	if (l)  in htab_lru_map_delete_elem()
1438 		hlist_nulls_del_rcu(&l->hash_node);  in htab_lru_map_delete_elem()
1443 	if (l)  in htab_lru_map_delete_elem()
1444 		htab_lru_push_free(htab, l);  in htab_lru_map_delete_elem()
1459 		struct htab_elem *l;  in delete_all_elements()  local
1461 		hlist_nulls_for_each_entry_safe(l, n, head, hash_node) {  in delete_all_elements()
1462 			hlist_nulls_del_rcu(&l->hash_node);  in delete_all_elements()
1463 			htab_elem_free(htab, l);  in delete_all_elements()
1477 		struct htab_elem *l;  in htab_free_malloced_timers()  local
1479 		hlist_nulls_for_each_entry(l, n, head, hash_node) {  in htab_free_malloced_timers()
1483 			bpf_timer_cancel_and_free(l->key +  in htab_free_malloced_timers()
1568 	struct htab_elem *l;  in __htab_map_lookup_and_delete_elem()  local
1583 	l = lookup_elem_raw(head, hash, key, key_size);  in __htab_map_lookup_and_delete_elem()
1584 	if (!l) {  in __htab_map_lookup_and_delete_elem()
1592 			pptr = htab_elem_get_ptr(l, key_size);  in __htab_map_lookup_and_delete_elem()
1603 				copy_map_value_locked(map, value, l->key +  in __htab_map_lookup_and_delete_elem()
1607 				copy_map_value(map, value, l->key +  in __htab_map_lookup_and_delete_elem()
1613 		hlist_nulls_del_rcu(&l->hash_node);  in __htab_map_lookup_and_delete_elem()
1615 			free_htab_elem(htab, l);  in __htab_map_lookup_and_delete_elem()
1620 	if (is_lru_map && l)  in __htab_map_lookup_and_delete_elem()
1621 		htab_lru_push_free(htab, l);  in __htab_map_lookup_and_delete_elem()
1676 	struct htab_elem *l;  in __htab_map_lookup_and_delete_batch()  local
1745 	hlist_nulls_for_each_entry_rcu(l, n, head, hash_node)  in __htab_map_lookup_and_delete_batch()
1782 	hlist_nulls_for_each_entry_safe(l, n, head, hash_node) {  in __htab_map_lookup_and_delete_batch()
1783 		memcpy(dst_key, l->key, key_size);  in __htab_map_lookup_and_delete_batch()
1789 			pptr = htab_elem_get_ptr(l, map->key_size);  in __htab_map_lookup_and_delete_batch()
1796 			value = l->key + roundup_key_size;  in __htab_map_lookup_and_delete_batch()
1814 			hlist_nulls_del_rcu(&l->hash_node);  in __htab_map_lookup_and_delete_batch()
1822 				l->batch_flink = node_to_free;  in __htab_map_lookup_and_delete_batch()
1823 				node_to_free = l;  in __htab_map_lookup_and_delete_batch()
1825 				free_htab_elem(htab, l);  in __htab_map_lookup_and_delete_batch()
1836 		l = node_to_free;  in __htab_map_lookup_and_delete_batch()
1838 		htab_lru_push_free(htab, l);  in __htab_map_lookup_and_delete_batch()
2239 	struct htab_elem *l = __htab_map_lookup_elem(map, key);  in htab_percpu_map_lookup_elem()  local
2241 	if (l)  in htab_percpu_map_lookup_elem()
2242 		return this_cpu_ptr(htab_elem_get_ptr(l, map->key_size));  in htab_percpu_map_lookup_elem()
2249 	struct htab_elem *l;  in htab_percpu_map_lookup_percpu_elem()  local
2254 	l = __htab_map_lookup_elem(map, key);  in htab_percpu_map_lookup_percpu_elem()
2255 	if (l)  in htab_percpu_map_lookup_percpu_elem()
2256 		return per_cpu_ptr(htab_elem_get_ptr(l, map->key_size), cpu);  in htab_percpu_map_lookup_percpu_elem()
2263 	struct htab_elem *l = __htab_map_lookup_elem(map, key);  in htab_lru_percpu_map_lookup_elem()  local
2265 	if (l) {  in htab_lru_percpu_map_lookup_elem()
2266 		bpf_lru_node_set_ref(&l->lru_node);  in htab_lru_percpu_map_lookup_elem()
2267 		return this_cpu_ptr(htab_elem_get_ptr(l, map->key_size));  in htab_lru_percpu_map_lookup_elem()
2275 	struct htab_elem *l;  in htab_lru_percpu_map_lookup_percpu_elem()  local
2280 	l = __htab_map_lookup_elem(map, key);  in htab_lru_percpu_map_lookup_percpu_elem()
2281 	if (l) {  in htab_lru_percpu_map_lookup_percpu_elem()
2282 		bpf_lru_node_set_ref(&l->lru_node);  in htab_lru_percpu_map_lookup_percpu_elem()
2283 		return per_cpu_ptr(htab_elem_get_ptr(l, map->key_size), cpu);  in htab_lru_percpu_map_lookup_percpu_elem()
2291 	struct htab_elem *l;  in bpf_percpu_hash_copy()  local
2303 	l = __htab_map_lookup_elem(map, key);  in bpf_percpu_hash_copy()
2304 	if (!l)  in bpf_percpu_hash_copy()
2309 	pptr = htab_elem_get_ptr(l, map->key_size);  in bpf_percpu_hash_copy()
2342 	struct htab_elem *l;  in htab_percpu_map_seq_show_elem()  local
2348 	l = __htab_map_lookup_elem(map, key);  in htab_percpu_map_seq_show_elem()
2349 	if (!l) {  in htab_percpu_map_seq_show_elem()
2356 	pptr = htab_elem_get_ptr(l, map->key_size);  in htab_percpu_map_seq_show_elem()
2418 	struct htab_elem *l;  in fd_htab_map_free()  local
2424 		hlist_nulls_for_each_entry_safe(l, n, head, hash_node) {  in fd_htab_map_free()
2425 			void *ptr = fd_htab_map_get_ptr(map, l);  in fd_htab_map_free()