79 static inline void htab_elem_set_ptr(struct htab_elem *l, u32 key_size,  in htab_elem_set_ptr()  argument
82 	*(void __percpu **)(l->key + key_size) = pptr;  in htab_elem_set_ptr()
85 static inline void __percpu *htab_elem_get_ptr(struct htab_elem *l, u32 key_size)  in htab_elem_get_ptr()  argument
87 	return *(void __percpu **)(l->key + key_size);  in htab_elem_get_ptr()
90 static void *fd_htab_map_get_ptr(const struct bpf_map *map, struct htab_elem *l)  in fd_htab_map_get_ptr()  argument
92 	return *(void **)(l->key + roundup(map->key_size, 8));  in fd_htab_map_get_ptr()
123 	struct htab_elem *l;  in prealloc_lru_pop()  local
126 		l = container_of(node, struct htab_elem, lru_node);  in prealloc_lru_pop()
127 		memcpy(l->key, key, htab->map.key_size);  in prealloc_lru_pop()
128 		return l;  in prealloc_lru_pop()
205 	struct pcpu_freelist_node *l;  in alloc_extra_elems()  local
214 		l = pcpu_freelist_pop(&htab->freelist);  in alloc_extra_elems()
218 		l_new = container_of(l, struct htab_elem, fnode);  in alloc_extra_elems()
425 	struct htab_elem *l;  in lookup_elem_raw()  local
427 	hlist_nulls_for_each_entry_rcu(l, n, head, hash_node)  in lookup_elem_raw()
428 		if (l->hash == hash && !memcmp(&l->key, key, key_size))  in lookup_elem_raw()
429 			return l;  in lookup_elem_raw()
443 	struct htab_elem *l;  in lookup_nulls_elem_raw()  local
446 	hlist_nulls_for_each_entry_rcu(l, n, head, hash_node)  in lookup_nulls_elem_raw()
447 		if (l->hash == hash && !memcmp(&l->key, key, key_size))  in lookup_nulls_elem_raw()
448 			return l;  in lookup_nulls_elem_raw()
465 	struct htab_elem *l;  in __htab_map_lookup_elem()  local
477 	l = lookup_nulls_elem_raw(head, hash, key, key_size, htab->n_buckets);  in __htab_map_lookup_elem()
479 	return l;  in __htab_map_lookup_elem()
484 	struct htab_elem *l = __htab_map_lookup_elem(map, key);  in htab_map_lookup_elem()  local
486 	if (l)  in htab_map_lookup_elem()
487 		return l->key + round_up(map->key_size, 8);  in htab_map_lookup_elem()
521 	struct htab_elem *l = __htab_map_lookup_elem(map, key);  in __htab_lru_map_lookup_elem()  local
523 	if (l) {  in __htab_lru_map_lookup_elem()
525 			bpf_lru_node_set_ref(&l->lru_node);  in __htab_lru_map_lookup_elem()
526 		return l->key + round_up(map->key_size, 8);  in __htab_lru_map_lookup_elem()
573 	struct htab_elem *l = NULL, *tgt_l;  in htab_lru_map_delete_node()  local
585 	hlist_nulls_for_each_entry_rcu(l, n, head, hash_node)  in htab_lru_map_delete_node()
586 		if (l == tgt_l) {  in htab_lru_map_delete_node()
587 			hlist_nulls_del_rcu(&l->hash_node);  in htab_lru_map_delete_node()
593 	return l == tgt_l;  in htab_lru_map_delete_node()
601 	struct htab_elem *l, *next_l;  in htab_map_get_next_key()  local
617 	l = lookup_nulls_elem_raw(head, hash, key, key_size, htab->n_buckets);  in htab_map_get_next_key()
619 	if (!l)  in htab_map_get_next_key()
623 	next_l = hlist_nulls_entry_safe(rcu_dereference_raw(hlist_nulls_next_rcu(&l->hash_node)),  in htab_map_get_next_key()
655 static void htab_elem_free(struct bpf_htab *htab, struct htab_elem *l)  in htab_elem_free()  argument
658 		free_percpu(htab_elem_get_ptr(l, htab->map.key_size));  in htab_elem_free()
659 	kfree(l);  in htab_elem_free()
664 	struct htab_elem *l = container_of(head, struct htab_elem, rcu);  in htab_elem_free_rcu()  local
665 	struct bpf_htab *htab = l->htab;  in htab_elem_free_rcu()
673 	htab_elem_free(htab, l);  in htab_elem_free_rcu()
678 static void free_htab_elem(struct bpf_htab *htab, struct htab_elem *l)  in free_htab_elem()  argument
683 		void *ptr = fd_htab_map_get_ptr(map, l);  in free_htab_elem()
689 		__pcpu_freelist_push(&htab->freelist, &l->fnode);  in free_htab_elem()
692 		l->htab = htab;  in free_htab_elem()
693 		call_rcu(&l->rcu, htab_elem_free_rcu);  in free_htab_elem()
740 			struct pcpu_freelist_node *l;  in alloc_htab_elem()  local
742 			l = __pcpu_freelist_pop(&htab->freelist);  in alloc_htab_elem()
743 			if (!l)  in alloc_htab_elem()
745 			l_new = container_of(l, struct htab_elem, fnode);  in alloc_htab_elem()
1112 	struct htab_elem *l;  in htab_map_delete_elem()  local
1127 	l = lookup_elem_raw(head, hash, key, key_size);  in htab_map_delete_elem()
1129 	if (l) {  in htab_map_delete_elem()
1130 		hlist_nulls_del_rcu(&l->hash_node);  in htab_map_delete_elem()
1131 		free_htab_elem(htab, l);  in htab_map_delete_elem()
1144 	struct htab_elem *l;  in htab_lru_map_delete_elem()  local
1159 	l = lookup_elem_raw(head, hash, key, key_size);  in htab_lru_map_delete_elem()
1161 	if (l) {  in htab_lru_map_delete_elem()
1162 		hlist_nulls_del_rcu(&l->hash_node);  in htab_lru_map_delete_elem()
1167 	if (l)  in htab_lru_map_delete_elem()
1168 		bpf_lru_push_free(&htab->lru, &l->lru_node);  in htab_lru_map_delete_elem()
1179 		struct htab_elem *l;  in delete_all_elements()  local
1181 		hlist_nulls_for_each_entry_safe(l, n, head, hash_node) {  in delete_all_elements()
1182 			hlist_nulls_del_rcu(&l->hash_node);  in delete_all_elements()
1183 			htab_elem_free(htab, l);  in delete_all_elements()
1263 	struct htab_elem *l = __htab_map_lookup_elem(map, key);  in htab_percpu_map_lookup_elem()  local
1265 	if (l)  in htab_percpu_map_lookup_elem()
1266 		return this_cpu_ptr(htab_elem_get_ptr(l, map->key_size));  in htab_percpu_map_lookup_elem()
1273 	struct htab_elem *l = __htab_map_lookup_elem(map, key);  in htab_lru_percpu_map_lookup_elem()  local
1275 	if (l) {  in htab_lru_percpu_map_lookup_elem()
1276 		bpf_lru_node_set_ref(&l->lru_node);  in htab_lru_percpu_map_lookup_elem()
1277 		return this_cpu_ptr(htab_elem_get_ptr(l, map->key_size));  in htab_lru_percpu_map_lookup_elem()
1285 	struct htab_elem *l;  in bpf_percpu_hash_copy()  local
1297 	l = __htab_map_lookup_elem(map, key);  in bpf_percpu_hash_copy()
1298 	if (!l)  in bpf_percpu_hash_copy()
1303 	pptr = htab_elem_get_ptr(l, map->key_size);  in bpf_percpu_hash_copy()
1336 	struct htab_elem *l;  in htab_percpu_map_seq_show_elem()  local
1342 	l = __htab_map_lookup_elem(map, key);  in htab_percpu_map_seq_show_elem()
1343 	if (!l) {  in htab_percpu_map_seq_show_elem()
1350 	pptr = htab_elem_get_ptr(l, map->key_size);  in htab_percpu_map_seq_show_elem()
1396 	struct htab_elem *l;  in fd_htab_map_free()  local
1402 		hlist_nulls_for_each_entry_safe(l, n, head, hash_node) {  in fd_htab_map_free()
1403 			void *ptr = fd_htab_map_get_ptr(map, l);  in fd_htab_map_free()