99 	struct bpf_local_storage *local_storage;  in bpf_local_storage_free_rcu()  local
101 	local_storage = container_of(rcu, struct bpf_local_storage, rcu);  in bpf_local_storage_free_rcu()
102 	kfree_rcu(local_storage, rcu);  in bpf_local_storage_free_rcu()
117 bool bpf_selem_unlink_storage_nolock(struct bpf_local_storage *local_storage,  in bpf_selem_unlink_storage_nolock()  argument
126 	owner = local_storage->owner;  in bpf_selem_unlink_storage_nolock()
136 						    &local_storage->list);  in bpf_selem_unlink_storage_nolock()
139 		local_storage->owner = NULL;  in bpf_selem_unlink_storage_nolock()
159 	if (rcu_access_pointer(local_storage->cache[smap->cache_idx]) ==  in bpf_selem_unlink_storage_nolock()
161 		RCU_INIT_POINTER(local_storage->cache[smap->cache_idx], NULL);  in bpf_selem_unlink_storage_nolock()
174 	struct bpf_local_storage *local_storage;  in __bpf_selem_unlink_storage()  local
182 	local_storage = rcu_dereference_check(selem->local_storage,  in __bpf_selem_unlink_storage()
184 	raw_spin_lock_irqsave(&local_storage->lock, flags);  in __bpf_selem_unlink_storage()
187 			local_storage, selem, true, use_trace_rcu);  in __bpf_selem_unlink_storage()
188 	raw_spin_unlock_irqrestore(&local_storage->lock, flags);  in __bpf_selem_unlink_storage()
192 			call_rcu_tasks_trace(&local_storage->rcu,  in __bpf_selem_unlink_storage()
195 			kfree_rcu(local_storage, rcu);  in __bpf_selem_unlink_storage()
199 void bpf_selem_link_storage_nolock(struct bpf_local_storage *local_storage,  in bpf_selem_link_storage_nolock()  argument
202 	RCU_INIT_POINTER(selem->local_storage, local_storage);  in bpf_selem_link_storage_nolock()
203 	hlist_add_head_rcu(&selem->snode, &local_storage->list);  in bpf_selem_link_storage_nolock()
247 bpf_local_storage_lookup(struct bpf_local_storage *local_storage,  in bpf_local_storage_lookup()  argument
255 	sdata = rcu_dereference_check(local_storage->cache[smap->cache_idx],  in bpf_local_storage_lookup()
261 	hlist_for_each_entry_rcu(selem, &local_storage->list, snode,  in bpf_local_storage_lookup()
278 		raw_spin_lock_irqsave(&local_storage->lock, flags);  in bpf_local_storage_lookup()
280 			rcu_assign_pointer(local_storage->cache[smap->cache_idx],  in bpf_local_storage_lookup()
282 		raw_spin_unlock_irqrestore(&local_storage->lock, flags);  in bpf_local_storage_lookup()
377 	struct bpf_local_storage *local_storage;  in bpf_local_storage_update()  local
391 	local_storage = rcu_dereference_check(*owner_storage(smap, owner),  in bpf_local_storage_update()
393 	if (!local_storage || hlist_empty(&local_storage->list)) {  in bpf_local_storage_update()
419 			bpf_local_storage_lookup(local_storage, smap, false);  in bpf_local_storage_update()
436 	raw_spin_lock_irqsave(&local_storage->lock, flags);  in bpf_local_storage_update()
439 	if (unlikely(hlist_empty(&local_storage->list))) {  in bpf_local_storage_update()
449 	old_sdata = bpf_local_storage_lookup(local_storage, smap, false);  in bpf_local_storage_update()
482 	bpf_selem_link_storage_nolock(local_storage, selem);  in bpf_local_storage_update()
487 		bpf_selem_unlink_storage_nolock(local_storage, SELEM(old_sdata),  in bpf_local_storage_update()
492 	raw_spin_unlock_irqrestore(&local_storage->lock, flags);  in bpf_local_storage_update()
496 	raw_spin_unlock_irqrestore(&local_storage->lock, flags);  in bpf_local_storage_update()