60 		struct bpf_cgroup_storage *storage;  in cgroup_storage_lookup()  local
62 		storage = container_of(node, struct bpf_cgroup_storage, node);  in cgroup_storage_lookup()
64 		switch (bpf_cgroup_storage_key_cmp(key, &storage->key)) {  in cgroup_storage_lookup()
74 			return storage;  in cgroup_storage_lookup()
85 				 struct bpf_cgroup_storage *storage)  in cgroup_storage_insert()  argument
96 		switch (bpf_cgroup_storage_key_cmp(&storage->key, &this->key)) {  in cgroup_storage_insert()
108 	rb_link_node(&storage->node, parent, new);  in cgroup_storage_insert()
109 	rb_insert_color(&storage->node, root);  in cgroup_storage_insert()
118 	struct bpf_cgroup_storage *storage;  in cgroup_storage_lookup_elem()  local
120 	storage = cgroup_storage_lookup(map, key, false);  in cgroup_storage_lookup_elem()
121 	if (!storage)  in cgroup_storage_lookup_elem()
124 	return &READ_ONCE(storage->buf)->data[0];  in cgroup_storage_lookup_elem()
131 	struct bpf_cgroup_storage *storage;  in cgroup_storage_update_elem()  local
144 	storage = cgroup_storage_lookup((struct bpf_cgroup_storage_map *)map,  in cgroup_storage_update_elem()
146 	if (!storage)  in cgroup_storage_update_elem()
150 		copy_map_value_locked(map, storage->buf->data, value, false);  in cgroup_storage_update_elem()
164 	new = xchg(&storage->buf, new);  in cgroup_storage_update_elem()
175 	struct bpf_cgroup_storage *storage;  in bpf_percpu_cgroup_storage_copy()  local
180 	storage = cgroup_storage_lookup(map, key, false);  in bpf_percpu_cgroup_storage_copy()
181 	if (!storage) {  in bpf_percpu_cgroup_storage_copy()
193 				per_cpu_ptr(storage->percpu_buf, cpu), size);  in bpf_percpu_cgroup_storage_copy()
205 	struct bpf_cgroup_storage *storage;  in bpf_percpu_cgroup_storage_update()  local
213 	storage = cgroup_storage_lookup(map, key, false);  in bpf_percpu_cgroup_storage_update()
214 	if (!storage) {  in bpf_percpu_cgroup_storage_update()
227 		bpf_long_memcpy(per_cpu_ptr(storage->percpu_buf, cpu),  in bpf_percpu_cgroup_storage_update()
241 	struct bpf_cgroup_storage *storage;  in cgroup_storage_get_next_key()  local
249 		storage = cgroup_storage_lookup(map, key, true);  in cgroup_storage_get_next_key()
250 		if (!storage)  in cgroup_storage_get_next_key()
253 		storage = list_next_entry(storage, list);  in cgroup_storage_get_next_key()
254 		if (!storage)  in cgroup_storage_get_next_key()
257 		storage = list_first_entry(&map->list,  in cgroup_storage_get_next_key()
262 	next->attach_type = storage->key.attach_type;  in cgroup_storage_get_next_key()
263 	next->cgroup_inode_id = storage->key.cgroup_inode_id;  in cgroup_storage_get_next_key()
381 	struct bpf_cgroup_storage *storage;  in cgroup_storage_seq_show_elem()  local
385 	storage = cgroup_storage_lookup(map_to_storage(map), key, false);  in cgroup_storage_seq_show_elem()
386 	if (!storage) {  in cgroup_storage_seq_show_elem()
396 				  &READ_ONCE(storage->buf)->data[0], m);  in cgroup_storage_seq_show_elem()
403 					  per_cpu_ptr(storage->percpu_buf, cpu),  in cgroup_storage_seq_show_elem()
480 	struct bpf_cgroup_storage *storage;  in bpf_cgroup_storage_alloc()  local
495 	storage = kmalloc_node(sizeof(struct bpf_cgroup_storage),  in bpf_cgroup_storage_alloc()
497 	if (!storage)  in bpf_cgroup_storage_alloc()
503 		storage->buf = kmalloc_node(size, flags, map->numa_node);  in bpf_cgroup_storage_alloc()
504 		if (!storage->buf)  in bpf_cgroup_storage_alloc()
506 		check_and_init_map_lock(map, storage->buf->data);  in bpf_cgroup_storage_alloc()
508 		storage->percpu_buf = __alloc_percpu_gfp(size, 8, flags);  in bpf_cgroup_storage_alloc()
509 		if (!storage->percpu_buf)  in bpf_cgroup_storage_alloc()
513 	storage->map = (struct bpf_cgroup_storage_map *)map;  in bpf_cgroup_storage_alloc()
515 	return storage;  in bpf_cgroup_storage_alloc()
519 	kfree(storage);  in bpf_cgroup_storage_alloc()
525 	struct bpf_cgroup_storage *storage =  in free_shared_cgroup_storage_rcu()  local
528 	kfree(storage->buf);  in free_shared_cgroup_storage_rcu()
529 	kfree(storage);  in free_shared_cgroup_storage_rcu()
534 	struct bpf_cgroup_storage *storage =  in free_percpu_cgroup_storage_rcu()  local
537 	free_percpu(storage->percpu_buf);  in free_percpu_cgroup_storage_rcu()
538 	kfree(storage);  in free_percpu_cgroup_storage_rcu()
541 void bpf_cgroup_storage_free(struct bpf_cgroup_storage *storage)  in bpf_cgroup_storage_free()  argument
547 	if (!storage)  in bpf_cgroup_storage_free()
550 	map = &storage->map->map;  in bpf_cgroup_storage_free()
557 		call_rcu(&storage->rcu, free_shared_cgroup_storage_rcu);  in bpf_cgroup_storage_free()
559 		call_rcu(&storage->rcu, free_percpu_cgroup_storage_rcu);  in bpf_cgroup_storage_free()
562 void bpf_cgroup_storage_link(struct bpf_cgroup_storage *storage,  in bpf_cgroup_storage_link()  argument
568 	if (!storage)  in bpf_cgroup_storage_link()
571 	storage->key.attach_type = type;  in bpf_cgroup_storage_link()
572 	storage->key.cgroup_inode_id = cgroup->kn->id.id;  in bpf_cgroup_storage_link()
574 	map = storage->map;  in bpf_cgroup_storage_link()
577 	WARN_ON(cgroup_storage_insert(map, storage));  in bpf_cgroup_storage_link()
578 	list_add(&storage->list, &map->list);  in bpf_cgroup_storage_link()
582 void bpf_cgroup_storage_unlink(struct bpf_cgroup_storage *storage)  in bpf_cgroup_storage_unlink()  argument
587 	if (!storage)  in bpf_cgroup_storage_unlink()
590 	map = storage->map;  in bpf_cgroup_storage_unlink()
594 	rb_erase(&storage->node, root);  in bpf_cgroup_storage_unlink()
596 	list_del(&storage->list);  in bpf_cgroup_storage_unlink()