Lines Matching full:storage
76 struct bpf_cgroup_storage *storage; in cgroup_storage_lookup() local
78 storage = container_of(node, struct bpf_cgroup_storage, node); in cgroup_storage_lookup()
80 switch (bpf_cgroup_storage_key_cmp(map, key, &storage->key)) { in cgroup_storage_lookup()
90 return storage; in cgroup_storage_lookup()
101 struct bpf_cgroup_storage *storage) in cgroup_storage_insert() argument
112 switch (bpf_cgroup_storage_key_cmp(map, &storage->key, &this->key)) { in cgroup_storage_insert()
124 rb_link_node(&storage->node, parent, new); in cgroup_storage_insert()
125 rb_insert_color(&storage->node, root); in cgroup_storage_insert()
133 struct bpf_cgroup_storage *storage; in cgroup_storage_lookup_elem() local
135 storage = cgroup_storage_lookup(map, key, false); in cgroup_storage_lookup_elem()
136 if (!storage) in cgroup_storage_lookup_elem()
139 return &READ_ONCE(storage->buf)->data[0]; in cgroup_storage_lookup_elem()
145 struct bpf_cgroup_storage *storage; in cgroup_storage_update_elem() local
155 storage = cgroup_storage_lookup((struct bpf_cgroup_storage_map *)map, in cgroup_storage_update_elem()
157 if (!storage) in cgroup_storage_update_elem()
161 copy_map_value_locked(map, storage->buf->data, value, false); in cgroup_storage_update_elem()
175 new = xchg(&storage->buf, new); in cgroup_storage_update_elem()
185 struct bpf_cgroup_storage *storage; in bpf_percpu_cgroup_storage_copy() local
190 storage = cgroup_storage_lookup(map, key, false); in bpf_percpu_cgroup_storage_copy()
191 if (!storage) { in bpf_percpu_cgroup_storage_copy()
203 per_cpu_ptr(storage->percpu_buf, cpu), size); in bpf_percpu_cgroup_storage_copy()
214 struct bpf_cgroup_storage *storage; in bpf_percpu_cgroup_storage_update() local
222 storage = cgroup_storage_lookup(map, key, false); in bpf_percpu_cgroup_storage_update()
223 if (!storage) { in bpf_percpu_cgroup_storage_update()
236 bpf_long_memcpy(per_cpu_ptr(storage->percpu_buf, cpu), in bpf_percpu_cgroup_storage_update()
248 struct bpf_cgroup_storage *storage; in cgroup_storage_get_next_key() local
256 storage = cgroup_storage_lookup(map, key, true); in cgroup_storage_get_next_key()
257 if (!storage) in cgroup_storage_get_next_key()
260 storage = list_next_entry(storage, list_map); in cgroup_storage_get_next_key()
261 if (!storage) in cgroup_storage_get_next_key()
264 storage = list_first_entry(&map->list, in cgroup_storage_get_next_key()
272 *next = storage->key; in cgroup_storage_get_next_key()
275 *next = storage->key.cgroup_inode_id; in cgroup_storage_get_next_key()
336 struct bpf_cgroup_storage *storage, *stmp; in cgroup_storage_map_free() local
340 list_for_each_entry_safe(storage, stmp, storages, list_map) { in cgroup_storage_map_free()
341 bpf_cgroup_storage_unlink(storage); in cgroup_storage_map_free()
342 bpf_cgroup_storage_free(storage); in cgroup_storage_map_free()
420 struct bpf_cgroup_storage *storage; in cgroup_storage_seq_show_elem() local
424 storage = cgroup_storage_lookup(map_to_storage(map), key, false); in cgroup_storage_seq_show_elem()
425 if (!storage) { in cgroup_storage_seq_show_elem()
435 &READ_ONCE(storage->buf)->data[0], m); in cgroup_storage_seq_show_elem()
442 per_cpu_ptr(storage->percpu_buf, cpu), in cgroup_storage_seq_show_elem()
497 struct bpf_cgroup_storage *storage; in bpf_cgroup_storage_alloc() local
512 storage = kmalloc_node(sizeof(struct bpf_cgroup_storage), in bpf_cgroup_storage_alloc()
514 if (!storage) in bpf_cgroup_storage_alloc()
520 storage->buf = kmalloc_node(size, flags, map->numa_node); in bpf_cgroup_storage_alloc()
521 if (!storage->buf) in bpf_cgroup_storage_alloc()
523 check_and_init_map_lock(map, storage->buf->data); in bpf_cgroup_storage_alloc()
525 storage->percpu_buf = __alloc_percpu_gfp(size, 8, flags); in bpf_cgroup_storage_alloc()
526 if (!storage->percpu_buf) in bpf_cgroup_storage_alloc()
530 storage->map = (struct bpf_cgroup_storage_map *)map; in bpf_cgroup_storage_alloc()
532 return storage; in bpf_cgroup_storage_alloc()
536 kfree(storage); in bpf_cgroup_storage_alloc()
542 struct bpf_cgroup_storage *storage = in free_shared_cgroup_storage_rcu() local
545 kfree(storage->buf); in free_shared_cgroup_storage_rcu()
546 kfree(storage); in free_shared_cgroup_storage_rcu()
551 struct bpf_cgroup_storage *storage = in free_percpu_cgroup_storage_rcu() local
554 free_percpu(storage->percpu_buf); in free_percpu_cgroup_storage_rcu()
555 kfree(storage); in free_percpu_cgroup_storage_rcu()
558 void bpf_cgroup_storage_free(struct bpf_cgroup_storage *storage) in bpf_cgroup_storage_free() argument
564 if (!storage) in bpf_cgroup_storage_free()
567 map = &storage->map->map; in bpf_cgroup_storage_free()
574 call_rcu(&storage->rcu, free_shared_cgroup_storage_rcu); in bpf_cgroup_storage_free()
576 call_rcu(&storage->rcu, free_percpu_cgroup_storage_rcu); in bpf_cgroup_storage_free()
579 void bpf_cgroup_storage_link(struct bpf_cgroup_storage *storage, in bpf_cgroup_storage_link() argument
585 if (!storage) in bpf_cgroup_storage_link()
588 storage->key.attach_type = type; in bpf_cgroup_storage_link()
589 storage->key.cgroup_inode_id = cgroup_id(cgroup); in bpf_cgroup_storage_link()
591 map = storage->map; in bpf_cgroup_storage_link()
594 WARN_ON(cgroup_storage_insert(map, storage)); in bpf_cgroup_storage_link()
595 list_add(&storage->list_map, &map->list); in bpf_cgroup_storage_link()
596 list_add(&storage->list_cg, &cgroup->bpf.storages); in bpf_cgroup_storage_link()
600 void bpf_cgroup_storage_unlink(struct bpf_cgroup_storage *storage) in bpf_cgroup_storage_unlink() argument
605 if (!storage) in bpf_cgroup_storage_unlink()
608 map = storage->map; in bpf_cgroup_storage_unlink()
612 rb_erase(&storage->node, root); in bpf_cgroup_storage_unlink()
614 list_del(&storage->list_map); in bpf_cgroup_storage_unlink()
615 list_del(&storage->list_cg); in bpf_cgroup_storage_unlink()