23 	for (i = 0; i < array->map.max_entries; i++) {  in bpf_array_free_percpu()
34 	for (i = 0; i < array->map.max_entries; i++) {  in bpf_array_alloc_percpu()
126 	array->map.unpriv_array = unpriv;  in array_map_alloc()
129 	bpf_map_init_from_attr(&array->map, attr);  in array_map_alloc()
130 	bpf_map_charge_move(&array->map.memory, &mem);  in array_map_alloc()
134 		bpf_map_charge_finish(&array->map.memory);  in array_map_alloc()
139 	return &array->map;  in array_map_alloc()
143 static void *array_map_lookup_elem(struct bpf_map *map, void *key)  in array_map_lookup_elem()  argument
145 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in array_map_lookup_elem()
148 	if (unlikely(index >= array->map.max_entries))  in array_map_lookup_elem()
154 static int array_map_direct_value_addr(const struct bpf_map *map, u64 *imm,  in array_map_direct_value_addr()  argument
157 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in array_map_direct_value_addr()
159 	if (map->max_entries != 1)  in array_map_direct_value_addr()
161 	if (off >= map->value_size)  in array_map_direct_value_addr()
168 static int array_map_direct_value_meta(const struct bpf_map *map, u64 imm,  in array_map_direct_value_meta()  argument
171 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in array_map_direct_value_meta()
175 	if (map->max_entries != 1)  in array_map_direct_value_meta()
185 static u32 array_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf)  in array_map_gen_lookup()  argument
187 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in array_map_gen_lookup()
189 	u32 elem_size = round_up(map->value_size, 8);  in array_map_gen_lookup()
196 	if (map->unpriv_array) {  in array_map_gen_lookup()
197 		*insn++ = BPF_JMP_IMM(BPF_JGE, ret, map->max_entries, 4);  in array_map_gen_lookup()
200 		*insn++ = BPF_JMP_IMM(BPF_JGE, ret, map->max_entries, 3);  in array_map_gen_lookup()
215 static void *percpu_array_map_lookup_elem(struct bpf_map *map, void *key)  in percpu_array_map_lookup_elem()  argument
217 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in percpu_array_map_lookup_elem()
220 	if (unlikely(index >= array->map.max_entries))  in percpu_array_map_lookup_elem()
226 int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value)  in bpf_percpu_array_copy()  argument
228 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in bpf_percpu_array_copy()
234 	if (unlikely(index >= array->map.max_entries))  in bpf_percpu_array_copy()
241 	size = round_up(map->value_size, 8);  in bpf_percpu_array_copy()
253 static int array_map_get_next_key(struct bpf_map *map, void *key, void *next_key)  in array_map_get_next_key()  argument
255 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in array_map_get_next_key()
259 	if (index >= array->map.max_entries) {  in array_map_get_next_key()
264 	if (index == array->map.max_entries - 1)  in array_map_get_next_key()
272 static int array_map_update_elem(struct bpf_map *map, void *key, void *value,  in array_map_update_elem()  argument
275 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in array_map_update_elem()
283 	if (unlikely(index >= array->map.max_entries))  in array_map_update_elem()
292 		     !map_value_has_spin_lock(map)))  in array_map_update_elem()
295 	if (array->map.map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {  in array_map_update_elem()
297 		       value, map->value_size);  in array_map_update_elem()
302 			copy_map_value_locked(map, val, value, false);  in array_map_update_elem()
304 			copy_map_value(map, val, value);  in array_map_update_elem()
309 int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value,  in bpf_percpu_array_update()  argument
312 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in bpf_percpu_array_update()
322 	if (unlikely(index >= array->map.max_entries))  in bpf_percpu_array_update()
336 	size = round_up(map->value_size, 8);  in bpf_percpu_array_update()
348 static int array_map_delete_elem(struct bpf_map *map, void *key)  in array_map_delete_elem()  argument
354 static void array_map_free(struct bpf_map *map)  in array_map_free()  argument
356 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in array_map_free()
365 	if (array->map.map_type == BPF_MAP_TYPE_PERCPU_ARRAY)  in array_map_free()
371 static void array_map_seq_show_elem(struct bpf_map *map, void *key,  in array_map_seq_show_elem()  argument
378 	value = array_map_lookup_elem(map, key);  in array_map_seq_show_elem()
384 	if (map->btf_key_type_id)  in array_map_seq_show_elem()
386 	btf_type_seq_show(map->btf, map->btf_value_type_id, value, m);  in array_map_seq_show_elem()
392 static void percpu_array_map_seq_show_elem(struct bpf_map *map, void *key,  in percpu_array_map_seq_show_elem()  argument
395 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in percpu_array_map_seq_show_elem()
406 		btf_type_seq_show(map->btf, map->btf_value_type_id,  in percpu_array_map_seq_show_elem()
415 static int array_map_check_btf(const struct bpf_map *map,  in array_map_check_btf()  argument
424 		if (map->map_type != BPF_MAP_TYPE_ARRAY ||  in array_map_check_btf()
425 		    map->max_entries != 1)  in array_map_check_btf()
485 static void fd_array_map_free(struct bpf_map *map)  in fd_array_map_free()  argument
487 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in fd_array_map_free()
493 	for (i = 0; i < array->map.max_entries; i++)  in fd_array_map_free()
499 static void *fd_array_map_lookup_elem(struct bpf_map *map, void *key)  in fd_array_map_lookup_elem()  argument
505 int bpf_fd_array_map_lookup_elem(struct bpf_map *map, void *key, u32 *value)  in bpf_fd_array_map_lookup_elem()  argument
510 	if (!map->ops->map_fd_sys_lookup_elem)  in bpf_fd_array_map_lookup_elem()
514 	elem = array_map_lookup_elem(map, key);  in bpf_fd_array_map_lookup_elem()
516 		*value = map->ops->map_fd_sys_lookup_elem(ptr);  in bpf_fd_array_map_lookup_elem()
525 int bpf_fd_array_map_update_elem(struct bpf_map *map, struct file *map_file,  in bpf_fd_array_map_update_elem()  argument
528 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in bpf_fd_array_map_update_elem()
535 	if (index >= array->map.max_entries)  in bpf_fd_array_map_update_elem()
539 	new_ptr = map->ops->map_fd_get_ptr(map, map_file, ufd);  in bpf_fd_array_map_update_elem()
545 		map->ops->map_fd_put_ptr(old_ptr);  in bpf_fd_array_map_update_elem()
550 static int fd_array_map_delete_elem(struct bpf_map *map, void *key)  in fd_array_map_delete_elem()  argument
552 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in fd_array_map_delete_elem()
556 	if (index >= array->map.max_entries)  in fd_array_map_delete_elem()
561 		map->ops->map_fd_put_ptr(old_ptr);  in fd_array_map_delete_elem()
568 static void *prog_fd_array_get_ptr(struct bpf_map *map,  in prog_fd_array_get_ptr()  argument
571 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in prog_fd_array_get_ptr()
596 static void bpf_fd_array_map_clear(struct bpf_map *map)  in bpf_fd_array_map_clear()  argument
598 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in bpf_fd_array_map_clear()
601 	for (i = 0; i < array->map.max_entries; i++)  in bpf_fd_array_map_clear()
602 		fd_array_map_delete_elem(map, &i);  in bpf_fd_array_map_clear()
605 static void prog_array_map_seq_show_elem(struct bpf_map *map, void *key,  in prog_array_map_seq_show_elem()  argument
613 	elem = array_map_lookup_elem(map, key);  in prog_array_map_seq_show_elem()
619 			btf_type_seq_show(map->btf, map->btf_value_type_id,  in prog_array_map_seq_show_elem()
671 static void *perf_event_fd_array_get_ptr(struct bpf_map *map,  in perf_event_fd_array_get_ptr()  argument
702 static void perf_event_fd_array_release(struct bpf_map *map,  in perf_event_fd_array_release()  argument
705 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in perf_event_fd_array_release()
710 	for (i = 0; i < array->map.max_entries; i++) {  in perf_event_fd_array_release()
713 			fd_array_map_delete_elem(map, &i);  in perf_event_fd_array_release()
732 static void *cgroup_fd_array_get_ptr(struct bpf_map *map,  in cgroup_fd_array_get_ptr()  argument
745 static void cgroup_fd_array_free(struct bpf_map *map)  in cgroup_fd_array_free()  argument
747 	bpf_fd_array_map_clear(map);  in cgroup_fd_array_free()
748 	fd_array_map_free(map);  in cgroup_fd_array_free()
766 	struct bpf_map *map, *inner_map_meta;  in array_of_map_alloc()  local
772 	map = array_map_alloc(attr);  in array_of_map_alloc()
773 	if (IS_ERR(map)) {  in array_of_map_alloc()
775 		return map;  in array_of_map_alloc()
778 	map->inner_map_meta = inner_map_meta;  in array_of_map_alloc()
780 	return map;  in array_of_map_alloc()
783 static void array_of_map_free(struct bpf_map *map)  in array_of_map_free()  argument
788 	bpf_map_meta_free(map->inner_map_meta);  in array_of_map_free()
789 	bpf_fd_array_map_clear(map);  in array_of_map_free()
790 	fd_array_map_free(map);  in array_of_map_free()
793 static void *array_of_map_lookup_elem(struct bpf_map *map, void *key)  in array_of_map_lookup_elem()  argument
795 	struct bpf_map **inner_map = array_map_lookup_elem(map, key);  in array_of_map_lookup_elem()
803 static u32 array_of_map_gen_lookup(struct bpf_map *map,  in array_of_map_gen_lookup()  argument
806 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in array_of_map_gen_lookup()
807 	u32 elem_size = round_up(map->value_size, 8);  in array_of_map_gen_lookup()
815 	if (map->unpriv_array) {  in array_of_map_gen_lookup()
816 		*insn++ = BPF_JMP_IMM(BPF_JGE, ret, map->max_entries, 6);  in array_of_map_gen_lookup()
819 		*insn++ = BPF_JMP_IMM(BPF_JGE, ret, map->max_entries, 5);  in array_of_map_gen_lookup()