25 	for (i = 0; i < array->map.max_entries; i++) {  in bpf_array_free_percpu()
36 	for (i = 0; i < array->map.max_entries; i++) {  in bpf_array_alloc_percpu()
37 		ptr = bpf_map_alloc_percpu(&array->map, array->elem_size, 8,  in bpf_array_alloc_percpu()
144 	array->map.bypass_spec_v1 = bypass_spec_v1;  in array_map_alloc()
147 	bpf_map_init_from_attr(&array->map, attr);  in array_map_alloc()
155 	return &array->map;  in array_map_alloc()
159 static void *array_map_lookup_elem(struct bpf_map *map, void *key)  in array_map_lookup_elem()  argument
161 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in array_map_lookup_elem()
164 	if (unlikely(index >= array->map.max_entries))  in array_map_lookup_elem()
170 static int array_map_direct_value_addr(const struct bpf_map *map, u64 *imm,  in array_map_direct_value_addr()  argument
173 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in array_map_direct_value_addr()
175 	if (map->max_entries != 1)  in array_map_direct_value_addr()
177 	if (off >= map->value_size)  in array_map_direct_value_addr()
184 static int array_map_direct_value_meta(const struct bpf_map *map, u64 imm,  in array_map_direct_value_meta()  argument
187 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in array_map_direct_value_meta()
191 	if (map->max_entries != 1)  in array_map_direct_value_meta()
201 static int array_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf)  in array_map_gen_lookup()  argument
203 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in array_map_gen_lookup()
205 	u32 elem_size = round_up(map->value_size, 8);  in array_map_gen_lookup()
210 	if (map->map_flags & BPF_F_INNER_MAP)  in array_map_gen_lookup()
215 	if (!map->bypass_spec_v1) {  in array_map_gen_lookup()
216 		*insn++ = BPF_JMP_IMM(BPF_JGE, ret, map->max_entries, 4);  in array_map_gen_lookup()
219 		*insn++ = BPF_JMP_IMM(BPF_JGE, ret, map->max_entries, 3);  in array_map_gen_lookup()
234 static void *percpu_array_map_lookup_elem(struct bpf_map *map, void *key)  in percpu_array_map_lookup_elem()  argument
236 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in percpu_array_map_lookup_elem()
239 	if (unlikely(index >= array->map.max_entries))  in percpu_array_map_lookup_elem()
245 int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value)  in bpf_percpu_array_copy()  argument
247 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in bpf_percpu_array_copy()
253 	if (unlikely(index >= array->map.max_entries))  in bpf_percpu_array_copy()
260 	size = round_up(map->value_size, 8);  in bpf_percpu_array_copy()
272 static int array_map_get_next_key(struct bpf_map *map, void *key, void *next_key)  in array_map_get_next_key()  argument
274 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in array_map_get_next_key()
278 	if (index >= array->map.max_entries) {  in array_map_get_next_key()
283 	if (index == array->map.max_entries - 1)  in array_map_get_next_key()
292 	if (unlikely(map_value_has_timer(&arr->map)))  in check_and_free_timer_in_array()
293 		bpf_timer_cancel_and_free(val + arr->map.timer_off);  in check_and_free_timer_in_array()
297 static int array_map_update_elem(struct bpf_map *map, void *key, void *value,  in array_map_update_elem()  argument
300 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in array_map_update_elem()
308 	if (unlikely(index >= array->map.max_entries))  in array_map_update_elem()
317 		     !map_value_has_spin_lock(map)))  in array_map_update_elem()
320 	if (array->map.map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {  in array_map_update_elem()
322 		       value, map->value_size);  in array_map_update_elem()
327 			copy_map_value_locked(map, val, value, false);  in array_map_update_elem()
329 			copy_map_value(map, val, value);  in array_map_update_elem()
335 int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value,  in bpf_percpu_array_update()  argument
338 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in bpf_percpu_array_update()
348 	if (unlikely(index >= array->map.max_entries))  in bpf_percpu_array_update()
362 	size = round_up(map->value_size, 8);  in bpf_percpu_array_update()
374 static int array_map_delete_elem(struct bpf_map *map, void *key)  in array_map_delete_elem()  argument
384 static void array_map_free_timers(struct bpf_map *map)  in array_map_free_timers()  argument
386 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in array_map_free_timers()
389 	if (likely(!map_value_has_timer(map)))  in array_map_free_timers()
392 	for (i = 0; i < array->map.max_entries; i++)  in array_map_free_timers()
394 					  map->timer_off);  in array_map_free_timers()
398 static void array_map_free(struct bpf_map *map)  in array_map_free()  argument
400 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in array_map_free()
402 	if (array->map.map_type == BPF_MAP_TYPE_PERCPU_ARRAY)  in array_map_free()
405 	if (array->map.map_flags & BPF_F_MMAPABLE)  in array_map_free()
411 static void array_map_seq_show_elem(struct bpf_map *map, void *key,  in array_map_seq_show_elem()  argument
418 	value = array_map_lookup_elem(map, key);  in array_map_seq_show_elem()
424 	if (map->btf_key_type_id)  in array_map_seq_show_elem()
426 	btf_type_seq_show(map->btf, map->btf_value_type_id, value, m);  in array_map_seq_show_elem()
432 static void percpu_array_map_seq_show_elem(struct bpf_map *map, void *key,  in percpu_array_map_seq_show_elem()  argument
435 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in percpu_array_map_seq_show_elem()
446 		btf_type_seq_show(map->btf, map->btf_value_type_id,  in percpu_array_map_seq_show_elem()
455 static int array_map_check_btf(const struct bpf_map *map,  in array_map_check_btf()  argument
464 		if (map->map_type != BPF_MAP_TYPE_ARRAY ||  in array_map_check_btf()
465 		    map->max_entries != 1)  in array_map_check_btf()
487 static int array_map_mmap(struct bpf_map *map, struct vm_area_struct *vma)  in array_map_mmap()  argument
489 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in array_map_mmap()
492 	if (!(map->map_flags & BPF_F_MMAPABLE))  in array_map_mmap()
496 	    PAGE_ALIGN((u64)array->map.max_entries * array->elem_size))  in array_map_mmap()
513 	struct bpf_map *map;  member
521 	struct bpf_map *map = info->map;  in bpf_array_map_seq_start()  local
525 	if (info->index >= map->max_entries)  in bpf_array_map_seq_start()
530 	array = container_of(map, struct bpf_array, map);  in bpf_array_map_seq_start()
540 	struct bpf_map *map = info->map;  in bpf_array_map_seq_next()  local
546 	if (info->index >= map->max_entries)  in bpf_array_map_seq_next()
549 	array = container_of(map, struct bpf_array, map);  in bpf_array_map_seq_next()
560 	struct bpf_map *map = info->map;  in __bpf_array_map_seq_show()  local
573 	ctx.map = info->map;  in __bpf_array_map_seq_show()
581 			size = round_up(map->value_size, 8);  in __bpf_array_map_seq_show()
610 	struct bpf_map *map = aux->map;  in bpf_iter_init_array_map()  local
614 	if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {  in bpf_iter_init_array_map()
615 		buf_size = round_up(map->value_size, 8) * num_possible_cpus();  in bpf_iter_init_array_map()
627 	bpf_map_inc_with_uref(map);  in bpf_iter_init_array_map()
628 	seq_info->map = map;  in bpf_iter_init_array_map()
636 	bpf_map_put_with_uref(seq_info->map);  in bpf_iter_fini_array_map()
654 static int bpf_for_each_array_elem(struct bpf_map *map, void *callback_fn,  in bpf_for_each_array_elem()  argument
666 	is_percpu = map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY;  in bpf_for_each_array_elem()
667 	array = container_of(map, struct bpf_array, map);  in bpf_for_each_array_elem()
670 	for (i = 0; i < map->max_entries; i++) {  in bpf_for_each_array_elem()
677 		ret = BPF_CAST_CALL(callback_fn)((u64)(long)map,  in bpf_for_each_array_elem()
748 static void fd_array_map_free(struct bpf_map *map)  in fd_array_map_free()  argument
750 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in fd_array_map_free()
754 	for (i = 0; i < array->map.max_entries; i++)  in fd_array_map_free()
760 static void *fd_array_map_lookup_elem(struct bpf_map *map, void *key)  in fd_array_map_lookup_elem()  argument
766 int bpf_fd_array_map_lookup_elem(struct bpf_map *map, void *key, u32 *value)  in bpf_fd_array_map_lookup_elem()  argument
771 	if (!map->ops->map_fd_sys_lookup_elem)  in bpf_fd_array_map_lookup_elem()
775 	elem = array_map_lookup_elem(map, key);  in bpf_fd_array_map_lookup_elem()
777 		*value = map->ops->map_fd_sys_lookup_elem(ptr);  in bpf_fd_array_map_lookup_elem()
786 int bpf_fd_array_map_update_elem(struct bpf_map *map, struct file *map_file,  in bpf_fd_array_map_update_elem()  argument
789 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in bpf_fd_array_map_update_elem()
796 	if (index >= array->map.max_entries)  in bpf_fd_array_map_update_elem()
800 	new_ptr = map->ops->map_fd_get_ptr(map, map_file, ufd);  in bpf_fd_array_map_update_elem()
804 	if (map->ops->map_poke_run) {  in bpf_fd_array_map_update_elem()
807 		map->ops->map_poke_run(map, index, old_ptr, new_ptr);  in bpf_fd_array_map_update_elem()
814 		map->ops->map_fd_put_ptr(old_ptr);  in bpf_fd_array_map_update_elem()
818 static int fd_array_map_delete_elem(struct bpf_map *map, void *key)  in fd_array_map_delete_elem()  argument
820 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in fd_array_map_delete_elem()
824 	if (index >= array->map.max_entries)  in fd_array_map_delete_elem()
827 	if (map->ops->map_poke_run) {  in fd_array_map_delete_elem()
830 		map->ops->map_poke_run(map, index, old_ptr, NULL);  in fd_array_map_delete_elem()
837 		map->ops->map_fd_put_ptr(old_ptr);  in fd_array_map_delete_elem()
844 static void *prog_fd_array_get_ptr(struct bpf_map *map,  in prog_fd_array_get_ptr()  argument
847 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in prog_fd_array_get_ptr()
872 static void bpf_fd_array_map_clear(struct bpf_map *map)  in bpf_fd_array_map_clear()  argument
874 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in bpf_fd_array_map_clear()
877 	for (i = 0; i < array->map.max_entries; i++)  in bpf_fd_array_map_clear()
878 		fd_array_map_delete_elem(map, &i);  in bpf_fd_array_map_clear()
881 static void prog_array_map_seq_show_elem(struct bpf_map *map, void *key,  in prog_array_map_seq_show_elem()  argument
889 	elem = array_map_lookup_elem(map, key);  in prog_array_map_seq_show_elem()
895 			btf_type_seq_show(map->btf, map->btf_value_type_id,  in prog_array_map_seq_show_elem()
909 static int prog_array_map_poke_track(struct bpf_map *map,  in prog_array_map_poke_track()  argument
916 	aux = container_of(map, struct bpf_array, map)->aux;  in prog_array_map_poke_track()
942 static void prog_array_map_poke_untrack(struct bpf_map *map,  in prog_array_map_poke_untrack()  argument
948 	aux = container_of(map, struct bpf_array, map)->aux;  in prog_array_map_poke_untrack()
966 static void prog_array_map_poke_run(struct bpf_map *map, u32 key,  in prog_array_map_poke_run()  argument
973 	aux = container_of(map, struct bpf_array, map)->aux;  in prog_array_map_poke_run()
1006 			if (poke->tail_call.map != map ||  in prog_array_map_poke_run()
1017 	struct bpf_map *map = container_of(work, struct bpf_array_aux,  in prog_array_map_clear_deferred()  local
1018 					   work)->map;  in prog_array_map_clear_deferred()
1019 	bpf_fd_array_map_clear(map);  in prog_array_map_clear_deferred()
1020 	bpf_map_put(map);  in prog_array_map_clear_deferred()
1023 static void prog_array_map_clear(struct bpf_map *map)  in prog_array_map_clear()  argument
1025 	struct bpf_array_aux *aux = container_of(map, struct bpf_array,  in prog_array_map_clear()
1026 						 map)->aux;  in prog_array_map_clear()
1027 	bpf_map_inc(map);  in prog_array_map_clear()
1034 	struct bpf_map *map;  in prog_array_map_alloc()  local
1045 	map = array_map_alloc(attr);  in prog_array_map_alloc()
1046 	if (IS_ERR(map)) {  in prog_array_map_alloc()
1048 		return map;  in prog_array_map_alloc()
1051 	container_of(map, struct bpf_array, map)->aux = aux;  in prog_array_map_alloc()
1052 	aux->map = map;  in prog_array_map_alloc()
1054 	return map;  in prog_array_map_alloc()
1057 static void prog_array_map_free(struct bpf_map *map)  in prog_array_map_free()  argument
1062 	aux = container_of(map, struct bpf_array, map)->aux;  in prog_array_map_free()
1068 	fd_array_map_free(map);  in prog_array_map_free()
1125 static void *perf_event_fd_array_get_ptr(struct bpf_map *map,  in perf_event_fd_array_get_ptr()  argument
1156 static void perf_event_fd_array_release(struct bpf_map *map,  in perf_event_fd_array_release()  argument
1159 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in perf_event_fd_array_release()
1163 	if (map->map_flags & BPF_F_PRESERVE_ELEMS)  in perf_event_fd_array_release()
1167 	for (i = 0; i < array->map.max_entries; i++) {  in perf_event_fd_array_release()
1170 			fd_array_map_delete_elem(map, &i);  in perf_event_fd_array_release()
1175 static void perf_event_fd_array_map_free(struct bpf_map *map)  in perf_event_fd_array_map_free()  argument
1177 	if (map->map_flags & BPF_F_PRESERVE_ELEMS)  in perf_event_fd_array_map_free()
1178 		bpf_fd_array_map_clear(map);  in perf_event_fd_array_map_free()
1179 	fd_array_map_free(map);  in perf_event_fd_array_map_free()
1200 static void *cgroup_fd_array_get_ptr(struct bpf_map *map,  in cgroup_fd_array_get_ptr()  argument
1213 static void cgroup_fd_array_free(struct bpf_map *map)  in cgroup_fd_array_free()  argument
1215 	bpf_fd_array_map_clear(map);  in cgroup_fd_array_free()
1216 	fd_array_map_free(map);  in cgroup_fd_array_free()
1238 	struct bpf_map *map, *inner_map_meta;  in array_of_map_alloc()  local
1244 	map = array_map_alloc(attr);  in array_of_map_alloc()
1245 	if (IS_ERR(map)) {  in array_of_map_alloc()
1247 		return map;  in array_of_map_alloc()
1250 	map->inner_map_meta = inner_map_meta;  in array_of_map_alloc()
1252 	return map;  in array_of_map_alloc()
1255 static void array_of_map_free(struct bpf_map *map)  in array_of_map_free()  argument
1260 	bpf_map_meta_free(map->inner_map_meta);  in array_of_map_free()
1261 	bpf_fd_array_map_clear(map);  in array_of_map_free()
1262 	fd_array_map_free(map);  in array_of_map_free()
1265 static void *array_of_map_lookup_elem(struct bpf_map *map, void *key)  in array_of_map_lookup_elem()  argument
1267 	struct bpf_map **inner_map = array_map_lookup_elem(map, key);  in array_of_map_lookup_elem()
1275 static int array_of_map_gen_lookup(struct bpf_map *map,  in array_of_map_gen_lookup()  argument
1278 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in array_of_map_gen_lookup()
1279 	u32 elem_size = round_up(map->value_size, 8);  in array_of_map_gen_lookup()
1287 	if (!map->bypass_spec_v1) {  in array_of_map_gen_lookup()
1288 		*insn++ = BPF_JMP_IMM(BPF_JGE, ret, map->max_entries, 6);  in array_of_map_gen_lookup()
1291 		*insn++ = BPF_JMP_IMM(BPF_JGE, ret, map->max_entries, 5);  in array_of_map_gen_lookup()