19 static void bpf_array_free_percpu(struct bpf_array *array)  in bpf_array_free_percpu()  argument
23 	for (i = 0; i < array->map.max_entries; i++) {  in bpf_array_free_percpu()
24 		free_percpu(array->pptrs[i]);  in bpf_array_free_percpu()
29 static int bpf_array_alloc_percpu(struct bpf_array *array)  in bpf_array_alloc_percpu()  argument
34 	for (i = 0; i < array->map.max_entries; i++) {  in bpf_array_alloc_percpu()
35 		ptr = __alloc_percpu_gfp(array->elem_size, 8,  in bpf_array_alloc_percpu()
38 			bpf_array_free_percpu(array);  in bpf_array_alloc_percpu()
41 		array->pptrs[i] = ptr;  in bpf_array_alloc_percpu()
79 	struct bpf_array *array;  in array_map_alloc()  local
104 	array_size = sizeof(*array);  in array_map_alloc()
120 	array = bpf_map_area_alloc(array_size, numa_node);  in array_map_alloc()
121 	if (!array) {  in array_map_alloc()
125 	array->index_mask = index_mask;  in array_map_alloc()
126 	array->map.unpriv_array = unpriv;  in array_map_alloc()
129 	bpf_map_init_from_attr(&array->map, attr);  in array_map_alloc()
130 	bpf_map_charge_move(&array->map.memory, &mem);  in array_map_alloc()
131 	array->elem_size = elem_size;  in array_map_alloc()
133 	if (percpu && bpf_array_alloc_percpu(array)) {  in array_map_alloc()
134 		bpf_map_charge_finish(&array->map.memory);  in array_map_alloc()
135 		bpf_map_area_free(array);  in array_map_alloc()
139 	return &array->map;  in array_map_alloc()
145 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in array_map_lookup_elem()  local
148 	if (unlikely(index >= array->map.max_entries))  in array_map_lookup_elem()
151 	return array->value + array->elem_size * (index & array->index_mask);  in array_map_lookup_elem()
157 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in array_map_direct_value_addr()  local
164 	*imm = (unsigned long)array->value;  in array_map_direct_value_addr()
171 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in array_map_direct_value_meta()  local
172 	u64 base = (unsigned long)array->value;  in array_map_direct_value_meta()
173 	u64 range = array->elem_size;  in array_map_direct_value_meta()
187 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in array_map_gen_lookup()  local
198 		*insn++ = BPF_ALU32_IMM(BPF_AND, ret, array->index_mask);  in array_map_gen_lookup()
217 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in percpu_array_map_lookup_elem()  local
220 	if (unlikely(index >= array->map.max_entries))  in percpu_array_map_lookup_elem()
223 	return this_cpu_ptr(array->pptrs[index & array->index_mask]);  in percpu_array_map_lookup_elem()
228 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in bpf_percpu_array_copy()  local
234 	if (unlikely(index >= array->map.max_entries))  in bpf_percpu_array_copy()
243 	pptr = array->pptrs[index & array->index_mask];  in bpf_percpu_array_copy()
255 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in array_map_get_next_key()  local
259 	if (index >= array->map.max_entries) {  in array_map_get_next_key()
264 	if (index == array->map.max_entries - 1)  in array_map_get_next_key()
275 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in array_map_update_elem()  local
283 	if (unlikely(index >= array->map.max_entries))  in array_map_update_elem()
295 	if (array->map.map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {  in array_map_update_elem()
296 		memcpy(this_cpu_ptr(array->pptrs[index & array->index_mask]),  in array_map_update_elem()
299 		val = array->value +  in array_map_update_elem()
300 			array->elem_size * (index & array->index_mask);  in array_map_update_elem()
312 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in bpf_percpu_array_update()  local
322 	if (unlikely(index >= array->map.max_entries))  in bpf_percpu_array_update()
338 	pptr = array->pptrs[index & array->index_mask];  in bpf_percpu_array_update()
356 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in array_map_free()  local
365 	if (array->map.map_type == BPF_MAP_TYPE_PERCPU_ARRAY)  in array_map_free()
366 		bpf_array_free_percpu(array);  in array_map_free()
368 	bpf_map_area_free(array);  in array_map_free()
395 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in percpu_array_map_seq_show_elem()  local
403 	pptr = array->pptrs[index & array->index_mask];  in percpu_array_map_seq_show_elem()
487 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in fd_array_map_free()  local
493 	for (i = 0; i < array->map.max_entries; i++)  in fd_array_map_free()
494 		BUG_ON(array->ptrs[i] != NULL);  in fd_array_map_free()
496 	bpf_map_area_free(array);  in fd_array_map_free()
528 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in bpf_fd_array_map_update_elem()  local
535 	if (index >= array->map.max_entries)  in bpf_fd_array_map_update_elem()
543 	old_ptr = xchg(array->ptrs + index, new_ptr);  in bpf_fd_array_map_update_elem()
552 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in fd_array_map_delete_elem()  local
556 	if (index >= array->map.max_entries)  in fd_array_map_delete_elem()
559 	old_ptr = xchg(array->ptrs + index, NULL);  in fd_array_map_delete_elem()
571 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in prog_fd_array_get_ptr()  local
577 	if (!bpf_prog_array_compatible(array, prog)) {  in prog_fd_array_get_ptr()
598 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in bpf_fd_array_map_clear()  local
601 	for (i = 0; i < array->map.max_entries; i++)  in bpf_fd_array_map_clear()
705 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in perf_event_fd_array_release()  local
710 	for (i = 0; i < array->map.max_entries; i++) {  in perf_event_fd_array_release()
711 		ee = READ_ONCE(array->ptrs[i]);  in perf_event_fd_array_release()
806 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in array_of_map_gen_lookup()  local
817 		*insn++ = BPF_ALU32_IMM(BPF_AND, ret, array->index_mask);  in array_of_map_gen_lookup()