22 static void bpf_array_free_percpu(struct bpf_array *array)  in bpf_array_free_percpu()  argument
26 	for (i = 0; i < array->map.max_entries; i++) {  in bpf_array_free_percpu()
27 		free_percpu(array->pptrs[i]);  in bpf_array_free_percpu()
32 static int bpf_array_alloc_percpu(struct bpf_array *array)  in bpf_array_alloc_percpu()  argument
37 	for (i = 0; i < array->map.max_entries; i++) {  in bpf_array_alloc_percpu()
38 		ptr = __alloc_percpu_gfp(array->elem_size, 8,  in bpf_array_alloc_percpu()
41 			bpf_array_free_percpu(array);  in bpf_array_alloc_percpu()
44 		array->pptrs[i] = ptr;  in bpf_array_alloc_percpu()
58 	struct bpf_array *array;  in array_map_alloc()  local
96 	array_size = sizeof(*array);  in array_map_alloc()
118 	array = bpf_map_area_alloc(array_size);  in array_map_alloc()
119 	if (!array)  in array_map_alloc()
121 	array->index_mask = index_mask;  in array_map_alloc()
122 	array->map.unpriv_array = unpriv;  in array_map_alloc()
125 	array->map.map_type = attr->map_type;  in array_map_alloc()
126 	array->map.key_size = attr->key_size;  in array_map_alloc()
127 	array->map.value_size = attr->value_size;  in array_map_alloc()
128 	array->map.max_entries = attr->max_entries;  in array_map_alloc()
129 	array->map.map_flags = attr->map_flags;  in array_map_alloc()
130 	array->map.pages = cost;  in array_map_alloc()
131 	array->elem_size = elem_size;  in array_map_alloc()
135 	     bpf_array_alloc_percpu(array))) {  in array_map_alloc()
136 		bpf_map_area_free(array);  in array_map_alloc()
140 	return &array->map;  in array_map_alloc()
146 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in array_map_lookup_elem()  local
149 	if (unlikely(index >= array->map.max_entries))  in array_map_lookup_elem()
152 	return array->value + array->elem_size * (index & array->index_mask);  in array_map_lookup_elem()
158 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in percpu_array_map_lookup_elem()  local
161 	if (unlikely(index >= array->map.max_entries))  in percpu_array_map_lookup_elem()
164 	return this_cpu_ptr(array->pptrs[index & array->index_mask]);  in percpu_array_map_lookup_elem()
169 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in bpf_percpu_array_copy()  local
175 	if (unlikely(index >= array->map.max_entries))  in bpf_percpu_array_copy()
184 	pptr = array->pptrs[index & array->index_mask];  in bpf_percpu_array_copy()
196 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in array_map_get_next_key()  local
200 	if (index >= array->map.max_entries) {  in array_map_get_next_key()
205 	if (index == array->map.max_entries - 1)  in array_map_get_next_key()
216 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in array_map_update_elem()  local
223 	if (unlikely(index >= array->map.max_entries))  in array_map_update_elem()
231 	if (array->map.map_type == BPF_MAP_TYPE_PERCPU_ARRAY)  in array_map_update_elem()
232 		memcpy(this_cpu_ptr(array->pptrs[index & array->index_mask]),  in array_map_update_elem()
235 		memcpy(array->value +  in array_map_update_elem()
236 		       array->elem_size * (index & array->index_mask),  in array_map_update_elem()
244 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in bpf_percpu_array_update()  local
254 	if (unlikely(index >= array->map.max_entries))  in bpf_percpu_array_update()
270 	pptr = array->pptrs[index & array->index_mask];  in bpf_percpu_array_update()
288 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in array_map_free()  local
297 	if (array->map.map_type == BPF_MAP_TYPE_PERCPU_ARRAY)  in array_map_free()
298 		bpf_array_free_percpu(array);  in array_map_free()
300 	bpf_map_area_free(array);  in array_map_free()
349 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in fd_array_map_free()  local
355 	for (i = 0; i < array->map.max_entries; i++)  in fd_array_map_free()
356 		BUG_ON(array->ptrs[i] != NULL);  in fd_array_map_free()
358 	bpf_map_area_free(array);  in fd_array_map_free()
370 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in bpf_fd_array_map_update_elem()  local
377 	if (index >= array->map.max_entries)  in bpf_fd_array_map_update_elem()
385 	old_ptr = xchg(array->ptrs + index, new_ptr);  in bpf_fd_array_map_update_elem()
394 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in fd_array_map_delete_elem()  local
398 	if (index >= array->map.max_entries)  in fd_array_map_delete_elem()
401 	old_ptr = xchg(array->ptrs + index, NULL);  in fd_array_map_delete_elem()
413 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in prog_fd_array_get_ptr()  local
419 	if (!bpf_prog_array_compatible(array, prog)) {  in prog_fd_array_get_ptr()
435 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in bpf_fd_array_map_clear()  local
438 	for (i = 0; i < array->map.max_entries; i++)  in bpf_fd_array_map_clear()
541 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in perf_event_fd_array_release()  local
546 	for (i = 0; i < array->map.max_entries; i++) {  in perf_event_fd_array_release()
547 		ee = READ_ONCE(array->ptrs[i]);  in perf_event_fd_array_release()