60 	struct bpf_cpu_map *cmap;  member
82 	struct bpf_cpu_map *cmap;  in cpu_map_alloc()  local
95 	cmap = kzalloc(sizeof(*cmap), GFP_USER);  in cpu_map_alloc()
96 	if (!cmap)  in cpu_map_alloc()
99 	bpf_map_init_from_attr(&cmap->map, attr);  in cpu_map_alloc()
102 	if (cmap->map.max_entries > NR_CPUS) {  in cpu_map_alloc()
108 	cost = (u64) cmap->map.max_entries * sizeof(struct bpf_cpu_map_entry *);  in cpu_map_alloc()
112 	ret = bpf_map_charge_init(&cmap->map.memory, cost);  in cpu_map_alloc()
118 	cmap->flush_list = alloc_percpu(struct list_head);  in cpu_map_alloc()
119 	if (!cmap->flush_list)  in cpu_map_alloc()
123 		INIT_LIST_HEAD(per_cpu_ptr(cmap->flush_list, cpu));  in cpu_map_alloc()
126 	cmap->cpu_map = bpf_map_area_alloc(cmap->map.max_entries *  in cpu_map_alloc()
128 					   cmap->map.numa_node);  in cpu_map_alloc()
129 	if (!cmap->cpu_map)  in cpu_map_alloc()
132 	return &cmap->map;  in cpu_map_alloc()
134 	free_percpu(cmap->flush_list);  in cpu_map_alloc()
136 	bpf_map_charge_finish(&cmap->map.memory);  in cpu_map_alloc()
138 	kfree(cmap);  in cpu_map_alloc()
445 static void __cpu_map_entry_replace(struct bpf_cpu_map *cmap,  in __cpu_map_entry_replace()  argument
450 	old_rcpu = xchg(&cmap->cpu_map[key_cpu], rcpu);  in __cpu_map_entry_replace()
460 	struct bpf_cpu_map *cmap = container_of(map, struct bpf_cpu_map, map);  in cpu_map_delete_elem()  local
467 	__cpu_map_entry_replace(cmap, key_cpu, NULL);  in cpu_map_delete_elem()
474 	struct bpf_cpu_map *cmap = container_of(map, struct bpf_cpu_map, map);  in cpu_map_update_elem()  local
484 	if (unlikely(key_cpu >= cmap->map.max_entries))  in cpu_map_update_elem()
502 		rcpu->cmap = cmap;  in cpu_map_update_elem()
505 	__cpu_map_entry_replace(cmap, key_cpu, rcpu);  in cpu_map_update_elem()
512 	struct bpf_cpu_map *cmap = container_of(map, struct bpf_cpu_map, map);  in cpu_map_free()  local
534 		struct list_head *flush_list = per_cpu_ptr(cmap->flush_list, cpu);  in cpu_map_free()
543 	for (i = 0; i < cmap->map.max_entries; i++) {  in cpu_map_free()
546 		rcpu = READ_ONCE(cmap->cpu_map[i]);  in cpu_map_free()
551 		__cpu_map_entry_replace(cmap, i, NULL); /* call_rcu */  in cpu_map_free()
553 	free_percpu(cmap->flush_list);  in cpu_map_free()
554 	bpf_map_area_free(cmap->cpu_map);  in cpu_map_free()
555 	kfree(cmap);  in cpu_map_free()
560 	struct bpf_cpu_map *cmap = container_of(map, struct bpf_cpu_map, map);  in __cpu_map_lookup_elem()  local
566 	rcpu = READ_ONCE(cmap->cpu_map[key]);  in __cpu_map_lookup_elem()
580 	struct bpf_cpu_map *cmap = container_of(map, struct bpf_cpu_map, map);  in cpu_map_get_next_key()  local
584 	if (index >= cmap->map.max_entries) {  in cpu_map_get_next_key()
589 	if (index == cmap->map.max_entries - 1)  in cpu_map_get_next_key()
648 	struct list_head *flush_list = this_cpu_ptr(rcpu->cmap->flush_list);  in bq_enqueue()
689 	struct bpf_cpu_map *cmap = container_of(map, struct bpf_cpu_map, map);  in __cpu_map_flush()  local
690 	struct list_head *flush_list = this_cpu_ptr(cmap->flush_list);  in __cpu_map_flush()