26 	struct bpf_dispatcher *d, struct bpf_prog *prog)  in bpf_dispatcher_find_prog()  argument
31 		if (prog == d->progs[i].prog)  in bpf_dispatcher_find_prog()
32 			return &d->progs[i];  in bpf_dispatcher_find_prog()
38 	struct bpf_dispatcher *d)  in bpf_dispatcher_find_free()  argument
40 	return bpf_dispatcher_find_prog(d, NULL);  in bpf_dispatcher_find_free()
43 static bool bpf_dispatcher_add_prog(struct bpf_dispatcher *d,  in bpf_dispatcher_add_prog()  argument
51 	entry = bpf_dispatcher_find_prog(d, prog);  in bpf_dispatcher_add_prog()
57 	entry = bpf_dispatcher_find_free(d);  in bpf_dispatcher_add_prog()
64 	d->num_progs++;  in bpf_dispatcher_add_prog()
68 static bool bpf_dispatcher_remove_prog(struct bpf_dispatcher *d,  in bpf_dispatcher_remove_prog()  argument
76 	entry = bpf_dispatcher_find_prog(d, prog);  in bpf_dispatcher_remove_prog()
83 		d->num_progs--;  in bpf_dispatcher_remove_prog()
94 static int bpf_dispatcher_prepare(struct bpf_dispatcher *d, void *image, void *buf)  in bpf_dispatcher_prepare()  argument
100 		if (d->progs[i].prog)  in bpf_dispatcher_prepare()
101 			*ipsp++ = (s64)(uintptr_t)d->progs[i].prog->bpf_func;  in bpf_dispatcher_prepare()
103 	return arch_prepare_bpf_dispatcher(image, buf, &ips[0], d->num_progs);  in bpf_dispatcher_prepare()
106 static void bpf_dispatcher_update(struct bpf_dispatcher *d, int prev_num_progs)  in bpf_dispatcher_update()  argument
112 		noff = d->image_off ^ (PAGE_SIZE / 2);  in bpf_dispatcher_update()
114 	new = d->num_progs ? d->image + noff : NULL;  in bpf_dispatcher_update()
115 	tmp = d->num_progs ? d->rw_image + noff : NULL;  in bpf_dispatcher_update()
120 		if (bpf_dispatcher_prepare(d, new, tmp))  in bpf_dispatcher_update()
126 	__BPF_DISPATCHER_UPDATE(d, new ?: (void *)&bpf_dispatcher_nop_func);  in bpf_dispatcher_update()
129 		d->image_off = noff;  in bpf_dispatcher_update()
132 void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, struct bpf_prog *from,  in bpf_dispatcher_change_prog()  argument
141 	mutex_lock(&d->mutex);  in bpf_dispatcher_change_prog()
142 	if (!d->image) {  in bpf_dispatcher_change_prog()
143 		d->image = bpf_prog_pack_alloc(PAGE_SIZE, bpf_jit_fill_hole_with_zero);  in bpf_dispatcher_change_prog()
144 		if (!d->image)  in bpf_dispatcher_change_prog()
146 		d->rw_image = bpf_jit_alloc_exec(PAGE_SIZE);  in bpf_dispatcher_change_prog()
147 		if (!d->rw_image) {  in bpf_dispatcher_change_prog()
150 			bpf_arch_text_copy(d->image, &size, sizeof(size));  in bpf_dispatcher_change_prog()
151 			bpf_prog_pack_free((struct bpf_binary_header *)d->image);  in bpf_dispatcher_change_prog()
152 			d->image = NULL;  in bpf_dispatcher_change_prog()
155 		bpf_image_ksym_add(d->image, &d->ksym);  in bpf_dispatcher_change_prog()
158 	prev_num_progs = d->num_progs;  in bpf_dispatcher_change_prog()
159 	changed |= bpf_dispatcher_remove_prog(d, from);  in bpf_dispatcher_change_prog()
160 	changed |= bpf_dispatcher_add_prog(d, to);  in bpf_dispatcher_change_prog()
165 	bpf_dispatcher_update(d, prev_num_progs);  in bpf_dispatcher_change_prog()
167 	mutex_unlock(&d->mutex);  in bpf_dispatcher_change_prog()