25 	struct bpf_dispatcher *d, struct bpf_prog *prog)  in bpf_dispatcher_find_prog()  argument
30 		if (prog == d->progs[i].prog)  in bpf_dispatcher_find_prog()
31 			return &d->progs[i];  in bpf_dispatcher_find_prog()
37 	struct bpf_dispatcher *d)  in bpf_dispatcher_find_free()  argument
39 	return bpf_dispatcher_find_prog(d, NULL);  in bpf_dispatcher_find_free()
42 static bool bpf_dispatcher_add_prog(struct bpf_dispatcher *d,  in bpf_dispatcher_add_prog()  argument
50 	entry = bpf_dispatcher_find_prog(d, prog);  in bpf_dispatcher_add_prog()
56 	entry = bpf_dispatcher_find_free(d);  in bpf_dispatcher_add_prog()
63 	d->num_progs++;  in bpf_dispatcher_add_prog()
67 static bool bpf_dispatcher_remove_prog(struct bpf_dispatcher *d,  in bpf_dispatcher_remove_prog()  argument
75 	entry = bpf_dispatcher_find_prog(d, prog);  in bpf_dispatcher_remove_prog()
82 		d->num_progs--;  in bpf_dispatcher_remove_prog()
93 static int bpf_dispatcher_prepare(struct bpf_dispatcher *d, void *image)  in bpf_dispatcher_prepare()  argument
99 		if (d->progs[i].prog)  in bpf_dispatcher_prepare()
100 			*ipsp++ = (s64)(uintptr_t)d->progs[i].prog->bpf_func;  in bpf_dispatcher_prepare()
102 	return arch_prepare_bpf_dispatcher(image, &ips[0], d->num_progs);  in bpf_dispatcher_prepare()
105 static void bpf_dispatcher_update(struct bpf_dispatcher *d, int prev_num_progs)  in bpf_dispatcher_update()  argument
115 		old = d->image + d->image_off;  in bpf_dispatcher_update()
116 		noff = d->image_off ^ (PAGE_SIZE / 2);  in bpf_dispatcher_update()
119 	new = d->num_progs ? d->image + noff : NULL;  in bpf_dispatcher_update()
121 		if (bpf_dispatcher_prepare(d, new))  in bpf_dispatcher_update()
125 	err = bpf_arch_text_poke(d->func, BPF_MOD_JUMP, old, new);  in bpf_dispatcher_update()
129 	d->image_off = noff;  in bpf_dispatcher_update()
132 void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, struct bpf_prog *from,  in bpf_dispatcher_change_prog()  argument
141 	mutex_lock(&d->mutex);  in bpf_dispatcher_change_prog()
142 	if (!d->image) {  in bpf_dispatcher_change_prog()
143 		d->image = bpf_jit_alloc_exec_page();  in bpf_dispatcher_change_prog()
144 		if (!d->image)  in bpf_dispatcher_change_prog()
146 		bpf_image_ksym_add(d->image, &d->ksym);  in bpf_dispatcher_change_prog()
149 	prev_num_progs = d->num_progs;  in bpf_dispatcher_change_prog()
150 	changed |= bpf_dispatcher_remove_prog(d, from);  in bpf_dispatcher_change_prog()
151 	changed |= bpf_dispatcher_add_prog(d, to);  in bpf_dispatcher_change_prog()
156 	bpf_dispatcher_update(d, prev_num_progs);  in bpf_dispatcher_change_prog()
158 	mutex_unlock(&d->mutex);  in bpf_dispatcher_change_prog()