Lines Matching refs:prog
110 fp->aux->prog = fp; in bpf_prog_alloc_no_stats()
127 struct bpf_prog *prog; in bpf_prog_alloc() local
130 prog = bpf_prog_alloc_no_stats(size, gfp_extra_flags); in bpf_prog_alloc()
131 if (!prog) in bpf_prog_alloc()
134 prog->stats = alloc_percpu_gfp(struct bpf_prog_stats, gfp_flags); in bpf_prog_alloc()
135 if (!prog->stats) { in bpf_prog_alloc()
136 free_percpu(prog->active); in bpf_prog_alloc()
137 kfree(prog->aux); in bpf_prog_alloc()
138 vfree(prog); in bpf_prog_alloc()
145 pstats = per_cpu_ptr(prog->stats, cpu); in bpf_prog_alloc()
148 return prog; in bpf_prog_alloc()
152 int bpf_prog_alloc_jited_linfo(struct bpf_prog *prog) in bpf_prog_alloc_jited_linfo() argument
154 if (!prog->aux->nr_linfo || !prog->jit_requested) in bpf_prog_alloc_jited_linfo()
157 prog->aux->jited_linfo = kvcalloc(prog->aux->nr_linfo, in bpf_prog_alloc_jited_linfo()
158 sizeof(*prog->aux->jited_linfo), in bpf_prog_alloc_jited_linfo()
160 if (!prog->aux->jited_linfo) in bpf_prog_alloc_jited_linfo()
166 void bpf_prog_jit_attempt_done(struct bpf_prog *prog) in bpf_prog_jit_attempt_done() argument
168 if (prog->aux->jited_linfo && in bpf_prog_jit_attempt_done()
169 (!prog->jited || !prog->aux->jited_linfo[0])) { in bpf_prog_jit_attempt_done()
170 kvfree(prog->aux->jited_linfo); in bpf_prog_jit_attempt_done()
171 prog->aux->jited_linfo = NULL; in bpf_prog_jit_attempt_done()
174 kfree(prog->aux->kfunc_tab); in bpf_prog_jit_attempt_done()
175 prog->aux->kfunc_tab = NULL; in bpf_prog_jit_attempt_done()
202 void bpf_prog_fill_jited_linfo(struct bpf_prog *prog, in bpf_prog_fill_jited_linfo() argument
209 if (!prog->aux->jited_linfo) in bpf_prog_fill_jited_linfo()
213 linfo_idx = prog->aux->linfo_idx; in bpf_prog_fill_jited_linfo()
214 linfo = &prog->aux->linfo[linfo_idx]; in bpf_prog_fill_jited_linfo()
216 insn_end = insn_start + prog->len; in bpf_prog_fill_jited_linfo()
218 jited_linfo = &prog->aux->jited_linfo[linfo_idx]; in bpf_prog_fill_jited_linfo()
219 jited_linfo[0] = prog->bpf_func; in bpf_prog_fill_jited_linfo()
221 nr_linfo = prog->aux->nr_linfo - linfo_idx; in bpf_prog_fill_jited_linfo()
227 jited_linfo[i] = prog->bpf_func + in bpf_prog_fill_jited_linfo()
247 fp->aux->prog = fp; in bpf_prog_realloc()
383 static int bpf_adj_branches(struct bpf_prog *prog, u32 pos, s32 end_old, in bpf_adj_branches() argument
386 u32 i, insn_cnt = prog->len + (probe_pass ? end_new - end_old : 0); in bpf_adj_branches()
387 struct bpf_insn *insn = prog->insnsi; in bpf_adj_branches()
399 insn = prog->insnsi + end_old; in bpf_adj_branches()
430 static void bpf_adj_linfo(struct bpf_prog *prog, u32 off, u32 delta) in bpf_adj_linfo() argument
435 nr_linfo = prog->aux->nr_linfo; in bpf_adj_linfo()
439 linfo = prog->aux->linfo; in bpf_adj_linfo()
450 struct bpf_prog *bpf_patch_insn_single(struct bpf_prog *prog, u32 off, in bpf_patch_insn_single() argument
460 memcpy(prog->insnsi + off, patch, sizeof(*patch)); in bpf_patch_insn_single()
461 return prog; in bpf_patch_insn_single()
464 insn_adj_cnt = prog->len + insn_delta; in bpf_patch_insn_single()
472 (err = bpf_adj_branches(prog, off, off + 1, off + len, true))) in bpf_patch_insn_single()
479 prog_adj = bpf_prog_realloc(prog, bpf_prog_size(insn_adj_cnt), in bpf_patch_insn_single()
511 int bpf_remove_insns(struct bpf_prog *prog, u32 off, u32 cnt) in bpf_remove_insns() argument
516 memmove(prog->insnsi + off, prog->insnsi + off + cnt, in bpf_remove_insns()
517 sizeof(struct bpf_insn) * (prog->len - off - cnt)); in bpf_remove_insns()
518 prog->len -= cnt; in bpf_remove_insns()
520 return WARN_ON_ONCE(bpf_adj_branches(prog, off, off + cnt, off, false)); in bpf_remove_insns()
546 bpf_prog_ksym_set_addr(struct bpf_prog *prog) in bpf_prog_ksym_set_addr() argument
548 WARN_ON_ONCE(!bpf_prog_ebpf_jited(prog)); in bpf_prog_ksym_set_addr()
550 prog->aux->ksym.start = (unsigned long) prog->bpf_func; in bpf_prog_ksym_set_addr()
551 prog->aux->ksym.end = prog->aux->ksym.start + prog->jited_len; in bpf_prog_ksym_set_addr()
555 bpf_prog_ksym_set_name(struct bpf_prog *prog) in bpf_prog_ksym_set_name() argument
557 char *sym = prog->aux->ksym.name; in bpf_prog_ksym_set_name()
563 sizeof(prog->tag) * 2 + in bpf_prog_ksym_set_name()
571 sizeof(prog->aux->name) > KSYM_NAME_LEN); in bpf_prog_ksym_set_name()
574 sym = bin2hex(sym, prog->tag, sizeof(prog->tag)); in bpf_prog_ksym_set_name()
577 if (prog->aux->func_info_cnt) { in bpf_prog_ksym_set_name()
578 type = btf_type_by_id(prog->aux->btf, in bpf_prog_ksym_set_name()
579 prog->aux->func_info[prog->aux->func_idx].type_id); in bpf_prog_ksym_set_name()
580 func_name = btf_name_by_offset(prog->aux->btf, type->name_off); in bpf_prog_ksym_set_name()
585 if (prog->aux->name[0]) in bpf_prog_ksym_set_name()
586 snprintf(sym, (size_t)(end - sym), "_%s", prog->aux->name); in bpf_prog_ksym_set_name()
668 fp->aux->ksym.prog = true; in bpf_prog_kallsyms_add()
729 return ksym && ksym->prog ? in bpf_prog_ksym_find()
730 container_of(ksym, struct bpf_prog_aux, ksym)->prog : in bpf_prog_ksym_find()
737 struct bpf_prog *prog; in search_bpf_extables() local
740 prog = bpf_prog_ksym_find(addr); in search_bpf_extables()
741 if (!prog) in search_bpf_extables()
743 if (!prog->aux->num_exentries) in search_bpf_extables()
746 e = search_extable(prog->aux->extable, prog->aux->num_exentries, addr); in search_bpf_extables()
780 int bpf_jit_add_poke_descriptor(struct bpf_prog *prog, in bpf_jit_add_poke_descriptor() argument
783 struct bpf_jit_poke_descriptor *tab = prog->aux->poke_tab; in bpf_jit_add_poke_descriptor()
785 u32 slot = prog->aux->size_poke_tab; in bpf_jit_add_poke_descriptor()
808 prog->aux->size_poke_tab = size; in bpf_jit_add_poke_descriptor()
809 prog->aux->poke_tab = tab; in bpf_jit_add_poke_descriptor()
1111 int bpf_jit_binary_pack_finalize(struct bpf_prog *prog, in bpf_jit_binary_pack_finalize() argument
1184 int bpf_jit_get_func_addr(const struct bpf_prog *prog, in bpf_jit_get_func_addr() argument
1200 else if (prog->aux->func && in bpf_jit_get_func_addr()
1201 off >= 0 && off < prog->aux->func_cnt) in bpf_jit_get_func_addr()
1202 addr = (u8 *)prog->aux->func[off]->bpf_func; in bpf_jit_get_func_addr()
1392 fp->aux->prog = fp; in bpf_jit_prog_release_other()
1396 struct bpf_prog *bpf_jit_blind_constants(struct bpf_prog *prog) in bpf_jit_blind_constants() argument
1404 if (!prog->blinding_requested || prog->blinded) in bpf_jit_blind_constants()
1405 return prog; in bpf_jit_blind_constants()
1407 clone = bpf_prog_clone_create(prog, GFP_USER); in bpf_jit_blind_constants()
1444 bpf_jit_prog_release_other(prog, clone); in bpf_jit_blind_constants()
1837 struct bpf_prog *prog; in ___bpf_prog_run() local
1848 prog = READ_ONCE(array->ptrs[index]); in ___bpf_prog_run()
1849 if (!prog) in ___bpf_prog_run()
1857 insn = prog->insnsi; in ___bpf_prog_run()
2224 struct bpf_prog prog; member
2226 .prog = {
2274 for (item = array->items; item->prog; item++) in bpf_prog_array_length()
2275 if (item->prog != &dummy_bpf_prog.prog) in bpf_prog_array_length()
2284 for (item = array->items; item->prog; item++) in bpf_prog_array_is_empty()
2285 if (item->prog != &dummy_bpf_prog.prog) in bpf_prog_array_is_empty()
2297 for (item = array->items; item->prog; item++) { in bpf_prog_array_copy_core()
2298 if (item->prog == &dummy_bpf_prog.prog) in bpf_prog_array_copy_core()
2300 prog_ids[i] = item->prog->aux->id; in bpf_prog_array_copy_core()
2307 return !!(item->prog); in bpf_prog_array_copy_core()
2341 for (item = array->items; item->prog; item++) in bpf_prog_array_delete_safe()
2342 if (item->prog == old_prog) { in bpf_prog_array_delete_safe()
2343 WRITE_ONCE(item->prog, &dummy_bpf_prog.prog); in bpf_prog_array_delete_safe()
2365 return bpf_prog_array_update_at(array, index, &dummy_bpf_prog.prog); in bpf_prog_array_delete_safe_at()
2384 struct bpf_prog *prog) in bpf_prog_array_update_at() argument
2391 for (item = array->items; item->prog; item++) { in bpf_prog_array_update_at()
2392 if (item->prog == &dummy_bpf_prog.prog) in bpf_prog_array_update_at()
2395 WRITE_ONCE(item->prog, prog); in bpf_prog_array_update_at()
2419 for (; existing->prog; existing++) { in bpf_prog_array_copy()
2420 if (existing->prog == exclude_prog) { in bpf_prog_array_copy()
2424 if (existing->prog != &dummy_bpf_prog.prog) in bpf_prog_array_copy()
2426 if (existing->prog == include_prog) in bpf_prog_array_copy()
2454 for (; existing->prog; existing++) { in bpf_prog_array_copy()
2455 if (existing->prog == exclude_prog || in bpf_prog_array_copy()
2456 existing->prog == &dummy_bpf_prog.prog) in bpf_prog_array_copy()
2459 new->prog = existing->prog; in bpf_prog_array_copy()
2465 new->prog = include_prog; in bpf_prog_array_copy()
2469 new->prog = NULL; in bpf_prog_array_copy()
2552 bpf_prog_offload_destroy(aux->prog); in bpf_prog_free_deferred()
2554 if (aux->prog->has_callchain_buf) in bpf_prog_free_deferred()
2569 bpf_prog_unlock_free(aux->prog); in bpf_prog_free_deferred()
2571 bpf_jit_free(aux->prog); in bpf_prog_free_deferred()
2681 struct bpf_prog * __weak bpf_int_jit_compile(struct bpf_prog *prog) in bpf_int_jit_compile() argument
2683 return prog; in bpf_int_jit_compile()
2689 void __weak bpf_jit_compile(struct bpf_prog *prog) in bpf_jit_compile() argument