Lines Matching refs:prog
96 fp->aux->prog = fp; in bpf_prog_alloc_no_stats()
107 struct bpf_prog *prog; in bpf_prog_alloc() local
110 prog = bpf_prog_alloc_no_stats(size, gfp_extra_flags); in bpf_prog_alloc()
111 if (!prog) in bpf_prog_alloc()
114 prog->aux->stats = alloc_percpu_gfp(struct bpf_prog_stats, gfp_flags); in bpf_prog_alloc()
115 if (!prog->aux->stats) { in bpf_prog_alloc()
116 kfree(prog->aux); in bpf_prog_alloc()
117 vfree(prog); in bpf_prog_alloc()
124 pstats = per_cpu_ptr(prog->aux->stats, cpu); in bpf_prog_alloc()
127 return prog; in bpf_prog_alloc()
131 int bpf_prog_alloc_jited_linfo(struct bpf_prog *prog) in bpf_prog_alloc_jited_linfo() argument
133 if (!prog->aux->nr_linfo || !prog->jit_requested) in bpf_prog_alloc_jited_linfo()
136 prog->aux->jited_linfo = kcalloc(prog->aux->nr_linfo, in bpf_prog_alloc_jited_linfo()
137 sizeof(*prog->aux->jited_linfo), in bpf_prog_alloc_jited_linfo()
139 if (!prog->aux->jited_linfo) in bpf_prog_alloc_jited_linfo()
145 void bpf_prog_free_jited_linfo(struct bpf_prog *prog) in bpf_prog_free_jited_linfo() argument
147 kfree(prog->aux->jited_linfo); in bpf_prog_free_jited_linfo()
148 prog->aux->jited_linfo = NULL; in bpf_prog_free_jited_linfo()
151 void bpf_prog_free_unused_jited_linfo(struct bpf_prog *prog) in bpf_prog_free_unused_jited_linfo() argument
153 if (prog->aux->jited_linfo && !prog->aux->jited_linfo[0]) in bpf_prog_free_unused_jited_linfo()
154 bpf_prog_free_jited_linfo(prog); in bpf_prog_free_unused_jited_linfo()
181 void bpf_prog_fill_jited_linfo(struct bpf_prog *prog, in bpf_prog_fill_jited_linfo() argument
188 if (!prog->aux->jited_linfo) in bpf_prog_fill_jited_linfo()
192 linfo_idx = prog->aux->linfo_idx; in bpf_prog_fill_jited_linfo()
193 linfo = &prog->aux->linfo[linfo_idx]; in bpf_prog_fill_jited_linfo()
195 insn_end = insn_start + prog->len; in bpf_prog_fill_jited_linfo()
197 jited_linfo = &prog->aux->jited_linfo[linfo_idx]; in bpf_prog_fill_jited_linfo()
198 jited_linfo[0] = prog->bpf_func; in bpf_prog_fill_jited_linfo()
200 nr_linfo = prog->aux->nr_linfo - linfo_idx; in bpf_prog_fill_jited_linfo()
206 jited_linfo[i] = prog->bpf_func + in bpf_prog_fill_jited_linfo()
210 void bpf_prog_free_linfo(struct bpf_prog *prog) in bpf_prog_free_linfo() argument
212 bpf_prog_free_jited_linfo(prog); in bpf_prog_free_linfo()
213 kvfree(prog->aux->linfo); in bpf_prog_free_linfo()
242 fp->aux->prog = fp; in bpf_prog_realloc()
372 static int bpf_adj_branches(struct bpf_prog *prog, u32 pos, s32 end_old, in bpf_adj_branches() argument
375 u32 i, insn_cnt = prog->len + (probe_pass ? end_new - end_old : 0); in bpf_adj_branches()
376 struct bpf_insn *insn = prog->insnsi; in bpf_adj_branches()
388 insn = prog->insnsi + end_old; in bpf_adj_branches()
412 static void bpf_adj_linfo(struct bpf_prog *prog, u32 off, u32 delta) in bpf_adj_linfo() argument
417 nr_linfo = prog->aux->nr_linfo; in bpf_adj_linfo()
421 linfo = prog->aux->linfo; in bpf_adj_linfo()
432 struct bpf_prog *bpf_patch_insn_single(struct bpf_prog *prog, u32 off, in bpf_patch_insn_single() argument
442 memcpy(prog->insnsi + off, patch, sizeof(*patch)); in bpf_patch_insn_single()
443 return prog; in bpf_patch_insn_single()
446 insn_adj_cnt = prog->len + insn_delta; in bpf_patch_insn_single()
454 (err = bpf_adj_branches(prog, off, off + 1, off + len, true))) in bpf_patch_insn_single()
461 prog_adj = bpf_prog_realloc(prog, bpf_prog_size(insn_adj_cnt), in bpf_patch_insn_single()
493 int bpf_remove_insns(struct bpf_prog *prog, u32 off, u32 cnt) in bpf_remove_insns() argument
498 memmove(prog->insnsi + off, prog->insnsi + off + cnt, in bpf_remove_insns()
499 sizeof(struct bpf_insn) * (prog->len - off - cnt)); in bpf_remove_insns()
500 prog->len -= cnt; in bpf_remove_insns()
502 return WARN_ON_ONCE(bpf_adj_branches(prog, off, off + cnt, off, false)); in bpf_remove_insns()
527 bpf_get_prog_addr_region(const struct bpf_prog *prog, in bpf_get_prog_addr_region() argument
531 const struct bpf_binary_header *hdr = bpf_jit_binary_hdr(prog); in bpf_get_prog_addr_region()
534 WARN_ON_ONCE(!bpf_prog_ebpf_jited(prog)); in bpf_get_prog_addr_region()
540 void bpf_get_prog_name(const struct bpf_prog *prog, char *sym) in bpf_get_prog_name() argument
547 sizeof(prog->tag) * 2 + in bpf_get_prog_name()
555 sizeof(prog->aux->name) > KSYM_NAME_LEN); in bpf_get_prog_name()
558 sym = bin2hex(sym, prog->tag, sizeof(prog->tag)); in bpf_get_prog_name()
561 if (prog->aux->func_info_cnt) { in bpf_get_prog_name()
562 type = btf_type_by_id(prog->aux->btf, in bpf_get_prog_name()
563 prog->aux->func_info[prog->aux->func_idx].type_id); in bpf_get_prog_name()
564 func_name = btf_name_by_offset(prog->aux->btf, type->name_off); in bpf_get_prog_name()
569 if (prog->aux->name[0]) in bpf_get_prog_name()
570 snprintf(sym, (size_t)(end - sym), "_%s", prog->aux->name); in bpf_get_prog_name()
582 bpf_get_prog_addr_region(aux->prog, &symbol_start, &symbol_end); in bpf_get_prog_addr_start()
600 bpf_get_prog_addr_region(aux->prog, &symbol_start, &symbol_end); in bpf_tree_comp()
676 container_of(n, struct bpf_prog_aux, ksym_tnode)->prog : in bpf_prog_kallsyms_find()
684 struct bpf_prog *prog; in __bpf_address_lookup() local
688 prog = bpf_prog_kallsyms_find(addr); in __bpf_address_lookup()
689 if (prog) { in __bpf_address_lookup()
690 bpf_get_prog_addr_region(prog, &symbol_start, &symbol_end); in __bpf_address_lookup()
691 bpf_get_prog_name(prog, sym); in __bpf_address_lookup()
730 bpf_get_prog_name(aux->prog, sym); in bpf_get_kallsym()
732 *value = (unsigned long)aux->prog->bpf_func; in bpf_get_kallsym()
796 bool __weak arch_bpf_jit_check_func(const struct bpf_prog *prog) in arch_bpf_jit_check_func() argument
866 int bpf_jit_get_func_addr(const struct bpf_prog *prog, in bpf_jit_get_func_addr() argument
882 else if (prog->aux->func && in bpf_jit_get_func_addr()
883 off >= 0 && off < prog->aux->func_cnt) in bpf_jit_get_func_addr()
884 addr = (u8 *)prog->aux->func[off]->bpf_func; in bpf_jit_get_func_addr()
1072 fp->aux->prog = fp; in bpf_jit_prog_release_other()
1076 struct bpf_prog *bpf_jit_blind_constants(struct bpf_prog *prog) in bpf_jit_blind_constants() argument
1084 if (!bpf_jit_blinding_enabled(prog) || prog->blinded) in bpf_jit_blind_constants()
1085 return prog; in bpf_jit_blind_constants()
1087 clone = bpf_prog_clone_create(prog, GFP_USER); in bpf_jit_blind_constants()
1114 bpf_jit_prog_release_other(prog, clone); in bpf_jit_blind_constants()
1470 struct bpf_prog *prog; in ___bpf_prog_run() local
1480 prog = READ_ONCE(array->ptrs[index]); in ___bpf_prog_run()
1481 if (!prog) in ___bpf_prog_run()
1489 insn = prog->insnsi; in ___bpf_prog_run()
1775 struct bpf_prog prog; member
1777 .prog = {
1818 for (item = array->items; item->prog; item++) in bpf_prog_array_length()
1819 if (item->prog != &dummy_bpf_prog.prog) in bpf_prog_array_length()
1828 for (item = array->items; item->prog; item++) in bpf_prog_array_is_empty()
1829 if (item->prog != &dummy_bpf_prog.prog) in bpf_prog_array_is_empty()
1841 for (item = array->items; item->prog; item++) { in bpf_prog_array_copy_core()
1842 if (item->prog == &dummy_bpf_prog.prog) in bpf_prog_array_copy_core()
1844 prog_ids[i] = item->prog->aux->id; in bpf_prog_array_copy_core()
1851 return !!(item->prog); in bpf_prog_array_copy_core()
1885 for (item = array->items; item->prog; item++) in bpf_prog_array_delete_safe()
1886 if (item->prog == old_prog) { in bpf_prog_array_delete_safe()
1887 WRITE_ONCE(item->prog, &dummy_bpf_prog.prog); in bpf_prog_array_delete_safe()
1908 for (; existing->prog; existing++) { in bpf_prog_array_copy()
1909 if (existing->prog == exclude_prog) { in bpf_prog_array_copy()
1913 if (existing->prog != &dummy_bpf_prog.prog) in bpf_prog_array_copy()
1915 if (existing->prog == include_prog) in bpf_prog_array_copy()
1942 for (; existing->prog; existing++) in bpf_prog_array_copy()
1943 if (existing->prog != exclude_prog && in bpf_prog_array_copy()
1944 existing->prog != &dummy_bpf_prog.prog) { in bpf_prog_array_copy()
1945 array->items[new_prog_idx++].prog = in bpf_prog_array_copy()
1946 existing->prog; in bpf_prog_array_copy()
1950 array->items[new_prog_idx++].prog = include_prog; in bpf_prog_array_copy()
1951 array->items[new_prog_idx].prog = NULL; in bpf_prog_array_copy()
1983 bpf_prog_offload_destroy(aux->prog); in bpf_prog_free_deferred()
1985 if (aux->prog->has_callchain_buf) in bpf_prog_free_deferred()
1992 bpf_prog_unlock_free(aux->prog); in bpf_prog_free_deferred()
1994 bpf_jit_free(aux->prog); in bpf_prog_free_deferred()
2082 struct bpf_prog * __weak bpf_int_jit_compile(struct bpf_prog *prog) in bpf_int_jit_compile() argument
2084 return prog; in bpf_int_jit_compile()
2090 void __weak bpf_jit_compile(struct bpf_prog *prog) in bpf_jit_compile() argument