Lines Matching refs:aux
80 struct bpf_prog_aux *aux; in bpf_prog_alloc_no_stats() local
88 aux = kzalloc(sizeof(*aux), GFP_KERNEL | gfp_extra_flags); in bpf_prog_alloc_no_stats()
89 if (aux == NULL) { in bpf_prog_alloc_no_stats()
95 fp->aux = aux; in bpf_prog_alloc_no_stats()
96 fp->aux->prog = fp; in bpf_prog_alloc_no_stats()
99 INIT_LIST_HEAD_RCU(&fp->aux->ksym_lnode); in bpf_prog_alloc_no_stats()
114 prog->aux->stats = alloc_percpu_gfp(struct bpf_prog_stats, gfp_flags); in bpf_prog_alloc()
115 if (!prog->aux->stats) { in bpf_prog_alloc()
116 kfree(prog->aux); in bpf_prog_alloc()
124 pstats = per_cpu_ptr(prog->aux->stats, cpu); in bpf_prog_alloc()
133 if (!prog->aux->nr_linfo || !prog->jit_requested) in bpf_prog_alloc_jited_linfo()
136 prog->aux->jited_linfo = kcalloc(prog->aux->nr_linfo, in bpf_prog_alloc_jited_linfo()
137 sizeof(*prog->aux->jited_linfo), in bpf_prog_alloc_jited_linfo()
139 if (!prog->aux->jited_linfo) in bpf_prog_alloc_jited_linfo()
147 kfree(prog->aux->jited_linfo); in bpf_prog_free_jited_linfo()
148 prog->aux->jited_linfo = NULL; in bpf_prog_free_jited_linfo()
153 if (prog->aux->jited_linfo && !prog->aux->jited_linfo[0]) in bpf_prog_free_unused_jited_linfo()
188 if (!prog->aux->jited_linfo) in bpf_prog_fill_jited_linfo()
192 linfo_idx = prog->aux->linfo_idx; in bpf_prog_fill_jited_linfo()
193 linfo = &prog->aux->linfo[linfo_idx]; in bpf_prog_fill_jited_linfo()
197 jited_linfo = &prog->aux->jited_linfo[linfo_idx]; in bpf_prog_fill_jited_linfo()
200 nr_linfo = prog->aux->nr_linfo - linfo_idx; in bpf_prog_fill_jited_linfo()
213 kvfree(prog->aux->linfo); in bpf_prog_free_linfo()
232 ret = __bpf_prog_charge(fp_old->aux->user, delta); in bpf_prog_realloc()
238 __bpf_prog_uncharge(fp_old->aux->user, delta); in bpf_prog_realloc()
242 fp->aux->prog = fp; in bpf_prog_realloc()
247 fp_old->aux = NULL; in bpf_prog_realloc()
256 if (fp->aux) { in __bpf_prog_free()
257 free_percpu(fp->aux->stats); in __bpf_prog_free()
258 kfree(fp->aux); in __bpf_prog_free()
417 nr_linfo = prog->aux->nr_linfo; in bpf_adj_linfo()
421 linfo = prog->aux->linfo; in bpf_adj_linfo()
509 for (i = 0; i < fp->aux->func_cnt; i++) in bpf_prog_kallsyms_del_subprogs()
510 bpf_prog_kallsyms_del(fp->aux->func[i]); in bpf_prog_kallsyms_del_subprogs()
555 sizeof(prog->aux->name) > KSYM_NAME_LEN); in bpf_get_prog_name()
561 if (prog->aux->func_info_cnt) { in bpf_get_prog_name()
562 type = btf_type_by_id(prog->aux->btf, in bpf_get_prog_name()
563 prog->aux->func_info[prog->aux->func_idx].type_id); in bpf_get_prog_name()
564 func_name = btf_name_by_offset(prog->aux->btf, type->name_off); in bpf_get_prog_name()
569 if (prog->aux->name[0]) in bpf_get_prog_name()
570 snprintf(sym, (size_t)(end - sym), "_%s", prog->aux->name); in bpf_get_prog_name()
579 const struct bpf_prog_aux *aux; in bpf_get_prog_addr_start() local
581 aux = container_of(n, struct bpf_prog_aux, ksym_tnode); in bpf_get_prog_addr_start()
582 bpf_get_prog_addr_region(aux->prog, &symbol_start, &symbol_end); in bpf_get_prog_addr_start()
597 const struct bpf_prog_aux *aux; in bpf_tree_comp() local
599 aux = container_of(n, struct bpf_prog_aux, ksym_tnode); in bpf_tree_comp()
600 bpf_get_prog_addr_region(aux->prog, &symbol_start, &symbol_end); in bpf_tree_comp()
619 static void bpf_prog_ksym_node_add(struct bpf_prog_aux *aux) in bpf_prog_ksym_node_add() argument
621 WARN_ON_ONCE(!list_empty(&aux->ksym_lnode)); in bpf_prog_ksym_node_add()
622 list_add_tail_rcu(&aux->ksym_lnode, &bpf_kallsyms); in bpf_prog_ksym_node_add()
623 latch_tree_insert(&aux->ksym_tnode, &bpf_tree, &bpf_tree_ops); in bpf_prog_ksym_node_add()
626 static void bpf_prog_ksym_node_del(struct bpf_prog_aux *aux) in bpf_prog_ksym_node_del() argument
628 if (list_empty(&aux->ksym_lnode)) in bpf_prog_ksym_node_del()
631 latch_tree_erase(&aux->ksym_tnode, &bpf_tree, &bpf_tree_ops); in bpf_prog_ksym_node_del()
632 list_del_rcu(&aux->ksym_lnode); in bpf_prog_ksym_node_del()
642 return list_empty(&fp->aux->ksym_lnode) || in bpf_prog_kallsyms_verify_off()
643 fp->aux->ksym_lnode.prev == LIST_POISON2; in bpf_prog_kallsyms_verify_off()
653 bpf_prog_ksym_node_add(fp->aux); in bpf_prog_kallsyms_add()
663 bpf_prog_ksym_node_del(fp->aux); in bpf_prog_kallsyms_del()
718 struct bpf_prog_aux *aux; in bpf_get_kallsym() local
726 list_for_each_entry_rcu(aux, &bpf_kallsyms, ksym_lnode) { in bpf_get_kallsym()
730 bpf_get_prog_name(aux->prog, sym); in bpf_get_kallsym()
732 *value = (unsigned long)aux->prog->bpf_func; in bpf_get_kallsym()
882 else if (prog->aux->func && in bpf_jit_get_func_addr()
883 off >= 0 && off < prog->aux->func_cnt) in bpf_jit_get_func_addr()
884 addr = (u8 *)prog->aux->func[off]->bpf_func; in bpf_jit_get_func_addr()
901 const struct bpf_insn *aux, in bpf_jit_blind_insn() argument
1010 *to++ = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ aux[1].imm); in bpf_jit_blind_insn()
1013 *to++ = BPF_ALU64_REG(BPF_MOV, aux[0].dst_reg, BPF_REG_AX); in bpf_jit_blind_insn()
1016 *to++ = BPF_ALU32_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ aux[0].imm); in bpf_jit_blind_insn()
1020 *to++ = BPF_ALU64_REG(BPF_OR, aux[0].dst_reg, BPF_REG_AX); in bpf_jit_blind_insn()
1063 fp->aux = NULL; in bpf_prog_clone_free()
1072 fp->aux->prog = fp; in bpf_jit_prog_release_other()
1078 struct bpf_insn insn_buff[16], aux[2]; in bpf_jit_blind_constants() local
1101 memcpy(aux, insn, sizeof(aux)); in bpf_jit_blind_constants()
1103 rewritten = bpf_jit_blind_insn(insn, aux, insn_buff, in bpf_jit_blind_constants()
1104 clone->aux->verifier_zext); in bpf_jit_blind_constants()
1680 struct bpf_prog_aux *aux = fp->aux; in bpf_check_tail_call() local
1683 for (i = 0; i < aux->used_map_cnt; i++) { in bpf_check_tail_call()
1684 struct bpf_map *map = aux->used_maps[i]; in bpf_check_tail_call()
1701 u32 stack_depth = max_t(u32, fp->aux->stack_depth, 1); in bpf_prog_select_func()
1733 if (!bpf_prog_is_dev_bound(fp->aux)) { in bpf_prog_select_runtime()
1844 prog_ids[i] = item->prog->aux->id; in bpf_prog_array_copy_core()
1978 struct bpf_prog_aux *aux; in bpf_prog_free_deferred() local
1981 aux = container_of(work, struct bpf_prog_aux, work); in bpf_prog_free_deferred()
1982 if (bpf_prog_is_dev_bound(aux)) in bpf_prog_free_deferred()
1983 bpf_prog_offload_destroy(aux->prog); in bpf_prog_free_deferred()
1985 if (aux->prog->has_callchain_buf) in bpf_prog_free_deferred()
1988 for (i = 0; i < aux->func_cnt; i++) in bpf_prog_free_deferred()
1989 bpf_jit_free(aux->func[i]); in bpf_prog_free_deferred()
1990 if (aux->func_cnt) { in bpf_prog_free_deferred()
1991 kfree(aux->func); in bpf_prog_free_deferred()
1992 bpf_prog_unlock_free(aux->prog); in bpf_prog_free_deferred()
1994 bpf_jit_free(aux->prog); in bpf_prog_free_deferred()
2001 struct bpf_prog_aux *aux = fp->aux; in bpf_prog_free() local
2003 INIT_WORK(&aux->work, bpf_prog_free_deferred); in bpf_prog_free()
2004 schedule_work(&aux->work); in bpf_prog_free()