Lines Matching refs:env
191 static int acquire_reference_state(struct bpf_verifier_env *env, int insn_idx);
192 static int release_reference(struct bpf_verifier_env *env, int ref_obj_id);
274 find_linfo(const struct bpf_verifier_env *env, u32 insn_off) in find_linfo() argument
280 prog = env->prog; in find_linfo()
335 __printf(2, 3) void bpf_verifier_log_write(struct bpf_verifier_env *env, in bpf_verifier_log_write() argument
340 if (!bpf_verifier_log_needed(&env->log)) in bpf_verifier_log_write()
344 bpf_verifier_vlog(&env->log, fmt, args); in bpf_verifier_log_write()
351 struct bpf_verifier_env *env = private_data; in verbose() local
354 if (!bpf_verifier_log_needed(&env->log)) in verbose()
358 bpf_verifier_vlog(&env->log, fmt, args); in verbose()
384 __printf(3, 4) static void verbose_linfo(struct bpf_verifier_env *env, in verbose_linfo() argument
390 if (!bpf_verifier_log_needed(&env->log)) in verbose_linfo()
393 linfo = find_linfo(env, insn_off); in verbose_linfo()
394 if (!linfo || linfo == env->prev_linfo) in verbose_linfo()
401 bpf_verifier_vlog(&env->log, prefix_fmt, args); in verbose_linfo()
405 verbose(env, "%s\n", in verbose_linfo()
406 ltrim(btf_name_by_offset(env->prog->aux->btf, in verbose_linfo()
409 env->prev_linfo = linfo; in verbose_linfo()
412 static void verbose_invalid_scalar(struct bpf_verifier_env *env, in verbose_invalid_scalar() argument
419 verbose(env, "At %s the register %s ", ctx, reg_name); in verbose_invalid_scalar()
422 verbose(env, "has value %s", tn_buf); in verbose_invalid_scalar()
424 verbose(env, "has unknown scalar value"); in verbose_invalid_scalar()
427 verbose(env, " should have been in %s\n", tn_buf); in verbose_invalid_scalar()
550 static const char *reg_type_str(struct bpf_verifier_env *env, in reg_type_str() argument
596 snprintf(env->type_str_buf, TYPE_STR_BUF_LEN, "%s%s%s", in reg_type_str()
598 return env->type_str_buf; in reg_type_str()
609 static void print_liveness(struct bpf_verifier_env *env, in print_liveness() argument
613 verbose(env, "_"); in print_liveness()
615 verbose(env, "r"); in print_liveness()
617 verbose(env, "w"); in print_liveness()
619 verbose(env, "D"); in print_liveness()
641 static struct bpf_func_state *func(struct bpf_verifier_env *env, in func() argument
644 struct bpf_verifier_state *cur = env->cur_state; in func()
654 static void mark_reg_scratched(struct bpf_verifier_env *env, u32 regno) in mark_reg_scratched() argument
656 env->scratched_regs |= 1U << regno; in mark_reg_scratched()
659 static void mark_stack_slot_scratched(struct bpf_verifier_env *env, u32 spi) in mark_stack_slot_scratched() argument
661 env->scratched_stack_slots |= 1ULL << spi; in mark_stack_slot_scratched()
664 static bool reg_scratched(const struct bpf_verifier_env *env, u32 regno) in reg_scratched() argument
666 return (env->scratched_regs >> regno) & 1; in reg_scratched()
669 static bool stack_slot_scratched(const struct bpf_verifier_env *env, u64 regno) in stack_slot_scratched() argument
671 return (env->scratched_stack_slots >> regno) & 1; in stack_slot_scratched()
674 static bool verifier_state_scratched(const struct bpf_verifier_env *env) in verifier_state_scratched() argument
676 return env->scratched_regs || env->scratched_stack_slots; in verifier_state_scratched()
679 static void mark_verifier_state_clean(struct bpf_verifier_env *env) in mark_verifier_state_clean() argument
681 env->scratched_regs = 0U; in mark_verifier_state_clean()
682 env->scratched_stack_slots = 0ULL; in mark_verifier_state_clean()
686 static void mark_verifier_state_scratched(struct bpf_verifier_env *env) in mark_verifier_state_scratched() argument
688 env->scratched_regs = ~0U; in mark_verifier_state_scratched()
689 env->scratched_stack_slots = ~0ULL; in mark_verifier_state_scratched()
709 static int mark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_state *reg, in mark_stack_slots_dynptr() argument
712 struct bpf_func_state *state = func(env, reg); in mark_stack_slots_dynptr()
736 id = acquire_reference_state(env, insn_idx); in mark_stack_slots_dynptr()
747 static int unmark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_state *reg) in unmark_stack_slots_dynptr() argument
749 struct bpf_func_state *state = func(env, reg); in unmark_stack_slots_dynptr()
764 release_reference(env, state->stack[spi].spilled_ptr.id); in unmark_stack_slots_dynptr()
776 static bool is_dynptr_reg_valid_uninit(struct bpf_verifier_env *env, struct bpf_reg_state *reg) in is_dynptr_reg_valid_uninit() argument
778 struct bpf_func_state *state = func(env, reg); in is_dynptr_reg_valid_uninit()
794 bool is_dynptr_reg_valid_init(struct bpf_verifier_env *env, in is_dynptr_reg_valid_init() argument
797 struct bpf_func_state *state = func(env, reg); in is_dynptr_reg_valid_init()
814 bool is_dynptr_type_expected(struct bpf_verifier_env *env, in is_dynptr_type_expected() argument
818 struct bpf_func_state *state = func(env, reg); in is_dynptr_type_expected()
845 static void print_verifier_state(struct bpf_verifier_env *env, in print_verifier_state() argument
854 verbose(env, " frame%d:", state->frameno); in print_verifier_state()
860 if (!print_all && !reg_scratched(env, i)) in print_verifier_state()
862 verbose(env, " R%d", i); in print_verifier_state()
863 print_liveness(env, reg->live); in print_verifier_state()
864 verbose(env, "="); in print_verifier_state()
866 verbose(env, "P"); in print_verifier_state()
870 verbose(env, "%s", t == SCALAR_VALUE ? "" : reg_type_str(env, t)); in print_verifier_state()
871 verbose(env, "%lld", reg->var_off.value + reg->off); in print_verifier_state()
875 verbose(env, "%s", reg_type_str(env, t)); in print_verifier_state()
877 verbose(env, "%s", kernel_type_name(reg->btf, reg->btf_id)); in print_verifier_state()
878 verbose(env, "("); in print_verifier_state()
883 #define verbose_a(fmt, ...) ({ verbose(env, "%s" fmt, sep, __VA_ARGS__); sep = ","; }) in print_verifier_state()
937 verbose(env, ")"); in print_verifier_state()
954 if (!print_all && !stack_slot_scratched(env, i)) in print_verifier_state()
956 verbose(env, " fp%d", (-i - 1) * BPF_REG_SIZE); in print_verifier_state()
957 print_liveness(env, state->stack[i].spilled_ptr.live); in print_verifier_state()
961 verbose(env, "=%s", t == SCALAR_VALUE ? "" : reg_type_str(env, t)); in print_verifier_state()
963 verbose(env, "P"); in print_verifier_state()
965 verbose(env, "%lld", reg->var_off.value + reg->off); in print_verifier_state()
967 verbose(env, "=%s", types_buf); in print_verifier_state()
971 verbose(env, " refs=%d", state->refs[0].id); in print_verifier_state()
974 verbose(env, ",%d", state->refs[i].id); in print_verifier_state()
977 verbose(env, " cb"); in print_verifier_state()
979 verbose(env, " async_cb"); in print_verifier_state()
980 verbose(env, "\n"); in print_verifier_state()
982 mark_verifier_state_clean(env); in print_verifier_state()
991 static void print_insn_state(struct bpf_verifier_env *env, in print_insn_state() argument
994 if (env->prev_log_len && env->prev_log_len == env->log.len_used) { in print_insn_state()
996 bpf_vlog_reset(&env->log, env->prev_log_len - 1); in print_insn_state()
997 verbose(env, "%*c;", vlog_alignment(env->prev_insn_print_len), ' '); in print_insn_state()
999 verbose(env, "%d:", env->insn_idx); in print_insn_state()
1001 print_verifier_state(env, state, false); in print_insn_state()
1118 static int acquire_reference_state(struct bpf_verifier_env *env, int insn_idx) in acquire_reference_state() argument
1120 struct bpf_func_state *state = cur_func(env); in acquire_reference_state()
1127 id = ++env->id_gen; in acquire_reference_state()
1242 static void update_branch_counts(struct bpf_verifier_env *env, struct bpf_verifier_state *st) in update_branch_counts() argument
1259 static int pop_stack(struct bpf_verifier_env *env, int *prev_insn_idx, in pop_stack() argument
1262 struct bpf_verifier_state *cur = env->cur_state; in pop_stack()
1263 struct bpf_verifier_stack_elem *elem, *head = env->head; in pop_stack()
1266 if (env->head == NULL) in pop_stack()
1275 bpf_vlog_reset(&env->log, head->log_pos); in pop_stack()
1283 env->head = elem; in pop_stack()
1284 env->stack_size--; in pop_stack()
1288 static struct bpf_verifier_state *push_stack(struct bpf_verifier_env *env, in push_stack() argument
1292 struct bpf_verifier_state *cur = env->cur_state; in push_stack()
1302 elem->next = env->head; in push_stack()
1303 elem->log_pos = env->log.len_used; in push_stack()
1304 env->head = elem; in push_stack()
1305 env->stack_size++; in push_stack()
1310 if (env->stack_size > BPF_COMPLEXITY_LIMIT_JMP_SEQ) { in push_stack()
1311 verbose(env, "The sequence of %d jumps is too complex.\n", in push_stack()
1312 env->stack_size); in push_stack()
1329 free_verifier_state(env->cur_state, true); in push_stack()
1330 env->cur_state = NULL; in push_stack()
1332 while (!pop_stack(env, NULL, NULL, false)); in push_stack()
1341 static void __mark_reg_not_init(const struct bpf_verifier_env *env,
1393 static void mark_reg_known_zero(struct bpf_verifier_env *env, in mark_reg_known_zero() argument
1397 verbose(env, "mark_reg_known_zero(regs, %u)\n", regno); in mark_reg_known_zero()
1400 __mark_reg_not_init(env, regs + regno); in mark_reg_known_zero()
1695 static void __mark_reg_unknown(const struct bpf_verifier_env *env, in __mark_reg_unknown() argument
1706 reg->precise = !env->bpf_capable; in __mark_reg_unknown()
1710 static void mark_reg_unknown(struct bpf_verifier_env *env, in mark_reg_unknown() argument
1714 verbose(env, "mark_reg_unknown(regs, %u)\n", regno); in mark_reg_unknown()
1717 __mark_reg_not_init(env, regs + regno); in mark_reg_unknown()
1720 __mark_reg_unknown(env, regs + regno); in mark_reg_unknown()
1723 static void __mark_reg_not_init(const struct bpf_verifier_env *env, in __mark_reg_not_init() argument
1726 __mark_reg_unknown(env, reg); in __mark_reg_not_init()
1730 static void mark_reg_not_init(struct bpf_verifier_env *env, in mark_reg_not_init() argument
1734 verbose(env, "mark_reg_not_init(regs, %u)\n", regno); in mark_reg_not_init()
1737 __mark_reg_not_init(env, regs + regno); in mark_reg_not_init()
1740 __mark_reg_not_init(env, regs + regno); in mark_reg_not_init()
1743 static void mark_btf_ld_reg(struct bpf_verifier_env *env, in mark_btf_ld_reg() argument
1750 mark_reg_unknown(env, regs, regno); in mark_btf_ld_reg()
1753 mark_reg_known_zero(env, regs, regno); in mark_btf_ld_reg()
1760 static void init_reg_state(struct bpf_verifier_env *env, in init_reg_state() argument
1767 mark_reg_not_init(env, regs, i); in init_reg_state()
1775 mark_reg_known_zero(env, regs, BPF_REG_FP); in init_reg_state()
1780 static void init_func_state(struct bpf_verifier_env *env, in init_func_state() argument
1788 init_reg_state(env, state); in init_func_state()
1789 mark_verifier_state_scratched(env); in init_func_state()
1793 static struct bpf_verifier_state *push_async_cb(struct bpf_verifier_env *env, in push_async_cb() argument
1806 elem->next = env->head; in push_async_cb()
1807 elem->log_pos = env->log.len_used; in push_async_cb()
1808 env->head = elem; in push_async_cb()
1809 env->stack_size++; in push_async_cb()
1810 if (env->stack_size > BPF_COMPLEXITY_LIMIT_JMP_SEQ) { in push_async_cb()
1811 verbose(env, in push_async_cb()
1813 env->stack_size); in push_async_cb()
1825 init_func_state(env, frame, in push_async_cb()
1832 free_verifier_state(env->cur_state, true); in push_async_cb()
1833 env->cur_state = NULL; in push_async_cb()
1835 while (!pop_stack(env, NULL, NULL, false)); in push_async_cb()
1852 static int find_subprog(struct bpf_verifier_env *env, int off) in find_subprog() argument
1856 p = bsearch(&off, env->subprog_info, env->subprog_cnt, in find_subprog()
1857 sizeof(env->subprog_info[0]), cmp_subprogs); in find_subprog()
1860 return p - env->subprog_info; in find_subprog()
1864 static int add_subprog(struct bpf_verifier_env *env, int off) in add_subprog() argument
1866 int insn_cnt = env->prog->len; in add_subprog()
1870 verbose(env, "call to invalid destination\n"); in add_subprog()
1873 ret = find_subprog(env, off); in add_subprog()
1876 if (env->subprog_cnt >= BPF_MAX_SUBPROGS) { in add_subprog()
1877 verbose(env, "too many subprograms\n"); in add_subprog()
1881 env->subprog_info[env->subprog_cnt++].start = off; in add_subprog()
1882 sort(env->subprog_info, env->subprog_cnt, in add_subprog()
1883 sizeof(env->subprog_info[0]), cmp_subprogs, NULL); in add_subprog()
1884 return env->subprog_cnt - 1; in add_subprog()
1944 static struct btf *__find_kfunc_desc_btf(struct bpf_verifier_env *env, in __find_kfunc_desc_btf() argument
1954 tab = env->prog->aux->kfunc_btf_tab; in __find_kfunc_desc_btf()
1959 verbose(env, "too many different module BTFs\n"); in __find_kfunc_desc_btf()
1963 if (bpfptr_is_null(env->fd_array)) { in __find_kfunc_desc_btf()
1964 verbose(env, "kfunc offset > 0 without fd_array is invalid\n"); in __find_kfunc_desc_btf()
1968 if (copy_from_bpfptr_offset(&btf_fd, env->fd_array, in __find_kfunc_desc_btf()
1975 verbose(env, "invalid module BTF fd specified\n"); in __find_kfunc_desc_btf()
1980 verbose(env, "BTF fd for kfunc is not a module BTF\n"); in __find_kfunc_desc_btf()
2014 static struct btf *find_kfunc_desc_btf(struct bpf_verifier_env *env, s16 offset) in find_kfunc_desc_btf() argument
2021 verbose(env, "negative offset disallowed for kernel module function call\n"); in find_kfunc_desc_btf()
2025 return __find_kfunc_desc_btf(env, offset); in find_kfunc_desc_btf()
2030 static int add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, s16 offset) in add_kfunc_call() argument
2043 prog_aux = env->prog->aux; in add_kfunc_call()
2048 verbose(env, "calling kernel function is not supported without CONFIG_DEBUG_INFO_BTF\n"); in add_kfunc_call()
2052 if (!env->prog->jit_requested) { in add_kfunc_call()
2053 verbose(env, "JIT is required for calling kernel function\n"); in add_kfunc_call()
2058 verbose(env, "JIT does not support calling kernel function\n"); in add_kfunc_call()
2062 if (!env->prog->gpl_compatible) { in add_kfunc_call()
2063 verbose(env, "cannot call kernel function from non-GPL compatible program\n"); in add_kfunc_call()
2089 desc_btf = find_kfunc_desc_btf(env, offset); in add_kfunc_call()
2091 verbose(env, "failed to find BTF for kernel function\n"); in add_kfunc_call()
2095 if (find_kfunc_desc(env->prog, func_id, offset)) in add_kfunc_call()
2099 verbose(env, "too many different kernel function calls\n"); in add_kfunc_call()
2105 verbose(env, "kernel btf_id %u is not a function\n", in add_kfunc_call()
2111 verbose(env, "kernel function btf_id %u does not have a valid func_proto\n", in add_kfunc_call()
2119 verbose(env, "cannot find address for kernel function %s\n", in add_kfunc_call()
2127 verbose(env, "address of kernel function %s is out of range\n", in add_kfunc_call()
2136 err = btf_distill_func_proto(&env->log, desc_btf, in add_kfunc_call()
2191 static int add_subprog_and_kfunc(struct bpf_verifier_env *env) in add_subprog_and_kfunc() argument
2193 struct bpf_subprog_info *subprog = env->subprog_info; in add_subprog_and_kfunc()
2194 struct bpf_insn *insn = env->prog->insnsi; in add_subprog_and_kfunc()
2195 int i, ret, insn_cnt = env->prog->len; in add_subprog_and_kfunc()
2198 ret = add_subprog(env, 0); in add_subprog_and_kfunc()
2207 if (!env->bpf_capable) { in add_subprog_and_kfunc()
2208 …verbose(env, "loading/calling other bpf or kernel functions are allowed for CAP_BPF and CAP_SYS_AD… in add_subprog_and_kfunc()
2213 ret = add_subprog(env, i + insn->imm + 1); in add_subprog_and_kfunc()
2215 ret = add_kfunc_call(env, insn->imm, insn->off); in add_subprog_and_kfunc()
2224 subprog[env->subprog_cnt].start = insn_cnt; in add_subprog_and_kfunc()
2226 if (env->log.level & BPF_LOG_LEVEL2) in add_subprog_and_kfunc()
2227 for (i = 0; i < env->subprog_cnt; i++) in add_subprog_and_kfunc()
2228 verbose(env, "func#%d @%d\n", i, subprog[i].start); in add_subprog_and_kfunc()
2233 static int check_subprogs(struct bpf_verifier_env *env) in check_subprogs() argument
2236 struct bpf_subprog_info *subprog = env->subprog_info; in check_subprogs()
2237 struct bpf_insn *insn = env->prog->insnsi; in check_subprogs()
2238 int insn_cnt = env->prog->len; in check_subprogs()
2259 verbose(env, "jump out of range from insn %d to %d\n", i, off); in check_subprogs()
2270 verbose(env, "last insn is not an exit or jmp\n"); in check_subprogs()
2275 if (cur_subprog < env->subprog_cnt) in check_subprogs()
2285 static int mark_reg_read(struct bpf_verifier_env *env, in mark_reg_read() argument
2297 verbose(env, "verifier BUG type %s var_off %lld off %d\n", in mark_reg_read()
2298 reg_type_str(env, parent->type), in mark_reg_read()
2328 if (env->longest_mark_read_walk < cnt) in mark_reg_read()
2329 env->longest_mark_read_walk = cnt; in mark_reg_read()
2337 static bool is_reg64(struct bpf_verifier_env *env, struct bpf_insn *insn, in is_reg64() argument
2445 static bool insn_has_def32(struct bpf_verifier_env *env, struct bpf_insn *insn) in insn_has_def32() argument
2452 return !is_reg64(env, insn, dst_reg, NULL, DST_OP); in insn_has_def32()
2455 static void mark_insn_zext(struct bpf_verifier_env *env, in mark_insn_zext() argument
2463 env->insn_aux_data[def_idx - 1].zext_dst = true; in mark_insn_zext()
2468 static int check_reg_arg(struct bpf_verifier_env *env, u32 regno, in check_reg_arg() argument
2471 struct bpf_verifier_state *vstate = env->cur_state; in check_reg_arg()
2473 struct bpf_insn *insn = env->prog->insnsi + env->insn_idx; in check_reg_arg()
2478 verbose(env, "R%d is invalid\n", regno); in check_reg_arg()
2482 mark_reg_scratched(env, regno); in check_reg_arg()
2485 rw64 = is_reg64(env, insn, regno, reg, t); in check_reg_arg()
2489 verbose(env, "R%d !read_ok\n", regno); in check_reg_arg()
2497 mark_insn_zext(env, reg); in check_reg_arg()
2499 return mark_reg_read(env, reg, reg->parent, in check_reg_arg()
2504 verbose(env, "frame pointer is read only\n"); in check_reg_arg()
2508 reg->subreg_def = rw64 ? DEF_NOT_SUBREG : env->insn_idx + 1; in check_reg_arg()
2510 mark_reg_unknown(env, regs, regno); in check_reg_arg()
2516 static int push_jmp_history(struct bpf_verifier_env *env, in push_jmp_history() argument
2528 p[cnt - 1].idx = env->insn_idx; in push_jmp_history()
2529 p[cnt - 1].prev_idx = env->prev_insn_idx; in push_jmp_history()
2589 static int backtrack_insn(struct bpf_verifier_env *env, int idx, in backtrack_insn() argument
2595 .private_data = env, in backtrack_insn()
2597 struct bpf_insn *insn = env->prog->insnsi + idx; in backtrack_insn()
2607 if (env->log.level & BPF_LOG_LEVEL2) { in backtrack_insn()
2608 verbose(env, "regs=%x stack=%llx before ", *reg_mask, *stack_mask); in backtrack_insn()
2609 verbose(env, "%d: ", idx); in backtrack_insn()
2610 print_bpf_insn(&cbs, insn, env->allow_ptr_leaks); in backtrack_insn()
2669 verbose(env, "BUG spi %d\n", spi); in backtrack_insn()
2686 verbose(env, "BUG spi %d\n", spi); in backtrack_insn()
2716 verbose(env, "BUG regs %x\n", *reg_mask); in backtrack_insn()
2805 static void mark_all_scalars_precise(struct bpf_verifier_env *env, in mark_all_scalars_precise() argument
2839 static void mark_all_scalars_imprecise(struct bpf_verifier_env *env, struct bpf_verifier_state *st) in mark_all_scalars_imprecise() argument
2951 static int __mark_chain_precision(struct bpf_verifier_env *env, int frame, int regno, in __mark_chain_precision() argument
2954 struct bpf_verifier_state *st = env->cur_state; in __mark_chain_precision()
2956 int last_idx = env->insn_idx; in __mark_chain_precision()
2965 if (!env->bpf_capable) in __mark_chain_precision()
3005 if (env->log.level & BPF_LOG_LEVEL2) in __mark_chain_precision()
3006 verbose(env, "last_idx %d first_idx %d\n", last_idx, first_idx); in __mark_chain_precision()
3030 verbose(env, "BUG backtracing func entry subprog %d reg_mask %x stack_mask %llx\n", in __mark_chain_precision()
3041 err = backtrack_insn(env, i, ®_mask, &stack_mask); in __mark_chain_precision()
3044 mark_all_scalars_precise(env, st); in __mark_chain_precision()
3058 if (i >= env->prog->len) { in __mark_chain_precision()
3065 verbose(env, "BUG backtracking idx %d\n", i); in __mark_chain_precision()
3104 mark_all_scalars_precise(env, st); in __mark_chain_precision()
3121 if (env->log.level & BPF_LOG_LEVEL2) { in __mark_chain_precision()
3122 verbose(env, "parent %s regs=%x stack=%llx marks:", in __mark_chain_precision()
3125 print_verifier_state(env, func, true); in __mark_chain_precision()
3139 int mark_chain_precision(struct bpf_verifier_env *env, int regno) in mark_chain_precision() argument
3141 return __mark_chain_precision(env, env->cur_state->curframe, regno, -1); in mark_chain_precision()
3144 static int mark_chain_precision_frame(struct bpf_verifier_env *env, int frame, int regno) in mark_chain_precision_frame() argument
3146 return __mark_chain_precision(env, frame, regno, -1); in mark_chain_precision_frame()
3149 static int mark_chain_precision_stack_frame(struct bpf_verifier_env *env, int frame, int spi) in mark_chain_precision_stack_frame() argument
3151 return __mark_chain_precision(env, frame, -1, spi); in mark_chain_precision_stack_frame()
3251 static int check_stack_write_fixed_off(struct bpf_verifier_env *env, in check_stack_write_fixed_off() argument
3259 struct bpf_insn *insn = &env->prog->insnsi[insn_idx]; in check_stack_write_fixed_off()
3269 if (!env->allow_ptr_leaks && in check_stack_write_fixed_off()
3272 verbose(env, "attempt to corrupt spilled pointer on stack\n"); in check_stack_write_fixed_off()
3276 cur = env->cur_state->frame[env->cur_state->curframe]; in check_stack_write_fixed_off()
3279 if (!env->bypass_spec_v4) { in check_stack_write_fixed_off()
3292 env->insn_aux_data[insn_idx].sanitize_stack_spill = true; in check_stack_write_fixed_off()
3295 mark_stack_slot_scratched(env, spi); in check_stack_write_fixed_off()
3297 !register_is_null(reg) && env->bpf_capable) { in check_stack_write_fixed_off()
3305 err = mark_chain_precision(env, value_regno); in check_stack_write_fixed_off()
3314 insn->imm != 0 && env->bpf_capable) { in check_stack_write_fixed_off()
3323 verbose_linfo(env, insn_idx, "; "); in check_stack_write_fixed_off()
3324 verbose(env, "invalid size of register spill\n"); in check_stack_write_fixed_off()
3328 verbose(env, "cannot spill pointers to stack into stack frame of the caller\n"); in check_stack_write_fixed_off()
3357 err = mark_chain_precision(env, value_regno); in check_stack_write_fixed_off()
3390 static int check_stack_write_var_off(struct bpf_verifier_env *env, in check_stack_write_var_off() argument
3406 cur = env->cur_state->frame[env->cur_state->curframe]; in check_stack_write_var_off()
3428 mark_stack_slot_scratched(env, spi); in check_stack_write_var_off()
3430 if (!env->allow_ptr_leaks && *stype != STACK_MISC && *stype != STACK_ZERO) { in check_stack_write_var_off()
3442 verbose(env, "spilled ptr in range of var-offset stack write; insn %d, ptr off: %d", in check_stack_write_var_off()
3464 if (*stype == STACK_INVALID && !env->allow_uninit_stack) { in check_stack_write_var_off()
3465 verbose(env, "uninit stack in range of var-offset write prohibited for !root; insn %d, off: %d", in check_stack_write_var_off()
3473 err = mark_chain_precision(env, value_regno); in check_stack_write_var_off()
3488 static void mark_reg_stack_read(struct bpf_verifier_env *env, in mark_reg_stack_read() argument
3493 struct bpf_verifier_state *vstate = env->cur_state; in mark_reg_stack_read()
3525 mark_reg_unknown(env, state->regs, dst_regno); in mark_reg_stack_read()
3539 static int check_stack_read_fixed_off(struct bpf_verifier_env *env, in check_stack_read_fixed_off() argument
3544 struct bpf_verifier_state *vstate = env->cur_state; in check_stack_read_fixed_off()
3561 verbose_linfo(env, env->insn_idx, "; "); in check_stack_read_fixed_off()
3562 verbose(env, "invalid size of register fill\n"); in check_stack_read_fixed_off()
3566 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64); in check_stack_read_fixed_off()
3585 verbose(env, "invalid read from stack off %d+%d size %d\n", in check_stack_read_fixed_off()
3589 mark_reg_unknown(env, state->regs, dst_regno); in check_stack_read_fixed_off()
3603 } else if (__is_pointer_value(env->allow_ptr_leaks, reg)) { in check_stack_read_fixed_off()
3610 verbose(env, "leaking pointer from stack off %d\n", in check_stack_read_fixed_off()
3614 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64); in check_stack_read_fixed_off()
3622 verbose(env, "invalid read from stack off %d+%d size %d\n", in check_stack_read_fixed_off()
3626 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64); in check_stack_read_fixed_off()
3628 mark_reg_stack_read(env, reg_state, off, off + size, dst_regno); in check_stack_read_fixed_off()
3638 static int check_stack_range_initialized(struct bpf_verifier_env *env,
3644 static struct bpf_reg_state *reg_state(struct bpf_verifier_env *env, int regno) in reg_state() argument
3646 return cur_regs(env) + regno; in reg_state()
3662 static int check_stack_read_var_off(struct bpf_verifier_env *env, in check_stack_read_var_off() argument
3666 struct bpf_reg_state *reg = reg_state(env, ptr_regno); in check_stack_read_var_off()
3667 struct bpf_func_state *ptr_state = func(env, reg); in check_stack_read_var_off()
3673 err = check_stack_range_initialized(env, ptr_regno, off, size, in check_stack_read_var_off()
3680 mark_reg_stack_read(env, ptr_state, min_off, max_off + size, dst_regno); in check_stack_read_var_off()
3693 static int check_stack_read(struct bpf_verifier_env *env, in check_stack_read() argument
3697 struct bpf_reg_state *reg = reg_state(env, ptr_regno); in check_stack_read()
3698 struct bpf_func_state *state = func(env, reg); in check_stack_read()
3711 …verbose(env, "variable offset stack pointer cannot be passed into helper function; var_off=%s off=… in check_stack_read()
3726 err = check_stack_read_fixed_off(env, state, off, size, in check_stack_read()
3733 err = check_stack_read_var_off(env, ptr_regno, off, size, in check_stack_read()
3750 static int check_stack_write(struct bpf_verifier_env *env, in check_stack_write() argument
3754 struct bpf_reg_state *reg = reg_state(env, ptr_regno); in check_stack_write()
3755 struct bpf_func_state *state = func(env, reg); in check_stack_write()
3760 err = check_stack_write_fixed_off(env, state, off, size, in check_stack_write()
3766 err = check_stack_write_var_off(env, state, in check_stack_write()
3773 static int check_map_access_type(struct bpf_verifier_env *env, u32 regno, in check_map_access_type() argument
3776 struct bpf_reg_state *regs = cur_regs(env); in check_map_access_type()
3781 verbose(env, "write into map forbidden, value_size=%d off=%d size=%d\n", in check_map_access_type()
3787 verbose(env, "read from map forbidden, value_size=%d off=%d size=%d\n", in check_map_access_type()
3796 static int __check_mem_access(struct bpf_verifier_env *env, int regno, in __check_mem_access() argument
3806 reg = &cur_regs(env)[regno]; in __check_mem_access()
3809 verbose(env, "invalid access to map key, key_size=%d off=%d size=%d\n", in __check_mem_access()
3813 verbose(env, "invalid access to map value, value_size=%d off=%d size=%d\n", in __check_mem_access()
3819 verbose(env, "invalid access to packet, off=%d size=%d, R%d(id=%d,off=%d,r=%d)\n", in __check_mem_access()
3824 verbose(env, "invalid access to memory, mem_size=%u off=%d size=%d\n", in __check_mem_access()
3832 static int check_mem_region_access(struct bpf_verifier_env *env, u32 regno, in check_mem_region_access() argument
3836 struct bpf_verifier_state *vstate = env->cur_state; in check_mem_region_access()
3855 …verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n… in check_mem_region_access()
3859 err = __check_mem_access(env, regno, reg->smin_value + off, size, in check_mem_region_access()
3862 verbose(env, "R%d min value is outside of the allowed memory range\n", in check_mem_region_access()
3872 verbose(env, "R%d unbounded memory access, make sure to bounds check any such access\n", in check_mem_region_access()
3876 err = __check_mem_access(env, regno, reg->umax_value + off, size, in check_mem_region_access()
3879 verbose(env, "R%d max value is outside of the allowed memory range\n", in check_mem_region_access()
3887 static int __check_ptr_off_reg(struct bpf_verifier_env *env, in __check_ptr_off_reg() argument
3896 verbose(env, "negative offset %s ptr R%d off=%d disallowed\n", in __check_ptr_off_reg()
3897 reg_type_str(env, reg->type), regno, reg->off); in __check_ptr_off_reg()
3902 verbose(env, "dereference of modified %s ptr R%d off=%d disallowed\n", in __check_ptr_off_reg()
3903 reg_type_str(env, reg->type), regno, reg->off); in __check_ptr_off_reg()
3911 verbose(env, "variable %s access var_off=%s disallowed\n", in __check_ptr_off_reg()
3912 reg_type_str(env, reg->type), tn_buf); in __check_ptr_off_reg()
3919 int check_ptr_off_reg(struct bpf_verifier_env *env, in check_ptr_off_reg() argument
3922 return __check_ptr_off_reg(env, reg, regno, false); in check_ptr_off_reg()
3925 static int map_kptr_match_type(struct bpf_verifier_env *env, in map_kptr_match_type() argument
3941 verbose(env, "R%d must point to kernel BTF\n", regno); in map_kptr_match_type()
3953 if (__check_ptr_off_reg(env, reg, regno, true)) in map_kptr_match_type()
3980 if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, reg->off, in map_kptr_match_type()
3986 verbose(env, "invalid kptr access, R%d type=%s%s ", regno, in map_kptr_match_type()
3987 reg_type_str(env, reg->type), reg_name); in map_kptr_match_type()
3988 verbose(env, "expected=%s%s", reg_type_str(env, PTR_TO_BTF_ID), targ_name); in map_kptr_match_type()
3990 verbose(env, " or %s%s\n", reg_type_str(env, PTR_TO_BTF_ID | PTR_UNTRUSTED), in map_kptr_match_type()
3993 verbose(env, "\n"); in map_kptr_match_type()
3997 static int check_map_kptr_access(struct bpf_verifier_env *env, u32 regno, in check_map_kptr_access() argument
4001 struct bpf_insn *insn = &env->prog->insnsi[insn_idx]; in check_map_kptr_access()
4013 verbose(env, "kptr in map can only be accessed using BPF_MEM instruction mode\n"); in check_map_kptr_access()
4021 verbose(env, "store to referenced kptr disallowed\n"); in check_map_kptr_access()
4026 val_reg = reg_state(env, value_regno); in check_map_kptr_access()
4030 mark_btf_ld_reg(env, cur_regs(env), value_regno, PTR_TO_BTF_ID, off_desc->kptr.btf, in check_map_kptr_access()
4033 val_reg->id = ++env->id_gen; in check_map_kptr_access()
4035 val_reg = reg_state(env, value_regno); in check_map_kptr_access()
4037 map_kptr_match_type(env, off_desc, val_reg, value_regno)) in check_map_kptr_access()
4041 verbose(env, "BPF_ST imm must be 0 when storing to kptr at off=%u\n", in check_map_kptr_access()
4046 verbose(env, "kptr in map can only be accessed using BPF_LDX/BPF_STX/BPF_ST\n"); in check_map_kptr_access()
4053 static int check_map_access(struct bpf_verifier_env *env, u32 regno, in check_map_access() argument
4057 struct bpf_verifier_state *vstate = env->cur_state; in check_map_access()
4063 err = check_mem_region_access(env, regno, off, size, map->value_size, in check_map_access()
4078 verbose(env, "bpf_spin_lock cannot be accessed directly by load/store\n"); in check_map_access()
4087 verbose(env, "bpf_timer cannot be accessed directly by load/store\n"); in check_map_access()
4101 verbose(env, "kptr cannot be accessed indirectly by helper\n"); in check_map_access()
4105 verbose(env, "kptr access cannot have variable offset\n"); in check_map_access()
4109 verbose(env, "kptr access misaligned expected=%u off=%llu\n", in check_map_access()
4114 verbose(env, "kptr access size must be BPF_DW\n"); in check_map_access()
4126 static bool may_access_direct_pkt_data(struct bpf_verifier_env *env, in may_access_direct_pkt_data() argument
4130 enum bpf_prog_type prog_type = resolve_prog_type(env->prog); in may_access_direct_pkt_data()
4154 env->seen_direct_write = true; in may_access_direct_pkt_data()
4159 env->seen_direct_write = true; in may_access_direct_pkt_data()
4168 static int check_packet_access(struct bpf_verifier_env *env, u32 regno, int off, in check_packet_access() argument
4171 struct bpf_reg_state *regs = cur_regs(env); in check_packet_access()
4184 …verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n… in check_packet_access()
4190 __check_mem_access(env, regno, off, size, reg->range, in check_packet_access()
4193 verbose(env, "R%d offset is outside of the packet\n", regno); in check_packet_access()
4203 env->prog->aux->max_pkt_offset = in check_packet_access()
4204 max_t(u32, env->prog->aux->max_pkt_offset, in check_packet_access()
4211 static int check_ctx_access(struct bpf_verifier_env *env, int insn_idx, int off, int size, in check_ctx_access() argument
4217 .log = &env->log, in check_ctx_access()
4220 if (env->ops->is_valid_access && in check_ctx_access()
4221 env->ops->is_valid_access(off, size, t, env->prog, &info)) { in check_ctx_access()
4235 env->insn_aux_data[insn_idx].ctx_field_size = info.ctx_field_size; in check_ctx_access()
4238 if (env->prog->aux->max_ctx_offset < off + size) in check_ctx_access()
4239 env->prog->aux->max_ctx_offset = off + size; in check_ctx_access()
4243 verbose(env, "invalid bpf_context access off=%d size=%d\n", off, size); in check_ctx_access()
4247 static int check_flow_keys_access(struct bpf_verifier_env *env, int off, in check_flow_keys_access() argument
4252 verbose(env, "invalid access to flow keys off=%d size=%d\n", in check_flow_keys_access()
4259 static int check_sock_access(struct bpf_verifier_env *env, int insn_idx, in check_sock_access() argument
4263 struct bpf_reg_state *regs = cur_regs(env); in check_sock_access()
4269 …verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n… in check_sock_access()
4293 env->insn_aux_data[insn_idx].ctx_field_size = in check_sock_access()
4298 verbose(env, "R%d invalid %s access off=%d size=%d\n", in check_sock_access()
4299 regno, reg_type_str(env, reg->type), off, size); in check_sock_access()
4304 static bool is_pointer_value(struct bpf_verifier_env *env, int regno) in is_pointer_value() argument
4306 return __is_pointer_value(env->allow_ptr_leaks, reg_state(env, regno)); in is_pointer_value()
4309 static bool is_ctx_reg(struct bpf_verifier_env *env, int regno) in is_ctx_reg() argument
4311 const struct bpf_reg_state *reg = reg_state(env, regno); in is_ctx_reg()
4316 static bool is_sk_reg(struct bpf_verifier_env *env, int regno) in is_sk_reg() argument
4318 const struct bpf_reg_state *reg = reg_state(env, regno); in is_sk_reg()
4323 static bool is_pkt_reg(struct bpf_verifier_env *env, int regno) in is_pkt_reg() argument
4325 const struct bpf_reg_state *reg = reg_state(env, regno); in is_pkt_reg()
4330 static bool is_flow_key_reg(struct bpf_verifier_env *env, int regno) in is_flow_key_reg() argument
4332 const struct bpf_reg_state *reg = reg_state(env, regno); in is_flow_key_reg()
4338 static int check_pkt_ptr_alignment(struct bpf_verifier_env *env, in check_pkt_ptr_alignment() argument
4364 verbose(env, in check_pkt_ptr_alignment()
4373 static int check_generic_ptr_alignment(struct bpf_verifier_env *env, in check_generic_ptr_alignment() argument
4389 verbose(env, "misaligned %saccess off %s+%d+%d size %d\n", in check_generic_ptr_alignment()
4397 static int check_ptr_alignment(struct bpf_verifier_env *env, in check_ptr_alignment() argument
4401 bool strict = env->strict_alignment || strict_alignment_once; in check_ptr_alignment()
4410 return check_pkt_ptr_alignment(env, reg, off, size, strict); in check_ptr_alignment()
4446 return check_generic_ptr_alignment(env, reg, pointer_desc, off, size, in check_ptr_alignment()
4450 static int update_stack_depth(struct bpf_verifier_env *env, in update_stack_depth() argument
4454 u16 stack = env->subprog_info[func->subprogno].stack_depth; in update_stack_depth()
4460 env->subprog_info[func->subprogno].stack_depth = -off; in update_stack_depth()
4470 static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx) in check_max_stack_depth_subprog() argument
4472 struct bpf_subprog_info *subprog = env->subprog_info; in check_max_stack_depth_subprog()
4473 struct bpf_insn *insn = env->prog->insnsi; in check_max_stack_depth_subprog()
4502 verbose(env, in check_max_stack_depth_subprog()
4512 verbose(env, "combined stack size of %d calls is %d. Too large\n", in check_max_stack_depth_subprog()
4529 sidx = find_subprog(env, next_insn); in check_max_stack_depth_subprog()
4537 verbose(env, "verifier bug. subprog has tail_call and async cb\n"); in check_max_stack_depth_subprog()
4552 verbose(env, "the call stack of %d frames is too deep !\n", in check_max_stack_depth_subprog()
4567 env->prog->aux->tail_call_reachable = true; in check_max_stack_depth_subprog()
4581 static int check_max_stack_depth(struct bpf_verifier_env *env) in check_max_stack_depth() argument
4583 struct bpf_subprog_info *si = env->subprog_info; in check_max_stack_depth()
4586 for (int i = 0; i < env->subprog_cnt; i++) { in check_max_stack_depth()
4588 ret = check_max_stack_depth_subprog(env, i); in check_max_stack_depth()
4598 static int get_callee_stack_depth(struct bpf_verifier_env *env, in get_callee_stack_depth() argument
4603 subprog = find_subprog(env, start); in get_callee_stack_depth()
4609 return env->subprog_info[subprog].stack_depth; in get_callee_stack_depth()
4613 static int __check_buffer_access(struct bpf_verifier_env *env, in __check_buffer_access() argument
4619 verbose(env, in __check_buffer_access()
4628 verbose(env, in __check_buffer_access()
4637 static int check_tp_buffer_access(struct bpf_verifier_env *env, in check_tp_buffer_access() argument
4643 err = __check_buffer_access(env, "tracepoint", reg, regno, off, size); in check_tp_buffer_access()
4647 if (off + size > env->prog->aux->max_tp_access) in check_tp_buffer_access()
4648 env->prog->aux->max_tp_access = off + size; in check_tp_buffer_access()
4653 static int check_buffer_access(struct bpf_verifier_env *env, in check_buffer_access() argument
4662 err = __check_buffer_access(env, buf_info, reg, regno, off, size); in check_buffer_access()
4760 static int check_ptr_to_btf_access(struct bpf_verifier_env *env, in check_ptr_to_btf_access() argument
4774 verbose(env, in check_ptr_to_btf_access()
4783 verbose(env, in check_ptr_to_btf_access()
4790 verbose(env, in check_ptr_to_btf_access()
4797 verbose(env, in check_ptr_to_btf_access()
4803 if (env->ops->btf_struct_access) { in check_ptr_to_btf_access()
4804 ret = env->ops->btf_struct_access(&env->log, reg->btf, t, in check_ptr_to_btf_access()
4808 verbose(env, "only read is supported\n"); in check_ptr_to_btf_access()
4812 ret = btf_struct_access(&env->log, reg->btf, t, off, size, in check_ptr_to_btf_access()
4826 mark_btf_ld_reg(env, regs, value_regno, ret, reg->btf, btf_id, flag); in check_ptr_to_btf_access()
4831 static int check_ptr_to_map_access(struct bpf_verifier_env *env, in check_ptr_to_map_access() argument
4846 verbose(env, "map_ptr access not supported without CONFIG_DEBUG_INFO_BTF\n"); in check_ptr_to_map_access()
4851 verbose(env, "map_ptr access not supported for map type %d\n", in check_ptr_to_map_access()
4859 if (!env->allow_ptr_to_map_access) { in check_ptr_to_map_access()
4860 verbose(env, in check_ptr_to_map_access()
4867 verbose(env, "R%d is %s invalid negative access: off=%d\n", in check_ptr_to_map_access()
4873 verbose(env, "only read from %s is supported\n", tname); in check_ptr_to_map_access()
4877 ret = btf_struct_access(&env->log, btf_vmlinux, t, off, size, atype, &btf_id, &flag); in check_ptr_to_map_access()
4882 mark_btf_ld_reg(env, regs, value_regno, ret, btf_vmlinux, btf_id, flag); in check_ptr_to_map_access()
4915 struct bpf_verifier_env *env, in check_stack_access_within_bounds() argument
4919 struct bpf_reg_state *regs = cur_regs(env); in check_stack_access_within_bounds()
4921 struct bpf_func_state *state = func(env, reg); in check_stack_access_within_bounds()
4940 verbose(env, "invalid unbounded variable-offset%s stack R%d\n", in check_stack_access_within_bounds()
4954 verbose(env, "invalid%s stack R%d off=%d size=%d\n", in check_stack_access_within_bounds()
4960 verbose(env, "invalid variable-offset%s stack R%d var_off=%s size=%d\n", in check_stack_access_within_bounds()
4973 static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regno, in check_mem_access() argument
4977 struct bpf_reg_state *regs = cur_regs(env); in check_mem_access()
4987 err = check_ptr_alignment(env, reg, off, size, strict_alignment_once); in check_mem_access()
4996 verbose(env, "write to change key R%d not allowed\n", regno); in check_mem_access()
5000 err = check_mem_region_access(env, regno, off, size, in check_mem_access()
5005 mark_reg_unknown(env, regs, value_regno); in check_mem_access()
5010 is_pointer_value(env, value_regno)) { in check_mem_access()
5011 verbose(env, "R%d leaks addr into map\n", value_regno); in check_mem_access()
5014 err = check_map_access_type(env, regno, off, size, t); in check_mem_access()
5017 err = check_map_access(env, regno, off, size, false, ACCESS_DIRECT); in check_mem_access()
5024 err = check_map_kptr_access(env, regno, value_regno, insn_idx, in check_mem_access()
5044 mark_reg_unknown(env, regs, value_regno); in check_mem_access()
5051 verbose(env, "R%d invalid mem access '%s'\n", regno, in check_mem_access()
5052 reg_type_str(env, reg->type)); in check_mem_access()
5057 verbose(env, "R%d cannot write into %s\n", in check_mem_access()
5058 regno, reg_type_str(env, reg->type)); in check_mem_access()
5063 is_pointer_value(env, value_regno)) { in check_mem_access()
5064 verbose(env, "R%d leaks addr into mem\n", value_regno); in check_mem_access()
5068 err = check_mem_region_access(env, regno, off, size, in check_mem_access()
5071 mark_reg_unknown(env, regs, value_regno); in check_mem_access()
5078 is_pointer_value(env, value_regno)) { in check_mem_access()
5079 verbose(env, "R%d leaks addr into ctx\n", value_regno); in check_mem_access()
5083 err = check_ptr_off_reg(env, reg, regno); in check_mem_access()
5087 err = check_ctx_access(env, insn_idx, off, size, t, ®_type, &btf, in check_mem_access()
5090 verbose_linfo(env, insn_idx, "; "); in check_mem_access()
5097 mark_reg_unknown(env, regs, value_regno); in check_mem_access()
5099 mark_reg_known_zero(env, regs, in check_mem_access()
5102 regs[value_regno].id = ++env->id_gen; in check_mem_access()
5119 err = check_stack_access_within_bounds(env, regno, off, size, ACCESS_DIRECT, t); in check_mem_access()
5123 state = func(env, reg); in check_mem_access()
5124 err = update_stack_depth(env, state, off); in check_mem_access()
5129 err = check_stack_read(env, regno, off, size, in check_mem_access()
5132 err = check_stack_write(env, regno, off, size, in check_mem_access()
5135 if (t == BPF_WRITE && !may_access_direct_pkt_data(env, NULL, t)) { in check_mem_access()
5136 verbose(env, "cannot write into packet\n"); in check_mem_access()
5140 is_pointer_value(env, value_regno)) { in check_mem_access()
5141 verbose(env, "R%d leaks addr into packet\n", in check_mem_access()
5145 err = check_packet_access(env, regno, off, size, false); in check_mem_access()
5147 mark_reg_unknown(env, regs, value_regno); in check_mem_access()
5150 is_pointer_value(env, value_regno)) { in check_mem_access()
5151 verbose(env, "R%d leaks addr into flow keys\n", in check_mem_access()
5156 err = check_flow_keys_access(env, off, size); in check_mem_access()
5158 mark_reg_unknown(env, regs, value_regno); in check_mem_access()
5161 verbose(env, "R%d cannot write into %s\n", in check_mem_access()
5162 regno, reg_type_str(env, reg->type)); in check_mem_access()
5165 err = check_sock_access(env, insn_idx, regno, off, size, t); in check_mem_access()
5167 mark_reg_unknown(env, regs, value_regno); in check_mem_access()
5169 err = check_tp_buffer_access(env, reg, regno, off, size); in check_mem_access()
5171 mark_reg_unknown(env, regs, value_regno); in check_mem_access()
5174 err = check_ptr_to_btf_access(env, regs, regno, off, size, t, in check_mem_access()
5177 err = check_ptr_to_map_access(env, regs, regno, off, size, t, in check_mem_access()
5185 verbose(env, "R%d cannot write into %s\n", in check_mem_access()
5186 regno, reg_type_str(env, reg->type)); in check_mem_access()
5189 max_access = &env->prog->aux->max_rdonly_access; in check_mem_access()
5191 max_access = &env->prog->aux->max_rdwr_access; in check_mem_access()
5194 err = check_buffer_access(env, reg, regno, off, size, false, in check_mem_access()
5198 mark_reg_unknown(env, regs, value_regno); in check_mem_access()
5200 verbose(env, "R%d invalid mem access '%s'\n", regno, in check_mem_access()
5201 reg_type_str(env, reg->type)); in check_mem_access()
5213 static int check_atomic(struct bpf_verifier_env *env, int insn_idx, struct bpf_insn *insn) in check_atomic() argument
5231 verbose(env, "BPF_ATOMIC uses invalid atomic opcode %02x\n", insn->imm); in check_atomic()
5236 verbose(env, "invalid atomic operand size\n"); in check_atomic()
5241 err = check_reg_arg(env, insn->src_reg, SRC_OP); in check_atomic()
5246 err = check_reg_arg(env, insn->dst_reg, SRC_OP); in check_atomic()
5254 err = check_reg_arg(env, aux_reg, SRC_OP); in check_atomic()
5258 if (is_pointer_value(env, aux_reg)) { in check_atomic()
5259 verbose(env, "R%d leaks addr into mem\n", aux_reg); in check_atomic()
5264 if (is_pointer_value(env, insn->src_reg)) { in check_atomic()
5265 verbose(env, "R%d leaks addr into mem\n", insn->src_reg); in check_atomic()
5269 if (is_ctx_reg(env, insn->dst_reg) || in check_atomic()
5270 is_pkt_reg(env, insn->dst_reg) || in check_atomic()
5271 is_flow_key_reg(env, insn->dst_reg) || in check_atomic()
5272 is_sk_reg(env, insn->dst_reg)) { in check_atomic()
5273 verbose(env, "BPF_ATOMIC stores into R%d %s is not allowed\n", in check_atomic()
5275 reg_type_str(env, reg_state(env, insn->dst_reg)->type)); in check_atomic()
5286 err = check_reg_arg(env, load_reg, DST_OP); in check_atomic()
5299 err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off, in check_atomic()
5302 err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off, in check_atomic()
5309 err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off, in check_atomic()
5327 struct bpf_verifier_env *env, int regno, int off, in check_stack_range_initialized() argument
5331 struct bpf_reg_state *reg = reg_state(env, regno); in check_stack_range_initialized()
5332 struct bpf_func_state *state = func(env, reg); in check_stack_range_initialized()
5342 verbose(env, "invalid zero-sized read\n"); in check_stack_range_initialized()
5356 err = check_stack_access_within_bounds(env, regno, off, access_size, in check_stack_range_initialized()
5370 if (!env->bypass_spec_v1) { in check_stack_range_initialized()
5374 verbose(env, "R%d%s variable offset stack access prohibited for !root, var_off=%s\n", in check_stack_range_initialized()
5417 env->allow_ptr_leaks)) { in check_stack_range_initialized()
5419 __mark_reg_unknown(env, &state->stack[spi].spilled_ptr); in check_stack_range_initialized()
5428 verbose(env, "invalid%s read from stack R%d off %d+%d size %d\n", in check_stack_range_initialized()
5434 verbose(env, "invalid%s read from stack R%d var_off %s+%d size %d\n", in check_stack_range_initialized()
5442 mark_reg_read(env, &state->stack[spi].spilled_ptr, in check_stack_range_initialized()
5451 return update_stack_depth(env, state, min_off); in check_stack_range_initialized()
5454 static int check_helper_mem_access(struct bpf_verifier_env *env, int regno, in check_helper_mem_access() argument
5458 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; in check_helper_mem_access()
5464 return check_packet_access(env, regno, reg->off, access_size, in check_helper_mem_access()
5468 verbose(env, "R%d cannot write into %s\n", regno, in check_helper_mem_access()
5469 reg_type_str(env, reg->type)); in check_helper_mem_access()
5472 return check_mem_region_access(env, regno, reg->off, access_size, in check_helper_mem_access()
5475 if (check_map_access_type(env, regno, reg->off, access_size, in check_helper_mem_access()
5479 return check_map_access(env, regno, reg->off, access_size, in check_helper_mem_access()
5484 verbose(env, "R%d cannot write into %s\n", regno, in check_helper_mem_access()
5485 reg_type_str(env, reg->type)); in check_helper_mem_access()
5489 return check_mem_region_access(env, regno, reg->off, in check_helper_mem_access()
5495 verbose(env, "R%d cannot write into %s\n", regno, in check_helper_mem_access()
5496 reg_type_str(env, reg->type)); in check_helper_mem_access()
5500 max_access = &env->prog->aux->max_rdonly_access; in check_helper_mem_access()
5502 max_access = &env->prog->aux->max_rdwr_access; in check_helper_mem_access()
5504 return check_buffer_access(env, reg, regno, reg->off, in check_helper_mem_access()
5509 env, in check_helper_mem_access()
5518 if (!env->ops->convert_ctx_access) { in check_helper_mem_access()
5526 return check_mem_access(env, env->insn_idx, regno, offset, BPF_B, in check_helper_mem_access()
5537 verbose(env, "R%d type=%s ", regno, in check_helper_mem_access()
5538 reg_type_str(env, reg->type)); in check_helper_mem_access()
5539 verbose(env, "expected=%s\n", reg_type_str(env, PTR_TO_STACK)); in check_helper_mem_access()
5544 static int check_mem_size_reg(struct bpf_verifier_env *env, in check_mem_size_reg() argument
5573 verbose(env, "R%d min value is negative, either use unsigned or 'var &= const'\n", in check_mem_size_reg()
5579 err = check_helper_mem_access(env, regno - 1, 0, in check_mem_size_reg()
5587 verbose(env, "R%d unbounded memory access, use 'var &= const' or 'if (var < const)'\n", in check_mem_size_reg()
5591 err = check_helper_mem_access(env, regno - 1, in check_mem_size_reg()
5595 err = mark_chain_precision(env, regno); in check_mem_size_reg()
5599 int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, in check_mem_reg() argument
5620 err = check_helper_mem_access(env, regno, mem_size, true, &meta); in check_mem_reg()
5623 err = err ?: check_helper_mem_access(env, regno, mem_size, true, &meta); in check_mem_reg()
5631 int check_kfunc_mem_size_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, in check_kfunc_mem_size_reg() argument
5634 struct bpf_reg_state *mem_reg = &cur_regs(env)[regno - 1]; in check_kfunc_mem_size_reg()
5649 err = check_mem_size_reg(env, reg, regno, true, &meta); in check_kfunc_mem_size_reg()
5652 err = err ?: check_mem_size_reg(env, reg, regno, true, &meta); in check_kfunc_mem_size_reg()
5678 static int process_spin_lock(struct bpf_verifier_env *env, int regno, in process_spin_lock() argument
5681 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; in process_spin_lock()
5682 struct bpf_verifier_state *cur = env->cur_state; in process_spin_lock()
5688 verbose(env, in process_spin_lock()
5694 verbose(env, in process_spin_lock()
5701 verbose(env, in process_spin_lock()
5705 verbose(env, in process_spin_lock()
5709 verbose(env, in process_spin_lock()
5715 verbose(env, "off %lld doesn't point to 'struct bpf_spin_lock'\n", in process_spin_lock()
5721 verbose(env, in process_spin_lock()
5728 verbose(env, "bpf_spin_unlock without taking a lock\n"); in process_spin_lock()
5732 verbose(env, "bpf_spin_unlock of different lock\n"); in process_spin_lock()
5740 static int process_timer_func(struct bpf_verifier_env *env, int regno, in process_timer_func() argument
5743 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; in process_timer_func()
5749 verbose(env, in process_timer_func()
5755 verbose(env, "map '%s' has to have BTF in order to use bpf_timer\n", in process_timer_func()
5761 verbose(env, in process_timer_func()
5765 verbose(env, in process_timer_func()
5769 verbose(env, in process_timer_func()
5775 verbose(env, "off %lld doesn't point to 'struct bpf_timer' that is at %d\n", in process_timer_func()
5780 verbose(env, "verifier bug. Two map pointers in a timer helper\n"); in process_timer_func()
5788 static int process_kptr_func(struct bpf_verifier_env *env, int regno, in process_kptr_func() argument
5791 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; in process_kptr_func()
5798 verbose(env, in process_kptr_func()
5804 verbose(env, "map '%s' has to have BTF in order to use bpf_kptr_xchg\n", in process_kptr_func()
5811 verbose(env, "map '%s' has more than %d kptr\n", map_ptr->name, in process_kptr_func()
5814 verbose(env, "map '%s' has repeating kptr BTF tags\n", map_ptr->name); in process_kptr_func()
5816 verbose(env, "map '%s' has no valid kptr\n", map_ptr->name); in process_kptr_func()
5824 verbose(env, "off=%d doesn't point to kptr\n", kptr_off); in process_kptr_func()
5828 verbose(env, "off=%d kptr isn't referenced kptr\n", kptr_off); in process_kptr_func()
5861 static int resolve_map_arg_type(struct bpf_verifier_env *env, in resolve_map_arg_type() argument
5867 verbose(env, "invalid map_ptr to access map->type\n"); in resolve_map_arg_type()
5877 verbose(env, "invalid arg_type for sockmap/sockhash\n"); in resolve_map_arg_type()
5999 static int check_reg_type(struct bpf_verifier_env *env, u32 regno, in check_reg_type() argument
6004 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; in check_reg_type()
6011 verbose(env, "verifier internal error: unsupported arg type %d\n", arg_type); in check_reg_type()
6039 verbose(env, "R%d type=%s expected=", regno, reg_type_str(env, reg->type)); in check_reg_type()
6041 verbose(env, "%s, ", reg_type_str(env, compatible->types[j])); in check_reg_type()
6042 verbose(env, "%s\n", reg_type_str(env, compatible->types[j])); in check_reg_type()
6056 verbose(env, "verifier internal error: missing arg compatible BTF ID\n"); in check_reg_type()
6063 if (map_kptr_match_type(env, meta->kptr_off_desc, reg, regno)) in check_reg_type()
6067 verbose(env, "verifier internal error:"); in check_reg_type()
6068 verbose(env, "R%d has non-overwritten BPF_PTR_POISON type\n", in check_reg_type()
6073 if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, reg->off, in check_reg_type()
6076 verbose(env, "R%d is of type %s but %s is expected\n", in check_reg_type()
6087 int check_func_arg_reg_off(struct bpf_verifier_env *env, in check_func_arg_reg_off() argument
6098 verbose(env, "cannot pass in dynptr at an offset\n"); in check_func_arg_reg_off()
6127 verbose(env, "R%d must have zero offset when passed to release func\n", in check_func_arg_reg_off()
6140 return __check_ptr_off_reg(env, reg, regno, fixed_off_ok); in check_func_arg_reg_off()
6143 static u32 stack_slot_get_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg) in stack_slot_get_id() argument
6145 struct bpf_func_state *state = func(env, reg); in stack_slot_get_id()
6151 static int check_func_arg(struct bpf_verifier_env *env, u32 arg, in check_func_arg() argument
6156 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; in check_func_arg()
6165 err = check_reg_arg(env, regno, SRC_OP); in check_func_arg()
6170 if (is_pointer_value(env, regno)) { in check_func_arg()
6171 verbose(env, "R%d leaks addr into helper function\n", in check_func_arg()
6179 !may_access_direct_pkt_data(env, meta, BPF_READ)) { in check_func_arg()
6180 verbose(env, "helper access to the packet is not allowed\n"); in check_func_arg()
6185 err = resolve_map_arg_type(env, meta, &arg_type); in check_func_arg()
6200 err = check_reg_type(env, regno, arg_type, arg_btf_id, meta); in check_func_arg()
6204 err = check_func_arg_reg_off(env, reg, regno, arg_type); in check_func_arg()
6211 struct bpf_func_state *state = func(env, reg); in check_func_arg()
6216 verbose(env, "arg %d is an unacquired reference\n", regno); in check_func_arg()
6220 verbose(env, "R%d must be referenced when passed to release function\n", in check_func_arg()
6225 verbose(env, "verifier internal error: more than one release argument\n"); in check_func_arg()
6233 verbose(env, "verifier internal error: more than one arg with ref_obj_id R%d %u %u\n", in check_func_arg()
6259 verbose(env, in check_func_arg()
6279 verbose(env, "invalid map_ptr to access map->key\n"); in check_func_arg()
6282 err = check_helper_mem_access(env, regno, in check_func_arg()
6295 verbose(env, "invalid map_ptr to access map->value\n"); in check_func_arg()
6299 err = check_helper_mem_access(env, regno, in check_func_arg()
6305 verbose(env, "Helper has invalid btf_id in R%d\n", regno); in check_func_arg()
6313 if (process_spin_lock(env, regno, true)) in check_func_arg()
6316 if (process_spin_lock(env, regno, false)) in check_func_arg()
6319 verbose(env, "verifier internal error\n"); in check_func_arg()
6324 if (process_timer_func(env, regno, meta)) in check_func_arg()
6336 err = check_helper_mem_access(env, regno, in check_func_arg()
6342 err = check_mem_size_reg(env, reg, regno, false, meta); in check_func_arg()
6345 err = check_mem_size_reg(env, reg, regno, true, meta); in check_func_arg()
6356 if (!is_dynptr_reg_valid_uninit(env, reg)) { in check_func_arg()
6357 verbose(env, "Dynptr has to be an uninitialized dynptr\n"); in check_func_arg()
6365 verbose(env, "verifier internal error: multiple uninitialized dynptr args\n"); in check_func_arg()
6370 } else if (!is_dynptr_reg_valid_init(env, reg)) { in check_func_arg()
6371 verbose(env, in check_func_arg()
6375 } else if (!is_dynptr_type_expected(env, reg, arg_type)) { in check_func_arg()
6389 verbose(env, in check_func_arg()
6397 verbose(env, "R%d is not a known constant'\n", in check_func_arg()
6402 err = mark_chain_precision(env, regno); in check_func_arg()
6411 err = check_helper_mem_access(env, regno, size, false, meta); in check_func_arg()
6414 err = check_ptr_alignment(env, reg, 0, size, true); in check_func_arg()
6425 verbose(env, "R%d does not point to a readonly map'\n", regno); in check_func_arg()
6430 verbose(env, "R%d is not a constant address'\n", regno); in check_func_arg()
6435 verbose(env, "no direct value access support for this map type\n"); in check_func_arg()
6439 err = check_map_access(env, regno, reg->off, in check_func_arg()
6448 verbose(env, "direct value access on string failed\n"); in check_func_arg()
6454 verbose(env, "string is not zero-terminated\n"); in check_func_arg()
6460 if (process_kptr_func(env, regno, meta)) in check_func_arg()
6468 static bool may_update_sockmap(struct bpf_verifier_env *env, int func_id) in may_update_sockmap() argument
6470 enum bpf_attach_type eatype = env->prog->expected_attach_type; in may_update_sockmap()
6471 enum bpf_prog_type type = resolve_prog_type(env->prog); in may_update_sockmap()
6496 verbose(env, "cannot update sockmap in this context\n"); in may_update_sockmap()
6500 static bool allow_tail_call_in_subprogs(struct bpf_verifier_env *env) in allow_tail_call_in_subprogs() argument
6502 return env->prog->jit_requested && in allow_tail_call_in_subprogs()
6506 static int check_map_func_compatibility(struct bpf_verifier_env *env, in check_map_func_compatibility() argument
6583 !may_update_sockmap(env, func_id)) in check_map_func_compatibility()
6593 !may_update_sockmap(env, func_id)) in check_map_func_compatibility()
6636 if (env->subprog_cnt > 1 && !allow_tail_call_in_subprogs(env)) { in check_map_func_compatibility()
6637 verbose(env, "tail_calls are not allowed in non-JITed programs with bpf-to-bpf calls\n"); in check_map_func_compatibility()
6740 verbose(env, "cannot pass map_type %d into func %s#%d\n", in check_map_func_compatibility()
6828 static void clear_all_pkt_pointers(struct bpf_verifier_env *env) in clear_all_pkt_pointers() argument
6833 bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({ in clear_all_pkt_pointers()
6835 __mark_reg_unknown(env, reg); in clear_all_pkt_pointers()
6868 static int release_reference(struct bpf_verifier_env *env, in release_reference() argument
6875 err = release_reference_state(cur_func(env), ref_obj_id); in release_reference()
6879 bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({ in release_reference()
6881 if (!env->allow_ptr_leaks) in release_reference()
6882 __mark_reg_not_init(env, reg); in release_reference()
6884 __mark_reg_unknown(env, reg); in release_reference()
6891 static void clear_caller_saved_regs(struct bpf_verifier_env *env, in clear_caller_saved_regs() argument
6898 mark_reg_not_init(env, regs, caller_saved[i]); in clear_caller_saved_regs()
6899 check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); in clear_caller_saved_regs()
6903 typedef int (*set_callee_state_fn)(struct bpf_verifier_env *env,
6908 static int set_callee_state(struct bpf_verifier_env *env,
6912 static int __check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, in __check_func_call() argument
6916 struct bpf_verifier_state *state = env->cur_state; in __check_func_call()
6923 verbose(env, "the call stack of %d frames is too deep\n", in __check_func_call()
6930 verbose(env, "verifier bug. Frame %d already allocated\n", in __check_func_call()
6935 func_info_aux = env->prog->aux->func_info_aux; in __check_func_call()
6938 err = btf_check_subprog_call(env, subprog, caller->regs); in __check_func_call()
6943 verbose(env, "Caller passes invalid args into func#%d\n", in __check_func_call()
6947 if (env->log.level & BPF_LOG_LEVEL) in __check_func_call()
6948 verbose(env, in __check_func_call()
6951 clear_caller_saved_regs(env, caller->regs); in __check_func_call()
6954 mark_reg_unknown(env, caller->regs, BPF_REG_0); in __check_func_call()
6967 verbose(env, "verifier bug: helper %s#%d is not marked as callback-calling\n", in __check_func_call()
6978 env->subprog_info[subprog].is_async_cb = true; in __check_func_call()
6979 async_cb = push_async_cb(env, env->subprog_info[subprog].start, in __check_func_call()
6987 err = set_callee_state_cb(env, caller, callee, *insn_idx); in __check_func_call()
6991 clear_caller_saved_regs(env, caller->regs); in __check_func_call()
6992 mark_reg_unknown(env, caller->regs, BPF_REG_0); in __check_func_call()
7007 init_func_state(env, callee, in __check_func_call()
7018 err = set_callee_state_cb(env, caller, callee, *insn_idx); in __check_func_call()
7022 clear_caller_saved_regs(env, caller->regs); in __check_func_call()
7028 *insn_idx = env->subprog_info[subprog].start - 1; in __check_func_call()
7030 if (env->log.level & BPF_LOG_LEVEL) { in __check_func_call()
7031 verbose(env, "caller:\n"); in __check_func_call()
7032 print_verifier_state(env, caller, true); in __check_func_call()
7033 verbose(env, "callee:\n"); in __check_func_call()
7034 print_verifier_state(env, callee, true); in __check_func_call()
7044 int map_set_for_each_callback_args(struct bpf_verifier_env *env, in map_set_for_each_callback_args() argument
7067 __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); in map_set_for_each_callback_args()
7071 static int set_callee_state(struct bpf_verifier_env *env, in set_callee_state() argument
7085 static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, in check_func_call() argument
7091 subprog = find_subprog(env, target_insn); in check_func_call()
7093 verbose(env, "verifier bug. No program starts at insn %d\n", in check_func_call()
7098 return __check_func_call(env, insn, insn_idx, subprog, set_callee_state); in check_func_call()
7101 static int set_map_elem_callback_state(struct bpf_verifier_env *env, in set_map_elem_callback_state() argument
7106 struct bpf_insn_aux_data *insn_aux = &env->insn_aux_data[insn_idx]; in set_map_elem_callback_state()
7111 verbose(env, "tail_call abusing map_ptr\n"); in set_map_elem_callback_state()
7118 verbose(env, "callback function not allowed for map\n"); in set_map_elem_callback_state()
7122 err = map->ops->map_set_for_each_callback_args(env, caller, callee); in set_map_elem_callback_state()
7131 static int set_loop_callback_state(struct bpf_verifier_env *env, in set_loop_callback_state() argument
7144 __mark_reg_not_init(env, &callee->regs[BPF_REG_3]); in set_loop_callback_state()
7145 __mark_reg_not_init(env, &callee->regs[BPF_REG_4]); in set_loop_callback_state()
7146 __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); in set_loop_callback_state()
7153 static int set_timer_callback_state(struct bpf_verifier_env *env, in set_timer_callback_state() argument
7176 __mark_reg_not_init(env, &callee->regs[BPF_REG_4]); in set_timer_callback_state()
7177 __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); in set_timer_callback_state()
7183 static int set_find_vma_callback_state(struct bpf_verifier_env *env, in set_find_vma_callback_state() argument
7204 __mark_reg_not_init(env, &callee->regs[BPF_REG_4]); in set_find_vma_callback_state()
7205 __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); in set_find_vma_callback_state()
7211 static int set_user_ringbuf_callback_state(struct bpf_verifier_env *env, in set_user_ringbuf_callback_state() argument
7220 __mark_reg_not_init(env, &callee->regs[BPF_REG_0]); in set_user_ringbuf_callback_state()
7226 __mark_reg_not_init(env, &callee->regs[BPF_REG_3]); in set_user_ringbuf_callback_state()
7227 __mark_reg_not_init(env, &callee->regs[BPF_REG_4]); in set_user_ringbuf_callback_state()
7228 __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); in set_user_ringbuf_callback_state()
7235 static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx) in prepare_func_exit() argument
7237 struct bpf_verifier_state *state = env->cur_state; in prepare_func_exit()
7251 verbose(env, "cannot return stack pointer to the caller\n"); in prepare_func_exit()
7261 verbose(env, "R0 not a scalar value\n"); in prepare_func_exit()
7266 err = mark_reg_read(env, r0, r0->parent, REG_LIVE_READ64); in prepare_func_exit()
7267 err = err ?: mark_chain_precision(env, BPF_REG_0); in prepare_func_exit()
7272 verbose_invalid_scalar(env, r0, &range, "callback return", "R0"); in prepare_func_exit()
7293 if (env->log.level & BPF_LOG_LEVEL) { in prepare_func_exit()
7294 verbose(env, "returning from callee:\n"); in prepare_func_exit()
7295 print_verifier_state(env, callee, true); in prepare_func_exit()
7296 verbose(env, "to caller at %d:\n", *insn_idx); in prepare_func_exit()
7297 print_verifier_state(env, caller, true); in prepare_func_exit()
7327 record_func_map(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, in record_func_map() argument
7330 struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx]; in record_func_map()
7346 verbose(env, "kernel subsystem misconfigured verifier\n"); in record_func_map()
7359 verbose(env, "write into map forbidden\n"); in record_func_map()
7373 record_func_key(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, in record_func_key() argument
7376 struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx]; in record_func_key()
7377 struct bpf_reg_state *regs = cur_regs(env), *reg; in record_func_key()
7385 verbose(env, "kernel subsystem misconfigured verifier\n"); in record_func_key()
7398 err = mark_chain_precision(env, BPF_REG_3); in record_func_key()
7409 static int check_reference_leak(struct bpf_verifier_env *env) in check_reference_leak() argument
7411 struct bpf_func_state *state = cur_func(env); in check_reference_leak()
7421 verbose(env, "Unreleased reference id=%d alloc_insn=%d\n", in check_reference_leak()
7428 static int check_bpf_snprintf_call(struct bpf_verifier_env *env, in check_bpf_snprintf_call() argument
7451 verbose(env, "verifier bug\n"); in check_bpf_snprintf_call()
7461 verbose(env, "Invalid format string\n"); in check_bpf_snprintf_call()
7466 static int check_get_func_ip(struct bpf_verifier_env *env) in check_get_func_ip() argument
7468 enum bpf_prog_type type = resolve_prog_type(env->prog); in check_get_func_ip()
7472 if (!bpf_prog_has_trampoline(env->prog)) { in check_get_func_ip()
7473 verbose(env, "func %s#%d supported only for fentry/fexit/fmod_ret programs\n", in check_get_func_ip()
7482 verbose(env, "func %s#%d not supported for program type %d\n", in check_get_func_ip()
7487 static struct bpf_insn_aux_data *cur_aux(struct bpf_verifier_env *env) in cur_aux() argument
7489 return &env->insn_aux_data[env->insn_idx]; in cur_aux()
7492 static bool loop_flag_is_zero(struct bpf_verifier_env *env) in loop_flag_is_zero() argument
7494 struct bpf_reg_state *regs = cur_regs(env); in loop_flag_is_zero()
7499 mark_chain_precision(env, BPF_REG_4); in loop_flag_is_zero()
7504 static void update_loop_inline_state(struct bpf_verifier_env *env, u32 subprogno) in update_loop_inline_state() argument
7506 struct bpf_loop_inline_state *state = &cur_aux(env)->loop_inline_state; in update_loop_inline_state()
7510 state->fit_for_inline = loop_flag_is_zero(env); in update_loop_inline_state()
7518 state->fit_for_inline = (loop_flag_is_zero(env) && in update_loop_inline_state()
7522 static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn, in check_helper_call() argument
7525 enum bpf_prog_type prog_type = resolve_prog_type(env->prog); in check_helper_call()
7538 verbose(env, "invalid func %s#%d\n", func_id_name(func_id), in check_helper_call()
7543 if (env->ops->get_func_proto) in check_helper_call()
7544 fn = env->ops->get_func_proto(func_id, env->prog); in check_helper_call()
7546 verbose(env, "unknown func %s#%d\n", func_id_name(func_id), in check_helper_call()
7552 if (!env->prog->gpl_compatible && fn->gpl_only) { in check_helper_call()
7553 verbose(env, "cannot call GPL-restricted function from non-GPL compatible program\n"); in check_helper_call()
7557 if (fn->allowed && !fn->allowed(env->prog)) { in check_helper_call()
7558 verbose(env, "helper call is not allowed in probe\n"); in check_helper_call()
7565 verbose(env, "kernel subsystem misconfigured func %s#%d: r1 != ctx\n", in check_helper_call()
7575 verbose(env, "kernel subsystem misconfigured func %s#%d\n", in check_helper_call()
7583 err = check_func_arg(env, i, &meta, fn); in check_helper_call()
7588 err = record_func_map(env, &meta, func_id, insn_idx); in check_helper_call()
7592 err = record_func_key(env, &meta, func_id, insn_idx); in check_helper_call()
7600 err = check_mem_access(env, insn_idx, meta.regno, i, BPF_B, in check_helper_call()
7606 regs = cur_regs(env); in check_helper_call()
7611 err = check_mem_access(env, insn_idx, meta.uninit_dynptr_regno, in check_helper_call()
7617 err = mark_stack_slots_dynptr(env, ®s[meta.uninit_dynptr_regno], in check_helper_call()
7627 err = unmark_stack_slots_dynptr(env, ®s[meta.release_regno]); in check_helper_call()
7629 err = release_reference(env, meta.ref_obj_id); in check_helper_call()
7636 verbose(env, "func %s#%d reference has not been acquired before\n", in check_helper_call()
7644 err = check_reference_leak(env); in check_helper_call()
7646 verbose(env, "tail_call would lead to reference leak\n"); in check_helper_call()
7655 verbose(env, "get_local_storage() doesn't support non-zero flags\n"); in check_helper_call()
7660 err = __check_func_call(env, insn, insn_idx_p, meta.subprogno, in check_helper_call()
7664 err = __check_func_call(env, insn, insn_idx_p, meta.subprogno, in check_helper_call()
7668 err = __check_func_call(env, insn, insn_idx_p, meta.subprogno, in check_helper_call()
7672 err = check_bpf_snprintf_call(env, regs); in check_helper_call()
7675 update_loop_inline_state(env, meta.subprogno); in check_helper_call()
7676 err = __check_func_call(env, insn, insn_idx_p, meta.subprogno, in check_helper_call()
7681 verbose(env, "Unsupported reg type %s for bpf_dynptr_from_mem data\n", in check_helper_call()
7682 reg_type_str(env, regs[BPF_REG_1].type)); in check_helper_call()
7688 env->prog->expected_attach_type == BPF_LSM_CGROUP) { in check_helper_call()
7689 if (!env->prog->aux->attach_func_proto->type) { in check_helper_call()
7693 verbose(env, "BPF_LSM_CGROUP that attach to void LSM hooks can't modify return value!\n"); in check_helper_call()
7704 verbose(env, "verifier internal error: meta.ref_obj_id already set\n"); in check_helper_call()
7712 meta.ref_obj_id = stack_slot_get_id(env, reg); in check_helper_call()
7717 verbose(env, "verifier internal error: no dynptr in bpf_dynptr_data()\n"); in check_helper_call()
7722 err = __check_func_call(env, insn, insn_idx_p, meta.subprogno, in check_helper_call()
7732 mark_reg_not_init(env, regs, caller_saved[i]); in check_helper_call()
7733 check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); in check_helper_call()
7746 mark_reg_unknown(env, regs, BPF_REG_0); in check_helper_call()
7753 mark_reg_known_zero(env, regs, BPF_REG_0); in check_helper_call()
7759 verbose(env, in check_helper_call()
7768 regs[BPF_REG_0].id = ++env->id_gen; in check_helper_call()
7772 mark_reg_known_zero(env, regs, BPF_REG_0); in check_helper_call()
7776 mark_reg_known_zero(env, regs, BPF_REG_0); in check_helper_call()
7780 mark_reg_known_zero(env, regs, BPF_REG_0); in check_helper_call()
7784 mark_reg_known_zero(env, regs, BPF_REG_0); in check_helper_call()
7792 mark_reg_known_zero(env, regs, BPF_REG_0); in check_helper_call()
7803 verbose(env, "unable to resolve the size of type '%s': %ld\n", in check_helper_call()
7828 mark_reg_known_zero(env, regs, BPF_REG_0); in check_helper_call()
7835 verbose(env, "verifier internal error:"); in check_helper_call()
7836 verbose(env, "func %s has non-overwritten BPF_PTR_POISON return type\n", in check_helper_call()
7844 verbose(env, "invalid return type %u of func %s#%d\n", in check_helper_call()
7854 verbose(env, "unknown return type %u of func %s#%d\n", in check_helper_call()
7860 regs[BPF_REG_0].id = ++env->id_gen; in check_helper_call()
7863 verbose(env, "verifier internal error: func %s#%d sets ref_obj_id more than once\n", in check_helper_call()
7872 int id = acquire_reference_state(env, insn_idx); in check_helper_call()
7884 err = check_map_func_compatibility(env, meta.map_ptr, func_id); in check_helper_call()
7890 !env->prog->has_callchain_buf) { in check_helper_call()
7901 verbose(env, err_str, func_id_name(func_id), func_id); in check_helper_call()
7905 env->prog->has_callchain_buf = true; in check_helper_call()
7909 env->prog->call_get_stack = true; in check_helper_call()
7912 if (check_get_func_ip(env)) in check_helper_call()
7914 env->prog->call_get_func_ip = true; in check_helper_call()
7918 clear_all_pkt_pointers(env); in check_helper_call()
7925 static void mark_btf_func_reg_size(struct bpf_verifier_env *env, u32 regno, in mark_btf_func_reg_size() argument
7928 struct bpf_reg_state *reg = &cur_regs(env)[regno]; in mark_btf_func_reg_size()
7934 DEF_NOT_SUBREG : env->insn_idx + 1; in mark_btf_func_reg_size()
7938 mark_insn_zext(env, reg); in mark_btf_func_reg_size()
7939 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64); in mark_btf_func_reg_size()
7941 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ32); in mark_btf_func_reg_size()
7946 static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, in check_kfunc_call() argument
7950 struct bpf_reg_state *regs = cur_regs(env); in check_kfunc_call()
7964 desc_btf = find_kfunc_desc_btf(env, insn->off); in check_kfunc_call()
7973 kfunc_flags = btf_kfunc_id_set_contains(desc_btf, resolve_prog_type(env->prog), func_id); in check_kfunc_call()
7975 verbose(env, "calling kernel function %s is not allowed\n", in check_kfunc_call()
7980 verbose(env, "destructive kfunc calls require CAP_SYS_BOOT capabilities\n"); in check_kfunc_call()
7989 err = btf_check_kfunc_arg_match(env, desc_btf, func_id, regs, &meta); in check_kfunc_call()
7996 err = release_reference(env, regs[err].ref_obj_id); in check_kfunc_call()
7998 verbose(env, "kfunc %s#%d reference has not been acquired before\n", in check_kfunc_call()
8005 mark_reg_not_init(env, regs, caller_saved[i]); in check_kfunc_call()
8011 verbose(env, "acquire kernel function does not return PTR_TO_BTF_ID\n"); in check_kfunc_call()
8016 mark_reg_unknown(env, regs, BPF_REG_0); in check_kfunc_call()
8017 mark_btf_func_reg_size(env, BPF_REG_0, t->size); in check_kfunc_call()
8025 verbose(env, in check_kfunc_call()
8033 mark_reg_known_zero(env, regs, BPF_REG_0); in check_kfunc_call()
8044 mark_reg_known_zero(env, regs, BPF_REG_0); in check_kfunc_call()
8052 regs[BPF_REG_0].id = ++env->id_gen; in check_kfunc_call()
8054 mark_btf_func_reg_size(env, BPF_REG_0, sizeof(void *)); in check_kfunc_call()
8056 int id = acquire_reference_state(env, insn_idx); in check_kfunc_call()
8072 mark_btf_func_reg_size(env, regno, sizeof(void *)); in check_kfunc_call()
8075 mark_btf_func_reg_size(env, regno, t->size); in check_kfunc_call()
8121 static bool check_reg_sane_offset(struct bpf_verifier_env *env, in check_reg_sane_offset() argument
8130 verbose(env, "math between %s pointer and %lld is not allowed\n", in check_reg_sane_offset()
8131 reg_type_str(env, type), val); in check_reg_sane_offset()
8136 verbose(env, "%s pointer offset %d is not allowed\n", in check_reg_sane_offset()
8137 reg_type_str(env, type), reg->off); in check_reg_sane_offset()
8142 verbose(env, "math between %s pointer and register with unbounded min value is not allowed\n", in check_reg_sane_offset()
8143 reg_type_str(env, type)); in check_reg_sane_offset()
8148 verbose(env, "value %lld makes %s pointer be out of bounds\n", in check_reg_sane_offset()
8149 smin, reg_type_str(env, type)); in check_reg_sane_offset()
8195 static bool can_skip_alu_sanitation(const struct bpf_verifier_env *env, in can_skip_alu_sanitation() argument
8198 return env->bypass_spec_v1 || BPF_SRC(insn->code) == BPF_K; in can_skip_alu_sanitation()
8218 static int sanitize_val_alu(struct bpf_verifier_env *env, in sanitize_val_alu() argument
8221 struct bpf_insn_aux_data *aux = cur_aux(env); in sanitize_val_alu()
8223 if (can_skip_alu_sanitation(env, insn)) in sanitize_val_alu()
8240 sanitize_speculative_path(struct bpf_verifier_env *env, in sanitize_speculative_path() argument
8247 branch = push_stack(env, next_idx, curr_idx, true); in sanitize_speculative_path()
8251 mark_reg_unknown(env, regs, insn->dst_reg); in sanitize_speculative_path()
8253 mark_reg_unknown(env, regs, insn->dst_reg); in sanitize_speculative_path()
8254 mark_reg_unknown(env, regs, insn->src_reg); in sanitize_speculative_path()
8260 static int sanitize_ptr_alu(struct bpf_verifier_env *env, in sanitize_ptr_alu() argument
8268 struct bpf_insn_aux_data *aux = commit_window ? cur_aux(env) : &info->aux; in sanitize_ptr_alu()
8269 struct bpf_verifier_state *vstate = env->cur_state; in sanitize_ptr_alu()
8279 if (can_skip_alu_sanitation(env, insn)) in sanitize_ptr_alu()
8318 env->explore_alu_limits = true; in sanitize_ptr_alu()
8349 ret = sanitize_speculative_path(env, NULL, env->insn_idx + 1, in sanitize_ptr_alu()
8350 env->insn_idx); in sanitize_ptr_alu()
8356 static void sanitize_mark_insn_seen(struct bpf_verifier_env *env) in sanitize_mark_insn_seen() argument
8358 struct bpf_verifier_state *vstate = env->cur_state; in sanitize_mark_insn_seen()
8366 env->insn_aux_data[env->insn_idx].seen = env->pass_cnt; in sanitize_mark_insn_seen()
8369 static int sanitize_err(struct bpf_verifier_env *env, in sanitize_err() argument
8380 verbose(env, "R%d has unknown scalar with mixed signed bounds, %s\n", in sanitize_err()
8384 verbose(env, "R%d has pointer with unsupported alu operation, %s\n", in sanitize_err()
8388 verbose(env, "R%d tried to %s from different maps, paths or scalars, %s\n", in sanitize_err()
8392 verbose(env, "R%d tried to %s beyond pointer bounds, %s\n", in sanitize_err()
8396 verbose(env, "R%d could not be pushed for speculative verification, %s\n", in sanitize_err()
8400 verbose(env, "verifier internal error: unknown reason (%d)\n", in sanitize_err()
8419 struct bpf_verifier_env *env, in check_stack_access_for_ptr_arithmetic() argument
8428 verbose(env, "R%d variable stack access prohibited for !root, var_off=%s off=%d\n", in check_stack_access_for_ptr_arithmetic()
8434 verbose(env, "R%d stack pointer arithmetic goes out of range, " in check_stack_access_for_ptr_arithmetic()
8442 static int sanitize_check_bounds(struct bpf_verifier_env *env, in sanitize_check_bounds() argument
8451 if (env->bypass_spec_v1) in sanitize_check_bounds()
8456 if (check_stack_access_for_ptr_arithmetic(env, dst, dst_reg, in sanitize_check_bounds()
8461 if (check_map_access(env, dst, dst_reg->off, 1, false, ACCESS_HELPER)) { in sanitize_check_bounds()
8462 verbose(env, "R%d pointer arithmetic of map value goes out of range, " in sanitize_check_bounds()
8479 static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, in adjust_ptr_min_max_vals() argument
8484 struct bpf_verifier_state *vstate = env->cur_state; in adjust_ptr_min_max_vals()
8504 __mark_reg_unknown(env, dst_reg); in adjust_ptr_min_max_vals()
8510 if (opcode == BPF_SUB && env->allow_ptr_leaks) { in adjust_ptr_min_max_vals()
8511 __mark_reg_unknown(env, dst_reg); in adjust_ptr_min_max_vals()
8515 verbose(env, in adjust_ptr_min_max_vals()
8522 verbose(env, "R%d pointer arithmetic on %s prohibited, null-check it first\n", in adjust_ptr_min_max_vals()
8523 dst, reg_type_str(env, ptr_reg->type)); in adjust_ptr_min_max_vals()
8542 verbose(env, "R%d pointer arithmetic on %s prohibited\n", in adjust_ptr_min_max_vals()
8543 dst, reg_type_str(env, ptr_reg->type)); in adjust_ptr_min_max_vals()
8555 if (!check_reg_sane_offset(env, off_reg, ptr_reg->type) || in adjust_ptr_min_max_vals()
8556 !check_reg_sane_offset(env, ptr_reg, ptr_reg->type)) in adjust_ptr_min_max_vals()
8563 ret = sanitize_ptr_alu(env, insn, ptr_reg, off_reg, dst_reg, in adjust_ptr_min_max_vals()
8566 return sanitize_err(env, insn, ret, off_reg, dst_reg); in adjust_ptr_min_max_vals()
8615 dst_reg->id = ++env->id_gen; in adjust_ptr_min_max_vals()
8623 verbose(env, "R%d tried to subtract pointer from scalar\n", in adjust_ptr_min_max_vals()
8632 verbose(env, "R%d subtraction from stack pointer prohibited\n", in adjust_ptr_min_max_vals()
8674 dst_reg->id = ++env->id_gen; in adjust_ptr_min_max_vals()
8684 verbose(env, "R%d bitwise operator %s on pointer prohibited\n", in adjust_ptr_min_max_vals()
8689 verbose(env, "R%d pointer arithmetic with %s operator prohibited\n", in adjust_ptr_min_max_vals()
8694 if (!check_reg_sane_offset(env, dst_reg, ptr_reg->type)) in adjust_ptr_min_max_vals()
8697 if (sanitize_check_bounds(env, insn, dst_reg) < 0) in adjust_ptr_min_max_vals()
8700 ret = sanitize_ptr_alu(env, insn, dst_reg, off_reg, dst_reg, in adjust_ptr_min_max_vals()
8703 return sanitize_err(env, insn, ret, off_reg, dst_reg); in adjust_ptr_min_max_vals()
9280 static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, in adjust_scalar_min_max_vals() argument
9285 struct bpf_reg_state *regs = cur_regs(env); in adjust_scalar_min_max_vals()
9314 __mark_reg_unknown(env, dst_reg); in adjust_scalar_min_max_vals()
9325 __mark_reg_unknown(env, dst_reg); in adjust_scalar_min_max_vals()
9332 __mark_reg_unknown(env, dst_reg); in adjust_scalar_min_max_vals()
9337 ret = sanitize_val_alu(env, insn); in adjust_scalar_min_max_vals()
9339 return sanitize_err(env, insn, ret, NULL, NULL); in adjust_scalar_min_max_vals()
9392 mark_reg_unknown(env, regs, insn->dst_reg); in adjust_scalar_min_max_vals()
9405 mark_reg_unknown(env, regs, insn->dst_reg); in adjust_scalar_min_max_vals()
9418 mark_reg_unknown(env, regs, insn->dst_reg); in adjust_scalar_min_max_vals()
9427 mark_reg_unknown(env, regs, insn->dst_reg); in adjust_scalar_min_max_vals()
9441 static int adjust_reg_min_max_vals(struct bpf_verifier_env *env, in adjust_reg_min_max_vals() argument
9444 struct bpf_verifier_state *vstate = env->cur_state; in adjust_reg_min_max_vals()
9468 if (opcode == BPF_SUB && env->allow_ptr_leaks) { in adjust_reg_min_max_vals()
9469 mark_reg_unknown(env, regs, insn->dst_reg); in adjust_reg_min_max_vals()
9472 verbose(env, "R%d pointer %s pointer prohibited\n", in adjust_reg_min_max_vals()
9481 err = mark_chain_precision(env, insn->dst_reg); in adjust_reg_min_max_vals()
9484 return adjust_ptr_min_max_vals(env, insn, in adjust_reg_min_max_vals()
9489 err = mark_chain_precision(env, insn->src_reg); in adjust_reg_min_max_vals()
9492 return adjust_ptr_min_max_vals(env, insn, in adjust_reg_min_max_vals()
9496 err = mark_chain_precision(env, insn->src_reg); in adjust_reg_min_max_vals()
9508 return adjust_ptr_min_max_vals(env, insn, in adjust_reg_min_max_vals()
9514 print_verifier_state(env, state, true); in adjust_reg_min_max_vals()
9515 verbose(env, "verifier internal error: unexpected ptr_reg\n"); in adjust_reg_min_max_vals()
9519 print_verifier_state(env, state, true); in adjust_reg_min_max_vals()
9520 verbose(env, "verifier internal error: no src_reg\n"); in adjust_reg_min_max_vals()
9523 return adjust_scalar_min_max_vals(env, insn, dst_reg, *src_reg); in adjust_reg_min_max_vals()
9527 static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn) in check_alu_op() argument
9529 struct bpf_reg_state *regs = cur_regs(env); in check_alu_op()
9538 verbose(env, "BPF_NEG uses reserved fields\n"); in check_alu_op()
9545 verbose(env, "BPF_END uses reserved fields\n"); in check_alu_op()
9551 err = check_reg_arg(env, insn->dst_reg, SRC_OP); in check_alu_op()
9555 if (is_pointer_value(env, insn->dst_reg)) { in check_alu_op()
9556 verbose(env, "R%d pointer arithmetic prohibited\n", in check_alu_op()
9562 err = check_reg_arg(env, insn->dst_reg, DST_OP); in check_alu_op()
9570 verbose(env, "BPF_MOV uses reserved fields\n"); in check_alu_op()
9575 err = check_reg_arg(env, insn->src_reg, SRC_OP); in check_alu_op()
9580 verbose(env, "BPF_MOV uses reserved fields\n"); in check_alu_op()
9586 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); in check_alu_op()
9603 src_reg->id = ++env->id_gen; in check_alu_op()
9609 if (is_pointer_value(env, insn->src_reg)) { in check_alu_op()
9610 verbose(env, in check_alu_op()
9622 dst_reg->subreg_def = env->insn_idx + 1; in check_alu_op()
9624 mark_reg_unknown(env, regs, in check_alu_op()
9635 mark_reg_unknown(env, regs, insn->dst_reg); in check_alu_op()
9647 verbose(env, "invalid BPF_ALU opcode %x\n", opcode); in check_alu_op()
9654 verbose(env, "BPF_ALU uses reserved fields\n"); in check_alu_op()
9658 err = check_reg_arg(env, insn->src_reg, SRC_OP); in check_alu_op()
9663 verbose(env, "BPF_ALU uses reserved fields\n"); in check_alu_op()
9669 err = check_reg_arg(env, insn->dst_reg, SRC_OP); in check_alu_op()
9675 verbose(env, "div by zero\n"); in check_alu_op()
9684 verbose(env, "invalid shift %d\n", insn->imm); in check_alu_op()
9690 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); in check_alu_op()
9694 return adjust_reg_min_max_vals(env, insn); in check_alu_op()
10415 static int check_cond_jmp_op(struct bpf_verifier_env *env, in check_cond_jmp_op() argument
10418 struct bpf_verifier_state *this_branch = env->cur_state; in check_cond_jmp_op()
10429 verbose(env, "invalid BPF_JMP/JMP32 opcode %x\n", opcode); in check_cond_jmp_op()
10434 err = check_reg_arg(env, insn->dst_reg, SRC_OP); in check_cond_jmp_op()
10441 verbose(env, "BPF_JMP/JMP32 uses reserved fields\n"); in check_cond_jmp_op()
10446 err = check_reg_arg(env, insn->src_reg, SRC_OP); in check_cond_jmp_op()
10452 is_pointer_value(env, insn->src_reg)) { in check_cond_jmp_op()
10453 verbose(env, "R%d pointer comparison prohibited\n", in check_cond_jmp_op()
10459 verbose(env, "BPF_JMP/JMP32 uses reserved fields\n"); in check_cond_jmp_op()
10491 err = mark_chain_precision(env, insn->dst_reg); in check_cond_jmp_op()
10494 err = mark_chain_precision(env, insn->src_reg); in check_cond_jmp_op()
10504 if (!env->bypass_spec_v1 && in check_cond_jmp_op()
10505 !sanitize_speculative_path(env, insn, *insn_idx + 1, in check_cond_jmp_op()
10508 if (env->log.level & BPF_LOG_LEVEL) in check_cond_jmp_op()
10509 print_insn_state(env, this_branch->frame[this_branch->curframe]); in check_cond_jmp_op()
10517 if (!env->bypass_spec_v1 && in check_cond_jmp_op()
10518 !sanitize_speculative_path(env, insn, in check_cond_jmp_op()
10522 if (env->log.level & BPF_LOG_LEVEL) in check_cond_jmp_op()
10523 print_insn_state(env, this_branch->frame[this_branch->curframe]); in check_cond_jmp_op()
10527 other_branch = push_stack(env, *insn_idx + insn->off + 1, *insn_idx, in check_cond_jmp_op()
10602 is_pointer_value(env, insn->dst_reg)) { in check_cond_jmp_op()
10603 verbose(env, "R%d pointer comparison prohibited\n", in check_cond_jmp_op()
10607 if (env->log.level & BPF_LOG_LEVEL) in check_cond_jmp_op()
10608 print_insn_state(env, this_branch->frame[this_branch->curframe]); in check_cond_jmp_op()
10613 static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn) in check_ld_imm() argument
10615 struct bpf_insn_aux_data *aux = cur_aux(env); in check_ld_imm()
10616 struct bpf_reg_state *regs = cur_regs(env); in check_ld_imm()
10622 verbose(env, "invalid BPF_LD_IMM insn\n"); in check_ld_imm()
10626 verbose(env, "BPF_LD_IMM64 uses reserved fields\n"); in check_ld_imm()
10630 err = check_reg_arg(env, insn->dst_reg, DST_OP); in check_ld_imm()
10647 mark_reg_known_zero(env, regs, insn->dst_reg); in check_ld_imm()
10660 verbose(env, "bpf verifier is misconfigured\n"); in check_ld_imm()
10667 struct bpf_prog_aux *aux = env->prog->aux; in check_ld_imm()
10668 u32 subprogno = find_subprog(env, in check_ld_imm()
10669 env->insn_idx + insn->imm + 1); in check_ld_imm()
10672 verbose(env, "missing btf func_info\n"); in check_ld_imm()
10676 verbose(env, "callback function not static\n"); in check_ld_imm()
10685 map = env->used_maps[aux->map_index]; in check_ld_imm()
10693 dst_reg->id = ++env->id_gen; in check_ld_imm()
10698 verbose(env, "bpf verifier is misconfigured\n"); in check_ld_imm()
10732 static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn) in check_ld_abs() argument
10734 struct bpf_reg_state *regs = cur_regs(env); in check_ld_abs()
10739 if (!may_access_skb(resolve_prog_type(env->prog))) { in check_ld_abs()
10740 verbose(env, "BPF_LD_[ABS|IND] instructions not allowed for this program type\n"); in check_ld_abs()
10744 if (!env->ops->gen_ld_abs) { in check_ld_abs()
10745 verbose(env, "bpf verifier is misconfigured\n"); in check_ld_abs()
10752 verbose(env, "BPF_LD_[ABS|IND] uses reserved fields\n"); in check_ld_abs()
10757 err = check_reg_arg(env, ctx_reg, SRC_OP); in check_ld_abs()
10765 err = check_reference_leak(env); in check_ld_abs()
10767 verbose(env, "BPF_LD_[ABS|IND] cannot be mixed with socket references\n"); in check_ld_abs()
10771 if (env->cur_state->active_spin_lock) { in check_ld_abs()
10772 verbose(env, "BPF_LD_[ABS|IND] cannot be used inside bpf_spin_lock-ed region\n"); in check_ld_abs()
10777 verbose(env, in check_ld_abs()
10784 err = check_reg_arg(env, insn->src_reg, SRC_OP); in check_ld_abs()
10789 err = check_ptr_off_reg(env, ®s[ctx_reg], ctx_reg); in check_ld_abs()
10795 mark_reg_not_init(env, regs, caller_saved[i]); in check_ld_abs()
10796 check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); in check_ld_abs()
10803 mark_reg_unknown(env, regs, BPF_REG_0); in check_ld_abs()
10805 regs[BPF_REG_0].subreg_def = env->insn_idx + 1; in check_ld_abs()
10809 static int check_return_code(struct bpf_verifier_env *env) in check_return_code() argument
10812 const struct bpf_prog *prog = env->prog; in check_return_code()
10815 enum bpf_prog_type prog_type = resolve_prog_type(env->prog); in check_return_code()
10817 struct bpf_func_state *frame = env->cur_state->frame[0]; in check_return_code()
10843 err = check_reg_arg(env, BPF_REG_0, SRC_OP); in check_return_code()
10847 if (is_pointer_value(env, BPF_REG_0)) { in check_return_code()
10848 verbose(env, "R0 leaks addr as return value\n"); in check_return_code()
10852 reg = cur_regs(env) + BPF_REG_0; in check_return_code()
10857 verbose(env, "In async callback the register R0 is not a known value (%s)\n", in check_return_code()
10858 reg_type_str(env, reg->type)); in check_return_code()
10863 verbose_invalid_scalar(env, reg, &const_0, "async callback", "R0"); in check_return_code()
10871 verbose(env, "At subprogram exit the register R0 is not a scalar value (%s)\n", in check_return_code()
10872 reg_type_str(env, reg->type)); in check_return_code()
10880 if (env->prog->expected_attach_type == BPF_CGROUP_UDP4_RECVMSG || in check_return_code()
10881 env->prog->expected_attach_type == BPF_CGROUP_UDP6_RECVMSG || in check_return_code()
10882 env->prog->expected_attach_type == BPF_CGROUP_INET4_GETPEERNAME || in check_return_code()
10883 env->prog->expected_attach_type == BPF_CGROUP_INET6_GETPEERNAME || in check_return_code()
10884 env->prog->expected_attach_type == BPF_CGROUP_INET4_GETSOCKNAME || in check_return_code()
10885 env->prog->expected_attach_type == BPF_CGROUP_INET6_GETSOCKNAME) in check_return_code()
10887 if (env->prog->expected_attach_type == BPF_CGROUP_INET4_BIND || in check_return_code()
10888 env->prog->expected_attach_type == BPF_CGROUP_INET6_BIND) in check_return_code()
10892 if (env->prog->expected_attach_type == BPF_CGROUP_INET_EGRESS) { in check_return_code()
10904 if (!env->prog->aux->attach_btf_id) in check_return_code()
10909 switch (env->prog->expected_attach_type) { in check_return_code()
10928 if (env->prog->expected_attach_type != BPF_LSM_CGROUP) { in check_return_code()
10934 if (!env->prog->aux->attach_func_proto->type) { in check_return_code()
10951 verbose(env, "At program exit the register R0 is not a known value (%s)\n", in check_return_code()
10952 reg_type_str(env, reg->type)); in check_return_code()
10957 verbose_invalid_scalar(env, reg, &range, "program exit", "R0"); in check_return_code()
10961 verbose(env, "Note, BPF_LSM_CGROUP that attach to void LSM hooks can't modify return value!\n"); in check_return_code()
10967 env->prog->enforce_expected_attach_type = 1; in check_return_code()
11011 static u32 state_htab_size(struct bpf_verifier_env *env) in state_htab_size() argument
11013 return env->prog->len; in state_htab_size()
11017 struct bpf_verifier_env *env, in explored_state() argument
11020 struct bpf_verifier_state *cur = env->cur_state; in explored_state()
11023 return &env->explored_states[(idx ^ state->callsite) % state_htab_size(env)]; in explored_state()
11026 static void init_explored_state(struct bpf_verifier_env *env, int idx) in init_explored_state() argument
11028 env->insn_aux_data[idx].prune_point = true; in init_explored_state()
11041 static int push_insn(int t, int w, int e, struct bpf_verifier_env *env, in push_insn() argument
11044 int *insn_stack = env->cfg.insn_stack; in push_insn()
11045 int *insn_state = env->cfg.insn_state; in push_insn()
11053 if (w < 0 || w >= env->prog->len) { in push_insn()
11054 verbose_linfo(env, t, "%d: ", t); in push_insn()
11055 verbose(env, "jump out of range from insn %d to %d\n", t, w); in push_insn()
11061 init_explored_state(env, w); in push_insn()
11067 if (env->cfg.cur_stack >= env->prog->len) in push_insn()
11069 insn_stack[env->cfg.cur_stack++] = w; in push_insn()
11072 if (loop_ok && env->bpf_capable) in push_insn()
11074 verbose_linfo(env, t, "%d: ", t); in push_insn()
11075 verbose_linfo(env, w, "%d: ", w); in push_insn()
11076 verbose(env, "back-edge from insn %d to %d\n", t, w); in push_insn()
11082 verbose(env, "insn state internal bug\n"); in push_insn()
11090 struct bpf_verifier_env *env, in visit_func_call_insn() argument
11095 ret = push_insn(t, t + 1, FALLTHROUGH, env, false); in visit_func_call_insn()
11100 init_explored_state(env, t + 1); in visit_func_call_insn()
11102 init_explored_state(env, t); in visit_func_call_insn()
11103 ret = push_insn(t, t + insns[t].imm + 1, BRANCH, env, in visit_func_call_insn()
11118 static int visit_insn(int t, int insn_cnt, struct bpf_verifier_env *env) in visit_insn() argument
11120 struct bpf_insn *insns = env->prog->insnsi; in visit_insn()
11124 return visit_func_call_insn(t, insn_cnt, insns, env, true); in visit_insn()
11129 return push_insn(t, t + 1, FALLTHROUGH, env, false); in visit_insn()
11142 init_explored_state(env, t); in visit_insn()
11143 return visit_func_call_insn(t, insn_cnt, insns, env, in visit_insn()
11151 ret = push_insn(t, t + insns[t].off + 1, FALLTHROUGH, env, in visit_insn()
11160 init_explored_state(env, t + insns[t].off + 1); in visit_insn()
11165 init_explored_state(env, t + 1); in visit_insn()
11171 init_explored_state(env, t); in visit_insn()
11172 ret = push_insn(t, t + 1, FALLTHROUGH, env, true); in visit_insn()
11176 return push_insn(t, t + insns[t].off + 1, BRANCH, env, true); in visit_insn()
11183 static int check_cfg(struct bpf_verifier_env *env) in check_cfg() argument
11185 int insn_cnt = env->prog->len; in check_cfg()
11190 insn_state = env->cfg.insn_state = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL); in check_cfg()
11194 insn_stack = env->cfg.insn_stack = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL); in check_cfg()
11202 env->cfg.cur_stack = 1; in check_cfg()
11204 while (env->cfg.cur_stack > 0) { in check_cfg()
11205 int t = insn_stack[env->cfg.cur_stack - 1]; in check_cfg()
11207 ret = visit_insn(t, insn_cnt, env); in check_cfg()
11211 env->cfg.cur_stack--; in check_cfg()
11217 verbose(env, "visit_insn internal bug\n"); in check_cfg()
11224 if (env->cfg.cur_stack < 0) { in check_cfg()
11225 verbose(env, "pop stack internal bug\n"); in check_cfg()
11232 verbose(env, "unreachable insn %d\n", i); in check_cfg()
11242 env->cfg.insn_state = env->cfg.insn_stack = NULL; in check_cfg()
11246 static int check_abnormal_return(struct bpf_verifier_env *env) in check_abnormal_return() argument
11250 for (i = 1; i < env->subprog_cnt; i++) { in check_abnormal_return()
11251 if (env->subprog_info[i].has_ld_abs) { in check_abnormal_return()
11252 verbose(env, "LD_ABS is not allowed in subprogs without BTF\n"); in check_abnormal_return()
11255 if (env->subprog_info[i].has_tail_call) { in check_abnormal_return()
11256 verbose(env, "tail_call is not allowed in subprogs without BTF\n"); in check_abnormal_return()
11267 static int check_btf_func(struct bpf_verifier_env *env, in check_btf_func() argument
11285 if (check_abnormal_return(env)) in check_btf_func()
11290 if (nfuncs != env->subprog_cnt) { in check_btf_func()
11291 verbose(env, "number of funcs in func_info doesn't match number of subprogs\n"); in check_btf_func()
11299 verbose(env, "invalid func info rec size %u\n", urec_size); in check_btf_func()
11303 prog = env->prog; in check_btf_func()
11320 verbose(env, "nonzero tailing record in func info"); in check_btf_func()
11341 verbose(env, in check_btf_func()
11347 verbose(env, in check_btf_func()
11353 if (env->subprog_info[i].start != krecord[i].insn_off) { in check_btf_func()
11354 verbose(env, "func_info BTF section doesn't match subprog layout in BPF program\n"); in check_btf_func()
11361 verbose(env, "invalid type id %d in func info", in check_btf_func()
11374 if (i && !scalar_return && env->subprog_info[i].has_ld_abs) { in check_btf_func()
11375 verbose(env, "LD_ABS is only allowed in functions that return 'int'.\n"); in check_btf_func()
11378 if (i && !scalar_return && env->subprog_info[i].has_tail_call) { in check_btf_func()
11379 verbose(env, "tail_call is only allowed in functions that return 'int'.\n"); in check_btf_func()
11398 static void adjust_btf_func(struct bpf_verifier_env *env) in adjust_btf_func() argument
11400 struct bpf_prog_aux *aux = env->prog->aux; in adjust_btf_func()
11406 for (i = 0; i < env->subprog_cnt; i++) in adjust_btf_func()
11407 aux->func_info[i].insn_off = env->subprog_info[i].start; in adjust_btf_func()
11413 static int check_btf_line(struct bpf_verifier_env *env, in check_btf_line() argument
11445 prog = env->prog; in check_btf_line()
11449 sub = env->subprog_info; in check_btf_line()
11457 verbose(env, "nonzero tailing record in line_info"); in check_btf_line()
11484 verbose(env, "Invalid line_info[%u].insn_off:%u (prev_offset:%u prog->len:%u)\n", in check_btf_line()
11492 verbose(env, in check_btf_line()
11501 verbose(env, "Invalid line_info[%u].line_off or .file_name_off\n", i); in check_btf_line()
11506 if (s != env->subprog_cnt) { in check_btf_line()
11511 verbose(env, "missing bpf_line_info for func#%u\n", s); in check_btf_line()
11521 if (s != env->subprog_cnt) { in check_btf_line()
11522 verbose(env, "missing bpf_line_info for %u funcs starting from func#%u\n", in check_btf_line()
11523 env->subprog_cnt - s, s); in check_btf_line()
11541 static int check_core_relo(struct bpf_verifier_env *env, in check_core_relo() argument
11547 struct bpf_prog *prog = env->prog; in check_core_relo()
11550 .log = &env->log, in check_core_relo()
11580 verbose(env, "nonzero tailing record in core_relo"); in check_core_relo()
11595 verbose(env, "Invalid core_relo[%u].insn_off:%u prog->len:%u\n", in check_core_relo()
11610 static int check_btf_info(struct bpf_verifier_env *env, in check_btf_info() argument
11618 if (check_abnormal_return(env)) in check_btf_info()
11630 env->prog->aux->btf = btf; in check_btf_info()
11632 err = check_btf_func(env, attr, uattr); in check_btf_info()
11636 err = check_btf_line(env, attr, uattr); in check_btf_info()
11640 err = check_core_relo(env, attr, uattr); in check_btf_info()
11690 static void clean_func_state(struct bpf_verifier_env *env, in clean_func_state() argument
11704 __mark_reg_not_init(env, &st->regs[i]); in clean_func_state()
11712 __mark_reg_not_init(env, &st->stack[i].spilled_ptr); in clean_func_state()
11719 static void clean_verifier_state(struct bpf_verifier_env *env, in clean_verifier_state() argument
11729 clean_func_state(env, st->frame[i]); in clean_verifier_state()
11764 static void clean_live_states(struct bpf_verifier_env *env, int insn, in clean_live_states() argument
11770 sl = *explored_state(env, insn); in clean_live_states()
11780 clean_verifier_state(env, &sl->state); in clean_live_states()
11787 static bool regsafe(struct bpf_verifier_env *env, struct bpf_reg_state *rold, in regsafe() argument
11814 if (env->explore_alu_limits) in regsafe()
11905 static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old, in stacksafe() argument
11951 if (!regsafe(env, &old->stack[spi].spilled_ptr, in stacksafe()
12002 static bool func_states_equal(struct bpf_verifier_env *env, struct bpf_func_state *old, in func_states_equal() argument
12007 memset(env->idmap_scratch, 0, sizeof(env->idmap_scratch)); in func_states_equal()
12009 if (!regsafe(env, &old->regs[i], &cur->regs[i], in func_states_equal()
12010 env->idmap_scratch)) in func_states_equal()
12013 if (!stacksafe(env, old, cur, env->idmap_scratch)) in func_states_equal()
12022 static bool states_equal(struct bpf_verifier_env *env, in states_equal() argument
12046 if (!func_states_equal(env, old->frame[i], cur->frame[i])) in states_equal()
12055 static int propagate_liveness_reg(struct bpf_verifier_env *env, in propagate_liveness_reg() argument
12074 err = mark_reg_read(env, reg, parent_reg, flag); in propagate_liveness_reg()
12088 static int propagate_liveness(struct bpf_verifier_env *env, in propagate_liveness() argument
12110 err = propagate_liveness_reg(env, &state_reg[i], in propagate_liveness()
12115 mark_insn_zext(env, &parent_reg[i]); in propagate_liveness()
12123 err = propagate_liveness_reg(env, state_reg, in propagate_liveness()
12135 static int propagate_precision(struct bpf_verifier_env *env, in propagate_precision() argument
12150 if (env->log.level & BPF_LOG_LEVEL2) in propagate_precision()
12151 verbose(env, "frame %d: propagating r%d\n", fr, i); in propagate_precision()
12152 err = mark_chain_precision_frame(env, fr, i); in propagate_precision()
12165 if (env->log.level & BPF_LOG_LEVEL2) in propagate_precision()
12166 verbose(env, "frame %d: propagating fp%d\n", in propagate_precision()
12168 err = mark_chain_precision_stack_frame(env, fr, i); in propagate_precision()
12195 static int is_state_visited(struct bpf_verifier_env *env, int insn_idx) in is_state_visited() argument
12199 struct bpf_verifier_state *cur = env->cur_state, *new; in is_state_visited()
12201 bool add_new_state = env->test_state_freq ? true : false; in is_state_visited()
12203 cur->last_insn_idx = env->prev_insn_idx; in is_state_visited()
12204 if (!env->insn_aux_data[insn_idx].prune_point) in is_state_visited()
12218 if (env->jmps_processed - env->prev_jmps_processed >= 2 && in is_state_visited()
12219 env->insn_processed - env->prev_insn_processed >= 8) in is_state_visited()
12222 pprev = explored_state(env, insn_idx); in is_state_visited()
12225 clean_live_states(env, insn_idx, cur); in is_state_visited()
12249 states_equal(env, &sl->state, cur)) { in is_state_visited()
12250 verbose_linfo(env, insn_idx, "; "); in is_state_visited()
12251 verbose(env, "infinite loop detected at insn %d\n", insn_idx); in is_state_visited()
12266 if (env->jmps_processed - env->prev_jmps_processed < 20 && in is_state_visited()
12267 env->insn_processed - env->prev_insn_processed < 100) in is_state_visited()
12271 if (states_equal(env, &sl->state, cur)) { in is_state_visited()
12283 err = propagate_liveness(env, &sl->state, cur); in is_state_visited()
12290 err = err ? : push_jmp_history(env, cur); in is_state_visited()
12291 err = err ? : propagate_precision(env, &sl->state); in is_state_visited()
12323 env->peak_states--; in is_state_visited()
12329 sl->next = env->free_list; in is_state_visited()
12330 env->free_list = sl; in is_state_visited()
12340 if (env->max_states_per_insn < states_cnt) in is_state_visited()
12341 env->max_states_per_insn = states_cnt; in is_state_visited()
12343 if (!env->bpf_capable && states_cnt > BPF_COMPLEXITY_LIMIT_STATES) in is_state_visited()
12344 return push_jmp_history(env, cur); in is_state_visited()
12347 return push_jmp_history(env, cur); in is_state_visited()
12361 env->total_states++; in is_state_visited()
12362 env->peak_states++; in is_state_visited()
12363 env->prev_jmps_processed = env->jmps_processed; in is_state_visited()
12364 env->prev_insn_processed = env->insn_processed; in is_state_visited()
12367 if (env->bpf_capable) in is_state_visited()
12368 mark_all_scalars_imprecise(env, cur); in is_state_visited()
12385 new_sl->next = *explored_state(env, insn_idx); in is_state_visited()
12386 *explored_state(env, insn_idx) = new_sl; in is_state_visited()
12455 static int do_check(struct bpf_verifier_env *env) in do_check() argument
12457 bool pop_log = !(env->log.level & BPF_LOG_LEVEL2); in do_check()
12458 struct bpf_verifier_state *state = env->cur_state; in do_check()
12459 struct bpf_insn *insns = env->prog->insnsi; in do_check()
12461 int insn_cnt = env->prog->len; in do_check()
12470 env->prev_insn_idx = prev_insn_idx; in do_check()
12471 if (env->insn_idx >= insn_cnt) { in do_check()
12472 verbose(env, "invalid insn idx %d insn_cnt %d\n", in do_check()
12473 env->insn_idx, insn_cnt); in do_check()
12477 insn = &insns[env->insn_idx]; in do_check()
12480 if (++env->insn_processed > BPF_COMPLEXITY_LIMIT_INSNS) { in do_check()
12481 verbose(env, in do_check()
12483 env->insn_processed); in do_check()
12487 err = is_state_visited(env, env->insn_idx); in do_check()
12492 if (env->log.level & BPF_LOG_LEVEL) { in do_check()
12494 verbose(env, "\nfrom %d to %d%s: safe\n", in do_check()
12495 env->prev_insn_idx, env->insn_idx, in do_check()
12496 env->cur_state->speculative ? in do_check()
12499 verbose(env, "%d: safe\n", env->insn_idx); in do_check()
12510 if (env->log.level & BPF_LOG_LEVEL2 && do_print_state) { in do_check()
12511 verbose(env, "\nfrom %d to %d%s:", in do_check()
12512 env->prev_insn_idx, env->insn_idx, in do_check()
12513 env->cur_state->speculative ? in do_check()
12515 print_verifier_state(env, state->frame[state->curframe], true); in do_check()
12519 if (env->log.level & BPF_LOG_LEVEL) { in do_check()
12523 .private_data = env, in do_check()
12526 if (verifier_state_scratched(env)) in do_check()
12527 print_insn_state(env, state->frame[state->curframe]); in do_check()
12529 verbose_linfo(env, env->insn_idx, "; "); in do_check()
12530 env->prev_log_len = env->log.len_used; in do_check()
12531 verbose(env, "%d: ", env->insn_idx); in do_check()
12532 print_bpf_insn(&cbs, insn, env->allow_ptr_leaks); in do_check()
12533 env->prev_insn_print_len = env->log.len_used - env->prev_log_len; in do_check()
12534 env->prev_log_len = env->log.len_used; in do_check()
12537 if (bpf_prog_is_dev_bound(env->prog->aux)) { in do_check()
12538 err = bpf_prog_offload_verify_insn(env, env->insn_idx, in do_check()
12539 env->prev_insn_idx); in do_check()
12544 regs = cur_regs(env); in do_check()
12545 sanitize_mark_insn_seen(env); in do_check()
12546 prev_insn_idx = env->insn_idx; in do_check()
12549 err = check_alu_op(env, insn); in do_check()
12559 err = check_reg_arg(env, insn->src_reg, SRC_OP); in do_check()
12563 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); in do_check()
12572 err = check_mem_access(env, env->insn_idx, insn->src_reg, in do_check()
12578 prev_src_type = &env->insn_aux_data[env->insn_idx].ptr_type; in do_check()
12595 verbose(env, "same insn cannot be used with different pointers\n"); in do_check()
12603 err = check_atomic(env, env->insn_idx, insn); in do_check()
12606 env->insn_idx++; in do_check()
12611 verbose(env, "BPF_STX uses reserved fields\n"); in do_check()
12616 err = check_reg_arg(env, insn->src_reg, SRC_OP); in do_check()
12620 err = check_reg_arg(env, insn->dst_reg, SRC_OP); in do_check()
12627 err = check_mem_access(env, env->insn_idx, insn->dst_reg, in do_check()
12633 prev_dst_type = &env->insn_aux_data[env->insn_idx].ptr_type; in do_check()
12638 verbose(env, "same insn cannot be used with different pointers\n"); in do_check()
12645 verbose(env, "BPF_ST uses reserved fields\n"); in do_check()
12649 err = check_reg_arg(env, insn->dst_reg, SRC_OP); in do_check()
12653 if (is_ctx_reg(env, insn->dst_reg)) { in do_check()
12654 verbose(env, "BPF_ST stores into R%d %s is not allowed\n", in do_check()
12656 reg_type_str(env, reg_state(env, insn->dst_reg)->type)); in do_check()
12661 err = check_mem_access(env, env->insn_idx, insn->dst_reg, in do_check()
12670 env->jmps_processed++; in do_check()
12680 verbose(env, "BPF_CALL uses reserved fields\n"); in do_check()
12684 if (env->cur_state->active_spin_lock && in do_check()
12687 verbose(env, "function calls are not allowed while holding a lock\n"); in do_check()
12691 err = check_func_call(env, insn, &env->insn_idx); in do_check()
12693 err = check_kfunc_call(env, insn, &env->insn_idx); in do_check()
12695 err = check_helper_call(env, insn, &env->insn_idx); in do_check()
12704 verbose(env, "BPF_JA uses reserved fields\n"); in do_check()
12708 env->insn_idx += insn->off + 1; in do_check()
12717 verbose(env, "BPF_EXIT uses reserved fields\n"); in do_check()
12721 if (env->cur_state->active_spin_lock) { in do_check()
12722 verbose(env, "bpf_spin_unlock is missing\n"); in do_check()
12732 err = check_reference_leak(env); in do_check()
12738 err = prepare_func_exit(env, &env->insn_idx); in do_check()
12745 err = check_return_code(env); in do_check()
12749 mark_verifier_state_scratched(env); in do_check()
12750 update_branch_counts(env, env->cur_state); in do_check()
12751 err = pop_stack(env, &prev_insn_idx, in do_check()
12752 &env->insn_idx, pop_log); in do_check()
12762 err = check_cond_jmp_op(env, insn, &env->insn_idx); in do_check()
12770 err = check_ld_abs(env, insn); in do_check()
12775 err = check_ld_imm(env, insn); in do_check()
12779 env->insn_idx++; in do_check()
12780 sanitize_mark_insn_seen(env); in do_check()
12782 verbose(env, "invalid BPF_LD mode\n"); in do_check()
12786 verbose(env, "unknown insn class %d\n", class); in do_check()
12790 env->insn_idx++; in do_check()
12827 static int check_pseudo_btf_id(struct bpf_verifier_env *env, in check_pseudo_btf_id() argument
12847 verbose(env, "invalid module BTF object FD specified.\n"); in check_pseudo_btf_id()
12852 …verbose(env, "kernel is missing BTF, make sure CONFIG_DEBUG_INFO_BTF=y is specified in Kconfig.\n"… in check_pseudo_btf_id()
12861 verbose(env, "ldimm64 insn specifies invalid btf_id %d.\n", id); in check_pseudo_btf_id()
12867 verbose(env, "pseudo btf_id %d in ldimm64 isn't KIND_VAR.\n", id); in check_pseudo_btf_id()
12875 verbose(env, "ldimm64 failed to find the address for kernel symbol '%s'.\n", in check_pseudo_btf_id()
12910 verbose(env, "ldimm64 unable to resolve the size of type '%s': %ld\n", in check_pseudo_btf_id()
12924 for (i = 0; i < env->used_btf_cnt; i++) { in check_pseudo_btf_id()
12925 if (env->used_btfs[i].btf == btf) { in check_pseudo_btf_id()
12931 if (env->used_btf_cnt >= MAX_USED_BTFS) { in check_pseudo_btf_id()
12936 btf_mod = &env->used_btfs[env->used_btf_cnt]; in check_pseudo_btf_id()
12949 env->used_btf_cnt++; in check_pseudo_btf_id()
12971 static int check_map_prog_compatibility(struct bpf_verifier_env *env, in check_map_prog_compatibility() argument
12980 verbose(env, "socket filter progs cannot use bpf_spin_lock yet\n"); in check_map_prog_compatibility()
12985 verbose(env, "tracing progs cannot use bpf_spin_lock yet\n"); in check_map_prog_compatibility()
12990 verbose(env, "sleepable progs cannot use bpf_spin_lock yet\n"); in check_map_prog_compatibility()
12997 verbose(env, "tracing progs cannot use bpf_timer yet\n"); in check_map_prog_compatibility()
13004 verbose(env, "offload device mismatch between prog and map\n"); in check_map_prog_compatibility()
13009 verbose(env, "bpf_struct_ops map cannot be used in prog\n"); in check_map_prog_compatibility()
13030 verbose(env, in check_map_prog_compatibility()
13051 static int resolve_pseudo_ldimm64(struct bpf_verifier_env *env) in resolve_pseudo_ldimm64() argument
13053 struct bpf_insn *insn = env->prog->insnsi; in resolve_pseudo_ldimm64()
13054 int insn_cnt = env->prog->len; in resolve_pseudo_ldimm64()
13057 err = bpf_prog_calc_tag(env->prog); in resolve_pseudo_ldimm64()
13064 verbose(env, "BPF_LDX uses reserved fields\n"); in resolve_pseudo_ldimm64()
13078 verbose(env, "invalid bpf_ld_imm64 insn\n"); in resolve_pseudo_ldimm64()
13087 aux = &env->insn_aux_data[i]; in resolve_pseudo_ldimm64()
13088 err = check_pseudo_btf_id(env, insn, aux); in resolve_pseudo_ldimm64()
13095 aux = &env->insn_aux_data[i]; in resolve_pseudo_ldimm64()
13113 verbose(env, "unrecognized bpf_ld_imm64 insn\n"); in resolve_pseudo_ldimm64()
13120 if (bpfptr_is_null(env->fd_array)) { in resolve_pseudo_ldimm64()
13121 verbose(env, "fd_idx without fd_array is invalid\n"); in resolve_pseudo_ldimm64()
13124 if (copy_from_bpfptr_offset(&fd, env->fd_array, in resolve_pseudo_ldimm64()
13137 verbose(env, "fd %d is not pointing to valid bpf_map\n", in resolve_pseudo_ldimm64()
13142 err = check_map_prog_compatibility(env, map, env->prog); in resolve_pseudo_ldimm64()
13148 aux = &env->insn_aux_data[i]; in resolve_pseudo_ldimm64()
13156 verbose(env, "direct value offset of %u is not allowed\n", off); in resolve_pseudo_ldimm64()
13162 verbose(env, "no direct value access support for this map type\n"); in resolve_pseudo_ldimm64()
13169 verbose(env, "invalid access to map value pointer, value_size=%u off=%u\n", in resolve_pseudo_ldimm64()
13183 for (j = 0; j < env->used_map_cnt; j++) { in resolve_pseudo_ldimm64()
13184 if (env->used_maps[j] == map) { in resolve_pseudo_ldimm64()
13191 if (env->used_map_cnt >= MAX_USED_MAPS) { in resolve_pseudo_ldimm64()
13203 aux->map_index = env->used_map_cnt; in resolve_pseudo_ldimm64()
13204 env->used_maps[env->used_map_cnt++] = map; in resolve_pseudo_ldimm64()
13207 bpf_cgroup_storage_assign(env->prog->aux, map)) { in resolve_pseudo_ldimm64()
13208 verbose(env, "only one cgroup storage of each type is allowed\n"); in resolve_pseudo_ldimm64()
13222 verbose(env, "unknown opcode %02x\n", insn->code); in resolve_pseudo_ldimm64()
13235 static void release_maps(struct bpf_verifier_env *env) in release_maps() argument
13237 __bpf_free_used_maps(env->prog->aux, env->used_maps, in release_maps()
13238 env->used_map_cnt); in release_maps()
13242 static void release_btfs(struct bpf_verifier_env *env) in release_btfs() argument
13244 __bpf_free_used_btfs(env->prog->aux, env->used_btfs, in release_btfs()
13245 env->used_btf_cnt); in release_btfs()
13249 static void convert_pseudo_ld_imm64(struct bpf_verifier_env *env) in convert_pseudo_ld_imm64() argument
13251 struct bpf_insn *insn = env->prog->insnsi; in convert_pseudo_ld_imm64()
13252 int insn_cnt = env->prog->len; in convert_pseudo_ld_imm64()
13268 static void adjust_insn_aux_data(struct bpf_verifier_env *env, in adjust_insn_aux_data() argument
13272 struct bpf_insn_aux_data *old_data = env->insn_aux_data; in adjust_insn_aux_data()
13282 old_data[off].zext_dst = insn_has_def32(env, insn + off + cnt - 1); in adjust_insn_aux_data()
13294 new_data[i].zext_dst = insn_has_def32(env, insn + i); in adjust_insn_aux_data()
13296 env->insn_aux_data = new_data; in adjust_insn_aux_data()
13300 static void adjust_subprog_starts(struct bpf_verifier_env *env, u32 off, u32 len) in adjust_subprog_starts() argument
13307 for (i = 0; i <= env->subprog_cnt; i++) { in adjust_subprog_starts()
13308 if (env->subprog_info[i].start <= off) in adjust_subprog_starts()
13310 env->subprog_info[i].start += len - 1; in adjust_subprog_starts()
13328 static struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, in bpf_patch_insn_data() argument
13335 new_data = vzalloc(array_size(env->prog->len + len - 1, in bpf_patch_insn_data()
13341 new_prog = bpf_patch_insn_single(env->prog, off, patch, len); in bpf_patch_insn_data()
13344 verbose(env, in bpf_patch_insn_data()
13346 env->insn_aux_data[off].orig_idx); in bpf_patch_insn_data()
13350 adjust_insn_aux_data(env, new_data, new_prog, off, len); in bpf_patch_insn_data()
13351 adjust_subprog_starts(env, off, len); in bpf_patch_insn_data()
13356 static int adjust_subprog_starts_after_remove(struct bpf_verifier_env *env, in adjust_subprog_starts_after_remove() argument
13362 for (i = 0; i < env->subprog_cnt; i++) in adjust_subprog_starts_after_remove()
13363 if (env->subprog_info[i].start >= off) in adjust_subprog_starts_after_remove()
13366 for (j = i; j < env->subprog_cnt; j++) in adjust_subprog_starts_after_remove()
13367 if (env->subprog_info[j].start >= off + cnt) in adjust_subprog_starts_after_remove()
13372 if (env->subprog_info[j].start != off + cnt) in adjust_subprog_starts_after_remove()
13376 struct bpf_prog_aux *aux = env->prog->aux; in adjust_subprog_starts_after_remove()
13380 move = env->subprog_cnt + 1 - j; in adjust_subprog_starts_after_remove()
13382 memmove(env->subprog_info + i, in adjust_subprog_starts_after_remove()
13383 env->subprog_info + j, in adjust_subprog_starts_after_remove()
13384 sizeof(*env->subprog_info) * move); in adjust_subprog_starts_after_remove()
13385 env->subprog_cnt -= j - i; in adjust_subprog_starts_after_remove()
13401 if (env->subprog_info[i].start == off) in adjust_subprog_starts_after_remove()
13406 for (; i <= env->subprog_cnt; i++) in adjust_subprog_starts_after_remove()
13407 env->subprog_info[i].start -= cnt; in adjust_subprog_starts_after_remove()
13412 static int bpf_adj_linfo_after_remove(struct bpf_verifier_env *env, u32 off, in bpf_adj_linfo_after_remove() argument
13415 struct bpf_prog *prog = env->prog; in bpf_adj_linfo_after_remove()
13462 for (i = 0; i <= env->subprog_cnt; i++) in bpf_adj_linfo_after_remove()
13463 if (env->subprog_info[i].linfo_idx > l_off) { in bpf_adj_linfo_after_remove()
13467 if (env->subprog_info[i].linfo_idx >= l_off + l_cnt) in bpf_adj_linfo_after_remove()
13468 env->subprog_info[i].linfo_idx -= l_cnt; in bpf_adj_linfo_after_remove()
13470 env->subprog_info[i].linfo_idx = l_off; in bpf_adj_linfo_after_remove()
13476 static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt) in verifier_remove_insns() argument
13478 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; in verifier_remove_insns()
13479 unsigned int orig_prog_len = env->prog->len; in verifier_remove_insns()
13482 if (bpf_prog_is_dev_bound(env->prog->aux)) in verifier_remove_insns()
13483 bpf_prog_offload_remove_insns(env, off, cnt); in verifier_remove_insns()
13485 err = bpf_remove_insns(env->prog, off, cnt); in verifier_remove_insns()
13489 err = adjust_subprog_starts_after_remove(env, off, cnt); in verifier_remove_insns()
13493 err = bpf_adj_linfo_after_remove(env, off, cnt); in verifier_remove_insns()
13514 static void sanitize_dead_code(struct bpf_verifier_env *env) in sanitize_dead_code() argument
13516 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; in sanitize_dead_code()
13518 struct bpf_insn *insn = env->prog->insnsi; in sanitize_dead_code()
13519 const int insn_cnt = env->prog->len; in sanitize_dead_code()
13544 static void opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env) in opt_hard_wire_dead_code_branches() argument
13546 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; in opt_hard_wire_dead_code_branches()
13548 struct bpf_insn *insn = env->prog->insnsi; in opt_hard_wire_dead_code_branches()
13549 const int insn_cnt = env->prog->len; in opt_hard_wire_dead_code_branches()
13563 if (bpf_prog_is_dev_bound(env->prog->aux)) in opt_hard_wire_dead_code_branches()
13564 bpf_prog_offload_replace_insn(env, i, &ja); in opt_hard_wire_dead_code_branches()
13570 static int opt_remove_dead_code(struct bpf_verifier_env *env) in opt_remove_dead_code() argument
13572 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; in opt_remove_dead_code()
13573 int insn_cnt = env->prog->len; in opt_remove_dead_code()
13585 err = verifier_remove_insns(env, i, j); in opt_remove_dead_code()
13588 insn_cnt = env->prog->len; in opt_remove_dead_code()
13594 static int opt_remove_nops(struct bpf_verifier_env *env) in opt_remove_nops() argument
13597 struct bpf_insn *insn = env->prog->insnsi; in opt_remove_nops()
13598 int insn_cnt = env->prog->len; in opt_remove_nops()
13605 err = verifier_remove_insns(env, i, 1); in opt_remove_nops()
13615 static int opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env, in opt_subreg_zext_lo32_rnd_hi32() argument
13619 struct bpf_insn_aux_data *aux = env->insn_aux_data; in opt_subreg_zext_lo32_rnd_hi32()
13620 int i, patch_len, delta = 0, len = env->prog->len; in opt_subreg_zext_lo32_rnd_hi32()
13621 struct bpf_insn *insns = env->prog->insnsi; in opt_subreg_zext_lo32_rnd_hi32()
13653 if (is_reg64(env, &insn, load_reg, NULL, DST_OP)) { in opt_subreg_zext_lo32_rnd_hi32()
13692 verbose(env, "verifier bug. zext_dst is set, but no reg is defined\n"); in opt_subreg_zext_lo32_rnd_hi32()
13702 new_prog = bpf_patch_insn_data(env, adj_idx, patch, patch_len); in opt_subreg_zext_lo32_rnd_hi32()
13705 env->prog = new_prog; in opt_subreg_zext_lo32_rnd_hi32()
13707 aux = env->insn_aux_data; in opt_subreg_zext_lo32_rnd_hi32()
13719 static int convert_ctx_accesses(struct bpf_verifier_env *env) in convert_ctx_accesses() argument
13721 const struct bpf_verifier_ops *ops = env->ops; in convert_ctx_accesses()
13723 const int insn_cnt = env->prog->len; in convert_ctx_accesses()
13730 if (ops->gen_prologue || env->seen_direct_write) { in convert_ctx_accesses()
13732 verbose(env, "bpf verifier is misconfigured\n"); in convert_ctx_accesses()
13735 cnt = ops->gen_prologue(insn_buf, env->seen_direct_write, in convert_ctx_accesses()
13736 env->prog); in convert_ctx_accesses()
13738 verbose(env, "bpf verifier is misconfigured\n"); in convert_ctx_accesses()
13741 new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt); in convert_ctx_accesses()
13745 env->prog = new_prog; in convert_ctx_accesses()
13750 if (bpf_prog_is_dev_bound(env->prog->aux)) in convert_ctx_accesses()
13753 insn = env->prog->insnsi + delta; in convert_ctx_accesses()
13780 env->insn_aux_data[i + delta].sanitize_stack_spill) { in convert_ctx_accesses()
13787 new_prog = bpf_patch_insn_data(env, i + delta, patch, cnt); in convert_ctx_accesses()
13792 env->prog = new_prog; in convert_ctx_accesses()
13800 switch ((int)env->insn_aux_data[i + delta].ptr_type) { in convert_ctx_accesses()
13821 env->prog->aux->num_exentries++; in convert_ctx_accesses()
13828 ctx_field_size = env->insn_aux_data[i + delta].ctx_field_size; in convert_ctx_accesses()
13843 verbose(env, "bpf verifier narrow ctx access misconfigured\n"); in convert_ctx_accesses()
13858 cnt = convert_ctx_access(type, insn, insn_buf, env->prog, in convert_ctx_accesses()
13862 verbose(env, "bpf verifier is misconfigured\n"); in convert_ctx_accesses()
13870 verbose(env, "bpf verifier narrow ctx load misconfigured\n"); in convert_ctx_accesses()
13890 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in convert_ctx_accesses()
13897 env->prog = new_prog; in convert_ctx_accesses()
13904 static int jit_subprogs(struct bpf_verifier_env *env) in jit_subprogs() argument
13906 struct bpf_prog *prog = env->prog, **func, *tmp; in jit_subprogs()
13913 if (env->subprog_cnt <= 1) in jit_subprogs()
13924 subprog = find_subprog(env, i + insn->imm + 1); in jit_subprogs()
13937 env->insn_aux_data[i].call_imm = insn->imm; in jit_subprogs()
13953 func = kcalloc(env->subprog_cnt, sizeof(prog), GFP_KERNEL); in jit_subprogs()
13957 for (i = 0; i < env->subprog_cnt; i++) { in jit_subprogs()
13959 subprog_end = env->subprog_info[i + 1].start; in jit_subprogs()
13995 func[i]->aux->stack_depth = env->subprog_info[i].stack_depth; in jit_subprogs()
14003 func[i]->aux->linfo_idx = env->subprog_info[i].linfo_idx; in jit_subprogs()
14012 func[i]->aux->tail_call_reachable = env->subprog_info[i].tail_call_reachable; in jit_subprogs()
14025 for (i = 0; i < env->subprog_cnt; i++) { in jit_subprogs()
14052 func[i]->aux->func_cnt = env->subprog_cnt; in jit_subprogs()
14054 for (i = 0; i < env->subprog_cnt; i++) { in jit_subprogs()
14058 verbose(env, "JIT doesn't support bpf-to-bpf calls\n"); in jit_subprogs()
14069 for (i = 1; i < env->subprog_cnt; i++) { in jit_subprogs()
14080 insn[0].imm = env->insn_aux_data[i].call_imm; in jit_subprogs()
14087 insn->off = env->insn_aux_data[i].call_imm; in jit_subprogs()
14088 subprog = find_subprog(env, i + insn->off + 1); in jit_subprogs()
14098 prog->aux->func_cnt = env->subprog_cnt; in jit_subprogs()
14114 for (i = 0; i < env->subprog_cnt; i++) { in jit_subprogs()
14129 insn->imm = env->insn_aux_data[i].call_imm; in jit_subprogs()
14135 static int fixup_call_args(struct bpf_verifier_env *env) in fixup_call_args() argument
14138 struct bpf_prog *prog = env->prog; in fixup_call_args()
14145 if (env->prog->jit_requested && in fixup_call_args()
14146 !bpf_prog_is_dev_bound(env->prog->aux)) { in fixup_call_args()
14147 err = jit_subprogs(env); in fixup_call_args()
14155 verbose(env, "calling kernel functions are not allowed in non-JITed programs\n"); in fixup_call_args()
14158 if (env->subprog_cnt > 1 && env->prog->aux->tail_call_reachable) { in fixup_call_args()
14162 verbose(env, "tail_calls are not allowed in non-JITed programs with bpf-to-bpf calls\n"); in fixup_call_args()
14170 verbose(env, "callbacks are not allowed in non-JITed programs\n"); in fixup_call_args()
14176 depth = get_callee_stack_depth(env, insn, i); in fixup_call_args()
14186 static int fixup_kfunc_call(struct bpf_verifier_env *env, in fixup_kfunc_call() argument
14192 verbose(env, "invalid kernel function call not eliminated in verifier pass\n"); in fixup_kfunc_call()
14199 desc = find_kfunc_desc(env->prog, insn->imm, insn->off); in fixup_kfunc_call()
14201 verbose(env, "verifier internal error: kernel function descriptor not found for func_id %u\n", in fixup_kfunc_call()
14214 static int do_misc_fixups(struct bpf_verifier_env *env) in do_misc_fixups() argument
14216 struct bpf_prog *prog = env->prog; in do_misc_fixups()
14261 new_prog = bpf_patch_insn_data(env, i + delta, patchlet, cnt); in do_misc_fixups()
14266 env->prog = prog = new_prog; in do_misc_fixups()
14275 cnt = env->ops->gen_ld_abs(insn, insn_buf); in do_misc_fixups()
14277 verbose(env, "bpf verifier is misconfigured\n"); in do_misc_fixups()
14281 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in do_misc_fixups()
14286 env->prog = prog = new_prog; in do_misc_fixups()
14300 aux = &env->insn_aux_data[i + delta]; in do_misc_fixups()
14334 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in do_misc_fixups()
14339 env->prog = prog = new_prog; in do_misc_fixups()
14349 ret = fixup_kfunc_call(env, insn); in do_misc_fixups()
14368 if (!allow_tail_call_in_subprogs(env)) in do_misc_fixups()
14380 aux = &env->insn_aux_data[i + delta]; in do_misc_fixups()
14381 if (env->bpf_capable && !prog->blinding_requested && in do_misc_fixups()
14395 verbose(env, "adding tail call poke descriptor failed\n"); in do_misc_fixups()
14413 verbose(env, "tail_call abusing map_ptr\n"); in do_misc_fixups()
14426 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in do_misc_fixups()
14431 env->prog = prog = new_prog; in do_misc_fixups()
14459 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in do_misc_fixups()
14464 env->prog = prog = new_prog; in do_misc_fixups()
14472 if (env->prog->aux->sleepable) in do_misc_fixups()
14479 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in do_misc_fixups()
14484 env->prog = prog = new_prog; in do_misc_fixups()
14503 aux = &env->insn_aux_data[i + delta]; in do_misc_fixups()
14515 verbose(env, "bpf verifier is misconfigured\n"); in do_misc_fixups()
14519 new_prog = bpf_patch_insn_data(env, i + delta, in do_misc_fixups()
14525 env->prog = prog = new_prog; in do_misc_fixups()
14602 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, in do_misc_fixups()
14608 env->prog = prog = new_prog; in do_misc_fixups()
14628 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in do_misc_fixups()
14633 env->prog = prog = new_prog; in do_misc_fixups()
14656 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in do_misc_fixups()
14661 env->prog = prog = new_prog; in do_misc_fixups()
14672 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 1); in do_misc_fixups()
14676 env->prog = prog = new_prog; in do_misc_fixups()
14687 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 1); in do_misc_fixups()
14691 env->prog = prog = new_prog; in do_misc_fixups()
14697 fn = env->ops->get_func_proto(insn->imm, env->prog); in do_misc_fixups()
14702 verbose(env, in do_misc_fixups()
14716 verbose(env, "bpf verifier is misconfigured\n"); in do_misc_fixups()
14722 verbose(env, "tracking tail call prog failed\n"); in do_misc_fixups()
14727 sort_kfunc_descs_by_imm(env->prog); in do_misc_fixups()
14732 static struct bpf_prog *inline_bpf_loop(struct bpf_verifier_env *env, in inline_bpf_loop() argument
14793 new_prog = bpf_patch_insn_data(env, position, insn_buf, *cnt); in inline_bpf_loop()
14798 callback_start = env->subprog_info[callback_subprogno].start; in inline_bpf_loop()
14823 static int optimize_bpf_loop(struct bpf_verifier_env *env) in optimize_bpf_loop() argument
14825 struct bpf_subprog_info *subprogs = env->subprog_info; in optimize_bpf_loop()
14827 struct bpf_insn *insn = env->prog->insnsi; in optimize_bpf_loop()
14828 int insn_cnt = env->prog->len; in optimize_bpf_loop()
14835 &env->insn_aux_data[i + delta].loop_inline_state; in optimize_bpf_loop()
14841 new_prog = inline_bpf_loop(env, in optimize_bpf_loop()
14850 env->prog = new_prog; in optimize_bpf_loop()
14863 env->prog->aux->stack_depth = env->subprog_info[0].stack_depth; in optimize_bpf_loop()
14868 static void free_states(struct bpf_verifier_env *env) in free_states() argument
14873 sl = env->free_list; in free_states()
14880 env->free_list = NULL; in free_states()
14882 if (!env->explored_states) in free_states()
14885 for (i = 0; i < state_htab_size(env); i++) { in free_states()
14886 sl = env->explored_states[i]; in free_states()
14894 env->explored_states[i] = NULL; in free_states()
14898 static int do_check_common(struct bpf_verifier_env *env, int subprog) in do_check_common() argument
14900 bool pop_log = !(env->log.level & BPF_LOG_LEVEL2); in do_check_common()
14905 env->prev_linfo = NULL; in do_check_common()
14906 env->pass_cnt++; in do_check_common()
14919 env->cur_state = state; in do_check_common()
14920 init_func_state(env, state->frame[0], in do_check_common()
14924 state->first_insn_idx = env->subprog_info[subprog].start; in do_check_common()
14928 if (subprog || env->prog->type == BPF_PROG_TYPE_EXT) { in do_check_common()
14929 ret = btf_prepare_func_args(env, subprog, regs); in do_check_common()
14934 mark_reg_known_zero(env, regs, i); in do_check_common()
14936 mark_reg_unknown(env, regs, i); in do_check_common()
14940 mark_reg_known_zero(env, regs, i); in do_check_common()
14942 regs[i].id = ++env->id_gen; in do_check_common()
14948 mark_reg_known_zero(env, regs, BPF_REG_1); in do_check_common()
14949 ret = btf_check_subprog_arg_match(env, subprog, regs); in do_check_common()
14963 ret = do_check(env); in do_check_common()
14968 if (env->cur_state) { in do_check_common()
14969 free_verifier_state(env->cur_state, true); in do_check_common()
14970 env->cur_state = NULL; in do_check_common()
14972 while (!pop_stack(env, NULL, NULL, false)); in do_check_common()
14974 bpf_vlog_reset(&env->log, 0); in do_check_common()
14975 free_states(env); in do_check_common()
14996 static int do_check_subprogs(struct bpf_verifier_env *env) in do_check_subprogs() argument
14998 struct bpf_prog_aux *aux = env->prog->aux; in do_check_subprogs()
15004 for (i = 1; i < env->subprog_cnt; i++) { in do_check_subprogs()
15007 env->insn_idx = env->subprog_info[i].start; in do_check_subprogs()
15008 WARN_ON_ONCE(env->insn_idx == 0); in do_check_subprogs()
15009 ret = do_check_common(env, i); in do_check_subprogs()
15012 } else if (env->log.level & BPF_LOG_LEVEL) { in do_check_subprogs()
15013 verbose(env, in do_check_subprogs()
15021 static int do_check_main(struct bpf_verifier_env *env) in do_check_main() argument
15025 env->insn_idx = 0; in do_check_main()
15026 ret = do_check_common(env, 0); in do_check_main()
15028 env->prog->aux->stack_depth = env->subprog_info[0].stack_depth; in do_check_main()
15033 static void print_verification_stats(struct bpf_verifier_env *env) in print_verification_stats() argument
15037 if (env->log.level & BPF_LOG_STATS) { in print_verification_stats()
15038 verbose(env, "verification time %lld usec\n", in print_verification_stats()
15039 div_u64(env->verification_time, 1000)); in print_verification_stats()
15040 verbose(env, "stack depth "); in print_verification_stats()
15041 for (i = 0; i < env->subprog_cnt; i++) { in print_verification_stats()
15042 u32 depth = env->subprog_info[i].stack_depth; in print_verification_stats()
15044 verbose(env, "%d", depth); in print_verification_stats()
15045 if (i + 1 < env->subprog_cnt) in print_verification_stats()
15046 verbose(env, "+"); in print_verification_stats()
15048 verbose(env, "\n"); in print_verification_stats()
15050 verbose(env, "processed %d insns (limit %d) max_states_per_insn %d " in print_verification_stats()
15052 env->insn_processed, BPF_COMPLEXITY_LIMIT_INSNS, in print_verification_stats()
15053 env->max_states_per_insn, env->total_states, in print_verification_stats()
15054 env->peak_states, env->longest_mark_read_walk); in print_verification_stats()
15057 static int check_struct_ops_btf_id(struct bpf_verifier_env *env) in check_struct_ops_btf_id() argument
15062 struct bpf_prog *prog = env->prog; in check_struct_ops_btf_id()
15067 verbose(env, "struct ops programs must have a GPL compatible license\n"); in check_struct_ops_btf_id()
15074 verbose(env, "attach_btf_id %u is not a supported struct\n", in check_struct_ops_btf_id()
15082 verbose(env, "attach to invalid member idx %u of struct %s\n", in check_struct_ops_btf_id()
15092 verbose(env, "attach to invalid member %s(@idx %u) of struct %s\n", in check_struct_ops_btf_id()
15101 verbose(env, "attach to unsupported member %s of struct %s\n", in check_struct_ops_btf_id()
15109 env->ops = st_ops->verifier_ops; in check_struct_ops_btf_id()
15391 static int check_attach_btf_id(struct bpf_verifier_env *env) in BTF_SET_START()
15393 struct bpf_prog *prog = env->prog; in BTF_SET_START()
15405 verbose(env, "Syscall programs can only be sleepable\n"); in BTF_SET_START()
15411 verbose(env, "Only fentry/fexit/fmod_ret, lsm, and kprobe/uprobe programs can be sleepable\n"); in BTF_SET_START()
15416 return check_struct_ops_btf_id(env); in BTF_SET_START()
15423 ret = bpf_check_attach_target(&env->log, prog, tgt_prog, btf_id, &tgt_info); in BTF_SET_START()
15432 env->ops = bpf_verifier_ops[tgt_prog->type]; in BTF_SET_START()
15455 ret = bpf_lsm_verify_prog(&env->log, prog); in BTF_SET_START()
15489 struct bpf_verifier_env *env; in bpf_check() local
15501 env = kzalloc(sizeof(struct bpf_verifier_env), GFP_KERNEL); in bpf_check()
15502 if (!env) in bpf_check()
15504 log = &env->log; in bpf_check()
15507 env->insn_aux_data = in bpf_check()
15510 if (!env->insn_aux_data) in bpf_check()
15513 env->insn_aux_data[i].orig_idx = i; in bpf_check()
15514 env->prog = *prog; in bpf_check()
15515 env->ops = bpf_verifier_ops[env->prog->type]; in bpf_check()
15516 env->fd_array = make_bpfptr(attr->fd_array, uattr.is_kernel); in bpf_check()
15540 mark_verifier_state_clean(env); in bpf_check()
15544 verbose(env, "in-kernel BTF is malformed\n"); in bpf_check()
15549 env->strict_alignment = !!(attr->prog_flags & BPF_F_STRICT_ALIGNMENT); in bpf_check()
15551 env->strict_alignment = true; in bpf_check()
15553 env->strict_alignment = false; in bpf_check()
15555 env->allow_ptr_leaks = bpf_allow_ptr_leaks(); in bpf_check()
15556 env->allow_uninit_stack = bpf_allow_uninit_stack(); in bpf_check()
15557 env->allow_ptr_to_map_access = bpf_allow_ptr_to_map_access(); in bpf_check()
15558 env->bypass_spec_v1 = bpf_bypass_spec_v1(); in bpf_check()
15559 env->bypass_spec_v4 = bpf_bypass_spec_v4(); in bpf_check()
15560 env->bpf_capable = bpf_capable(); in bpf_check()
15563 env->test_state_freq = attr->prog_flags & BPF_F_TEST_STATE_FREQ; in bpf_check()
15565 env->explored_states = kvcalloc(state_htab_size(env), in bpf_check()
15569 if (!env->explored_states) in bpf_check()
15572 ret = add_subprog_and_kfunc(env); in bpf_check()
15576 ret = check_subprogs(env); in bpf_check()
15580 ret = check_btf_info(env, attr, uattr); in bpf_check()
15584 ret = check_attach_btf_id(env); in bpf_check()
15588 ret = resolve_pseudo_ldimm64(env); in bpf_check()
15592 if (bpf_prog_is_dev_bound(env->prog->aux)) { in bpf_check()
15593 ret = bpf_prog_offload_verifier_prep(env->prog); in bpf_check()
15598 ret = check_cfg(env); in bpf_check()
15602 ret = do_check_subprogs(env); in bpf_check()
15603 ret = ret ?: do_check_main(env); in bpf_check()
15605 if (ret == 0 && bpf_prog_is_dev_bound(env->prog->aux)) in bpf_check()
15606 ret = bpf_prog_offload_finalize(env); in bpf_check()
15609 kvfree(env->explored_states); in bpf_check()
15612 ret = check_max_stack_depth(env); in bpf_check()
15616 ret = optimize_bpf_loop(env); in bpf_check()
15620 opt_hard_wire_dead_code_branches(env); in bpf_check()
15622 ret = opt_remove_dead_code(env); in bpf_check()
15624 ret = opt_remove_nops(env); in bpf_check()
15627 sanitize_dead_code(env); in bpf_check()
15632 ret = convert_ctx_accesses(env); in bpf_check()
15635 ret = do_misc_fixups(env); in bpf_check()
15640 if (ret == 0 && !bpf_prog_is_dev_bound(env->prog->aux)) { in bpf_check()
15641 ret = opt_subreg_zext_lo32_rnd_hi32(env, attr); in bpf_check()
15642 env->prog->aux->verifier_zext = bpf_jit_needs_zext() ? !ret in bpf_check()
15647 ret = fixup_call_args(env); in bpf_check()
15649 env->verification_time = ktime_get_ns() - start_time; in bpf_check()
15650 print_verification_stats(env); in bpf_check()
15651 env->prog->aux->verified_insns = env->insn_processed; in bpf_check()
15663 if (env->used_map_cnt) { in bpf_check()
15665 env->prog->aux->used_maps = kmalloc_array(env->used_map_cnt, in bpf_check()
15666 sizeof(env->used_maps[0]), in bpf_check()
15669 if (!env->prog->aux->used_maps) { in bpf_check()
15674 memcpy(env->prog->aux->used_maps, env->used_maps, in bpf_check()
15675 sizeof(env->used_maps[0]) * env->used_map_cnt); in bpf_check()
15676 env->prog->aux->used_map_cnt = env->used_map_cnt; in bpf_check()
15678 if (env->used_btf_cnt) { in bpf_check()
15680 env->prog->aux->used_btfs = kmalloc_array(env->used_btf_cnt, in bpf_check()
15681 sizeof(env->used_btfs[0]), in bpf_check()
15683 if (!env->prog->aux->used_btfs) { in bpf_check()
15688 memcpy(env->prog->aux->used_btfs, env->used_btfs, in bpf_check()
15689 sizeof(env->used_btfs[0]) * env->used_btf_cnt); in bpf_check()
15690 env->prog->aux->used_btf_cnt = env->used_btf_cnt; in bpf_check()
15692 if (env->used_map_cnt || env->used_btf_cnt) { in bpf_check()
15696 convert_pseudo_ld_imm64(env); in bpf_check()
15699 adjust_btf_func(env); in bpf_check()
15702 if (!env->prog->aux->used_maps) in bpf_check()
15706 release_maps(env); in bpf_check()
15707 if (!env->prog->aux->used_btfs) in bpf_check()
15708 release_btfs(env); in bpf_check()
15713 if (env->prog->type == BPF_PROG_TYPE_EXT) in bpf_check()
15714 env->prog->expected_attach_type = 0; in bpf_check()
15716 *prog = env->prog; in bpf_check()
15720 vfree(env->insn_aux_data); in bpf_check()
15722 kfree(env); in bpf_check()