Lines Matching refs:env
257 static const struct bpf_line_info *find_linfo(const struct bpf_verifier_env *env, u32 insn_off) in find_linfo() argument
263 prog = env->prog; in find_linfo()
320 __printf(2, 3) void bpf_verifier_log_write(struct bpf_verifier_env *env, const char *fmt, ...) in bpf_verifier_log_write() argument
324 if (!bpf_verifier_log_needed(&env->log)) { in bpf_verifier_log_write()
329 bpf_verifier_vlog(&env->log, fmt, args); in bpf_verifier_log_write()
336 struct bpf_verifier_env *env = private_data; in verbose() local
339 if (!bpf_verifier_log_needed(&env->log)) { in verbose()
344 bpf_verifier_vlog(&env->log, fmt, args); in verbose()
370 __printf(3, 4) static void verbose_linfo(struct bpf_verifier_env *env, u32 insn_off, const char *pr… in verbose_linfo() argument
374 if (!bpf_verifier_log_needed(&env->log)) { in verbose_linfo()
378 linfo = find_linfo(env, insn_off); in verbose_linfo()
379 if (!linfo || linfo == env->prev_linfo) { in verbose_linfo()
387 bpf_verifier_vlog(&env->log, prefix_fmt, args); in verbose_linfo()
391 verbose(env, "%s\n", ltrim(btf_name_by_offset(env->prog->aux->btf, linfo->line_off))); in verbose_linfo()
393 env->prev_linfo = linfo; in verbose_linfo()
481 static const char *reg_type_str(struct bpf_verifier_env *env, enum bpf_reg_type type) in reg_type_str() argument
521 …(void)snprintf(env->type_str_buf, TYPE_STR_BUF_LEN, "%s%s%s", prefix, str[base_type(type)], postfi… in reg_type_str()
522 return env->type_str_buf; in reg_type_str()
532 static void print_liveness(struct bpf_verifier_env *env, enum bpf_reg_liveness live) in print_liveness() argument
535 verbose(env, "_"); in print_liveness()
538 verbose(env, "r"); in print_liveness()
541 verbose(env, "w"); in print_liveness()
544 verbose(env, "D"); in print_liveness()
548 static struct bpf_func_state *func(struct bpf_verifier_env *env, const struct bpf_reg_state *reg) in func() argument
550 struct bpf_verifier_state *cur = env->cur_state; in func()
560 static void print_verifier_state(struct bpf_verifier_env *env, const struct bpf_func_state *state) in print_verifier_state() argument
567 verbose(env, " frame%d:", state->frameno); in print_verifier_state()
575 verbose(env, " R%d", i); in print_verifier_state()
576 print_liveness(env, reg->live); in print_verifier_state()
577 verbose(env, "=%s", reg_type_str(env, t)); in print_verifier_state()
579 verbose(env, "P"); in print_verifier_state()
583 verbose(env, "%lld", reg->var_off.value + reg->off); in print_verifier_state()
586 verbose(env, "%s", kernel_type_name(reg->btf_id)); in print_verifier_state()
588 verbose(env, "(id=%d", reg->id); in print_verifier_state()
590 verbose(env, ",ref_obj_id=%d", reg->ref_obj_id); in print_verifier_state()
593 verbose(env, ",off=%d", reg->off); in print_verifier_state()
596 verbose(env, ",r=%d", reg->range); in print_verifier_state()
598 verbose(env, ",ks=%d,vs=%d", reg->map_ptr->key_size, reg->map_ptr->value_size); in print_verifier_state()
605 verbose(env, ",imm=%llx", reg->var_off.value); in print_verifier_state()
608 verbose(env, ",smin_value=%lld", (long long)reg->smin_value); in print_verifier_state()
611 verbose(env, ",smax_value=%lld", (long long)reg->smax_value); in print_verifier_state()
614 verbose(env, ",umin_value=%llu", (unsigned long long)reg->umin_value); in print_verifier_state()
617 verbose(env, ",umax_value=%llu", (unsigned long long)reg->umax_value); in print_verifier_state()
623 verbose(env, ",var_off=%s", tn_buf); in print_verifier_state()
626 verbose(env, ",s32_min_value=%d", (int)(reg->s32_min_value)); in print_verifier_state()
629 verbose(env, ",s32_max_value=%d", (int)(reg->s32_max_value)); in print_verifier_state()
632 verbose(env, ",u32_min_value=%d", (int)(reg->u32_min_value)); in print_verifier_state()
635 verbose(env, ",u32_max_value=%d", (int)(reg->u32_max_value)); in print_verifier_state()
638 verbose(env, ")"); in print_verifier_state()
656 verbose(env, " fp%d", (-i - 1) * BPF_REG_SIZE); in print_verifier_state()
657 print_liveness(env, state->stack[i].spilled_ptr.live); in print_verifier_state()
661 verbose(env, "=%s", reg_type_str(env, t)); in print_verifier_state()
663 verbose(env, "P"); in print_verifier_state()
666 verbose(env, "%lld", reg->var_off.value + reg->off); in print_verifier_state()
669 verbose(env, "=%s", types_buf); in print_verifier_state()
673 verbose(env, " refs=%d", state->refs[0].id); in print_verifier_state()
676 verbose(env, ",%d", state->refs[i].id); in print_verifier_state()
680 verbose(env, "\n"); in print_verifier_state()
759 static int acquire_reference_state(struct bpf_verifier_env *env, int insn_idx) in acquire_reference_state() argument
761 struct bpf_func_state *state = cur_func(env); in acquire_reference_state()
769 id = ++env->id_gen; in acquire_reference_state()
903 static void update_branch_counts(struct bpf_verifier_env *env, struct bpf_verifier_state *st) in update_branch_counts() argument
919 static int pop_stack(struct bpf_verifier_env *env, int *prev_insn_idx, int *insn_idx, bool pop_log) in pop_stack() argument
921 struct bpf_verifier_state *cur = env->cur_state; in pop_stack()
922 struct bpf_verifier_stack_elem *elem, *head = env->head; in pop_stack()
925 if (env->head == NULL) { in pop_stack()
936 bpf_vlog_reset(&env->log, head->log_pos); in pop_stack()
947 env->head = elem; in pop_stack()
948 env->stack_size--; in pop_stack()
952 static struct bpf_verifier_state *push_stack(struct bpf_verifier_env *env, int insn_idx, int prev_i… in push_stack() argument
955 struct bpf_verifier_state *cur = env->cur_state; in push_stack()
966 elem->next = env->head; in push_stack()
967 elem->log_pos = env->log.len_used; in push_stack()
968 env->head = elem; in push_stack()
969 env->stack_size++; in push_stack()
975 if (env->stack_size > BPF_COMPLEXITY_LIMIT_JMP_SEQ) { in push_stack()
976 verbose(env, "The sequence of %d jumps is too complex.\n", env->stack_size); in push_stack()
993 free_verifier_state(env->cur_state, true); in push_stack()
994 env->cur_state = NULL; in push_stack()
996 while (!pop_stack(env, NULL, NULL, false)) { in push_stack()
1005 static void verifier_mark_reg_not_init(const struct bpf_verifier_env *env, struct bpf_reg_state *re…
1055 static void mark_reg_known_zero(struct bpf_verifier_env *env, struct bpf_reg_state *regs, u32 regno) in mark_reg_known_zero() argument
1058 verbose(env, "mark_reg_known_zero(regs, %u)\n", regno); in mark_reg_known_zero()
1061 verifier_mark_reg_not_init(env, regs + regno); in mark_reg_known_zero()
1318 static void __mark_reg_unknown(const struct bpf_verifier_env *env, struct bpf_reg_state *reg) in __mark_reg_unknown() argument
1328 reg->precise = env->subprog_cnt > 1 || !env->bpf_capable; in __mark_reg_unknown()
1332 static void mark_reg_unknown(struct bpf_verifier_env *env, struct bpf_reg_state *regs, u32 regno) in mark_reg_unknown() argument
1335 verbose(env, "mark_reg_unknown(regs, %u)\n", regno); in mark_reg_unknown()
1338 verifier_mark_reg_not_init(env, regs + regno); in mark_reg_unknown()
1342 __mark_reg_unknown(env, regs + regno); in mark_reg_unknown()
1345 static void verifier_mark_reg_not_init(const struct bpf_verifier_env *env, struct bpf_reg_state *re… in verifier_mark_reg_not_init() argument
1347 __mark_reg_unknown(env, reg); in verifier_mark_reg_not_init()
1351 static void mark_reg_not_init(struct bpf_verifier_env *env, struct bpf_reg_state *regs, u32 regno) in mark_reg_not_init() argument
1354 verbose(env, "mark_reg_not_init(regs, %u)\n", regno); in mark_reg_not_init()
1357 verifier_mark_reg_not_init(env, regs + regno); in mark_reg_not_init()
1361 verifier_mark_reg_not_init(env, regs + regno); in mark_reg_not_init()
1364 static void mark_btf_ld_reg(struct bpf_verifier_env *env, struct bpf_reg_state *regs, u32 regno, in mark_btf_ld_reg() argument
1368 mark_reg_unknown(env, regs, regno); in mark_btf_ld_reg()
1371 mark_reg_known_zero(env, regs, regno); in mark_btf_ld_reg()
1377 static void init_reg_state(struct bpf_verifier_env *env, struct bpf_func_state *state) in init_reg_state() argument
1383 mark_reg_not_init(env, regs, i); in init_reg_state()
1391 mark_reg_known_zero(env, regs, BPF_REG_FP); in init_reg_state()
1396 static void init_func_state(struct bpf_verifier_env *env, struct bpf_func_state *state, int callsit… in init_func_state() argument
1402 init_reg_state(env, state); in init_func_state()
1416 static int find_subprog(struct bpf_verifier_env *env, int off) in find_subprog() argument
1420 …p = bsearch(&off, env->subprog_info, env->subprog_cnt, sizeof(env->subprog_info[0]), cmp_subprogs); in find_subprog()
1424 return p - env->subprog_info; in find_subprog()
1427 static int add_subprog(struct bpf_verifier_env *env, int off) in add_subprog() argument
1429 int insn_cnt = env->prog->len; in add_subprog()
1433 verbose(env, "call to invalid destination\n"); in add_subprog()
1436 ret = find_subprog(env, off); in add_subprog()
1440 if (env->subprog_cnt >= BPF_MAX_SUBPROGS) { in add_subprog()
1441 verbose(env, "too many subprograms\n"); in add_subprog()
1444 env->subprog_info[env->subprog_cnt++].start = off; in add_subprog()
1445 sort(env->subprog_info, env->subprog_cnt, sizeof(env->subprog_info[0]), cmp_subprogs, NULL); in add_subprog()
1449 static int check_subprogs(struct bpf_verifier_env *env) in check_subprogs() argument
1452 struct bpf_subprog_info *subprog = env->subprog_info; in check_subprogs()
1453 struct bpf_insn *insn = env->prog->insnsi; in check_subprogs()
1454 int insn_cnt = env->prog->len; in check_subprogs()
1457 ret = add_subprog(env, 0); in check_subprogs()
1470 if (!env->bpf_capable) { in check_subprogs()
1471 … verbose(env, "function calls to other bpf functions are allowed for CAP_BPF and CAP_SYS_ADMIN\n"); in check_subprogs()
1474 ret = add_subprog(env, i + insn[i].imm + 1); in check_subprogs()
1483 subprog[env->subprog_cnt].start = insn_cnt; in check_subprogs()
1485 if (env->log.level & BPF_LOG_LEVEL2) { in check_subprogs()
1486 for (i = 0; i < env->subprog_cnt; i++) { in check_subprogs()
1487 verbose(env, "func#%d @%d\n", i, subprog[i].start); in check_subprogs()
1511 verbose(env, "jump out of range from insn %d to %d\n", i, off); in check_subprogs()
1521 verbose(env, "last insn is not an exit or jmp\n"); in check_subprogs()
1526 if (cur_subprog < env->subprog_cnt) { in check_subprogs()
1537 static int mark_reg_read(struct bpf_verifier_env *env, const struct bpf_reg_state *state, struct bp… in mark_reg_read() argument
1549 … verbose(env, "verifier BUG type %s var_off %lld off %d\n", reg_type_str(env, parent->type), in mark_reg_read()
1580 if (env->longest_mark_read_walk < cnt) { in mark_reg_read()
1581 env->longest_mark_read_walk = cnt; in mark_reg_read()
1590 static bool is_reg64(struct bpf_verifier_env *env, struct bpf_insn *insn, u32 regno, struct bpf_reg… in is_reg64() argument
1689 static bool insn_has_def32(struct bpf_verifier_env *env, struct bpf_insn *insn) in insn_has_def32() argument
1695 return !is_reg64(env, insn, insn->dst_reg, NULL, DST_OP); in insn_has_def32()
1698 static void mark_insn_zext(struct bpf_verifier_env *env, struct bpf_reg_state *reg) in mark_insn_zext() argument
1706 env->insn_aux_data[def_idx - 1].zext_dst = true; in mark_insn_zext()
1711 static int check_reg_arg(struct bpf_verifier_env *env, u32 regno, enum reg_arg_type t) in check_reg_arg() argument
1713 struct bpf_verifier_state *vstate = env->cur_state; in check_reg_arg()
1715 struct bpf_insn *insn = env->prog->insnsi + env->insn_idx; in check_reg_arg()
1720 verbose(env, "R%d is invalid\n", regno); in check_reg_arg()
1725 rw64 = is_reg64(env, insn, regno, reg, t); in check_reg_arg()
1729 verbose(env, "R%d !read_ok\n", regno); in check_reg_arg()
1738 mark_insn_zext(env, reg); in check_reg_arg()
1741 return mark_reg_read(env, reg, reg->parent, rw64 ? REG_LIVE_READ64 : REG_LIVE_READ32); in check_reg_arg()
1745 verbose(env, "frame pointer is read only\n"); in check_reg_arg()
1749 reg->subreg_def = rw64 ? DEF_NOT_SUBREG : env->insn_idx + 1; in check_reg_arg()
1751 mark_reg_unknown(env, regs, regno); in check_reg_arg()
1758 static int push_jmp_history(struct bpf_verifier_env *env, struct bpf_verifier_state *cur) in push_jmp_history() argument
1768 p[cnt - 1].idx = env->insn_idx; in push_jmp_history()
1769 p[cnt - 1].prev_idx = env->prev_insn_idx; in push_jmp_history()
1795 static int backtrack_insn(struct bpf_verifier_env *env, int idx, u32 *reg_mask, u64 *stack_mask) in backtrack_insn() argument
1799 .private_data = env, in backtrack_insn()
1801 struct bpf_insn *insn = env->prog->insnsi + idx; in backtrack_insn()
1812 if (env->log.level & BPF_LOG_LEVEL) { in backtrack_insn()
1813 verbose(env, "regs=%x stack=%llx before ", *reg_mask, *stack_mask); in backtrack_insn()
1814 verbose(env, "%d: ", idx); in backtrack_insn()
1815 print_bpf_insn(&cbs, insn, env->allow_ptr_leaks); in backtrack_insn()
1876 verbose(env, "BUG spi %d\n", spi); in backtrack_insn()
1898 verbose(env, "BUG spi %d\n", spi); in backtrack_insn()
1920 verbose(env, "BUG regs %x\n", *reg_mask); in backtrack_insn()
1996 static void mark_all_scalars_precise(struct bpf_verifier_env *env, struct bpf_verifier_state *st) in mark_all_scalars_precise() argument
2029 static int __mark_chain_precision(struct bpf_verifier_env *env, int regno, int spi) in __mark_chain_precision() argument
2031 struct bpf_verifier_state *st = env->cur_state; in __mark_chain_precision()
2033 int last_idx = env->insn_idx; in __mark_chain_precision()
2042 if (!env->bpf_capable) { in __mark_chain_precision()
2090 if (env->log.level & BPF_LOG_LEVEL) { in __mark_chain_precision()
2091 verbose(env, "last_idx %d first_idx %d\n", last_idx, first_idx); in __mark_chain_precision()
2098 err = backtrack_insn(env, i, ®_mask, &stack_mask); in __mark_chain_precision()
2101 mark_all_scalars_precise(env, st); in __mark_chain_precision()
2117 if (i >= env->prog->len) { in __mark_chain_precision()
2124 verbose(env, "BUG backtracking idx %d\n", i); in __mark_chain_precision()
2167 mark_all_scalars_precise(env, st); in __mark_chain_precision()
2185 if (env->log.level & BPF_LOG_LEVEL) { in __mark_chain_precision()
2186 print_verifier_state(env, func); in __mark_chain_precision()
2187 …verbose(env, "parent %s regs=%x stack=%llx marks\n", new_marks ? "didn't have" : "already had", re… in __mark_chain_precision()
2204 static int mark_chain_precision(struct bpf_verifier_env *env, int regno) in mark_chain_precision() argument
2206 return __mark_chain_precision(env, regno, -1); in mark_chain_precision()
2209 static int mark_chain_precision_stack(struct bpf_verifier_env *env, int spi) in mark_chain_precision_stack() argument
2211 return __mark_chain_precision(env, -1, spi); in mark_chain_precision_stack()
2286 static int check_stack_write_fixed_off(struct bpf_verifier_env *env, in check_stack_write_fixed_off() argument
2292 u32 dst_reg = env->prog->insnsi[insn_idx].dst_reg; in check_stack_write_fixed_off()
2302 …if (!env->allow_ptr_leaks && state->stack[spi].slot_type[0] == STACK_SPILL && size != BPF_REG_SIZE… in check_stack_write_fixed_off()
2303 verbose(env, "attempt to corrupt spilled pointer on stack\n"); in check_stack_write_fixed_off()
2307 cur = env->cur_state->frame[env->cur_state->curframe]; in check_stack_write_fixed_off()
2311 if (!env->bypass_spec_v4) { in check_stack_write_fixed_off()
2322 env->insn_aux_data[insn_idx].sanitize_stack_spill = true; in check_stack_write_fixed_off()
2326 …if (reg && size == BPF_REG_SIZE && register_is_bounded(reg) && !register_is_null(reg) && env->bpf_… in check_stack_write_fixed_off()
2334 err = mark_chain_precision(env, value_regno); in check_stack_write_fixed_off()
2343 verbose_linfo(env, insn_idx, "; "); in check_stack_write_fixed_off()
2344 verbose(env, "invalid size of register spill\n"); in check_stack_write_fixed_off()
2348 verbose(env, "cannot spill pointers to stack into stack frame of the caller\n"); in check_stack_write_fixed_off()
2379 err = mark_chain_precision(env, value_regno); in check_stack_write_fixed_off()
2413 static int check_stack_write_var_off(struct bpf_verifier_env *env, in check_stack_write_var_off() argument
2428 cur = env->cur_state->frame[env->cur_state->curframe]; in check_stack_write_var_off()
2453 if (!env->allow_ptr_leaks && *stype != NOT_INIT && *stype != SCALAR_VALUE) { in check_stack_write_var_off()
2460 … verbose(env, "spilled ptr in range of var-offset stack write; insn %d, ptr off: %d", insn_idx, i); in check_stack_write_var_off()
2481 if (*stype == STACK_INVALID && !env->allow_uninit_stack) { in check_stack_write_var_off()
2482 …verbose(env, "uninit stack in range of var-offset write prohibited for !root; insn %d, off: %d", i… in check_stack_write_var_off()
2490 err = mark_chain_precision(env, value_regno); in check_stack_write_var_off()
2506 static void mark_reg_stack_read(struct bpf_verifier_env *env, in mark_reg_stack_read() argument
2510 struct bpf_verifier_state *vstate = env->cur_state; in mark_reg_stack_read()
2543 mark_reg_unknown(env, state->regs, dst_regno); in mark_reg_stack_read()
2557 static int check_stack_read_fixed_off(struct bpf_verifier_env *env, in check_stack_read_fixed_off() argument
2561 struct bpf_verifier_state *vstate = env->cur_state; in check_stack_read_fixed_off()
2573 verbose_linfo(env, env->insn_idx, "; "); in check_stack_read_fixed_off()
2574 verbose(env, "invalid size of register fill\n"); in check_stack_read_fixed_off()
2578 mark_reg_unknown(env, state->regs, dst_regno); in check_stack_read_fixed_off()
2581 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64); in check_stack_read_fixed_off()
2586 verbose(env, "corrupted spill memory\n"); in check_stack_read_fixed_off()
2599 } else if (__is_pointer_value(env->allow_ptr_leaks, reg)) { in check_stack_read_fixed_off()
2606 verbose(env, "leaking pointer from stack off %d\n", off); in check_stack_read_fixed_off()
2609 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64); in check_stack_read_fixed_off()
2621 verbose(env, "invalid read from stack off %d+%d size %d\n", off, i, size); in check_stack_read_fixed_off()
2624 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64); in check_stack_read_fixed_off()
2626 mark_reg_stack_read(env, reg_state, off, off + size, dst_regno); in check_stack_read_fixed_off()
2637 static int check_stack_range_initialized(struct bpf_verifier_env *env, int regno, int off, int acce…
2641 static struct bpf_reg_state *reg_state(struct bpf_verifier_env *env, int regno) in reg_state() argument
2643 return cur_regs(env) + regno; in reg_state()
2659 static int check_stack_read_var_off(struct bpf_verifier_env *env, int ptr_regno, int off, int size,… in check_stack_read_var_off() argument
2662 struct bpf_reg_state *reg = reg_state(env, ptr_regno); in check_stack_read_var_off()
2663 struct bpf_func_state *ptr_state = func(env, reg); in check_stack_read_var_off()
2669 err = check_stack_range_initialized(env, ptr_regno, off, size, false, ACCESS_DIRECT, NULL); in check_stack_read_var_off()
2676 mark_reg_stack_read(env, ptr_state, min_off, max_off + size, dst_regno); in check_stack_read_var_off()
2689 static int check_stack_read(struct bpf_verifier_env *env, int ptr_regno, int off, int size, int dst… in check_stack_read() argument
2691 struct bpf_reg_state *reg = reg_state(env, ptr_regno); in check_stack_read()
2692 struct bpf_func_state *state = func(env, reg); in check_stack_read()
2704 …verbose(env, "variable offset stack pointer cannot be passed into helper function; var_off=%s off=… in check_stack_read()
2712 if (!env->bypass_spec_v1 && var_off) { in check_stack_read()
2716 …verbose(env, "R%d variable offset stack access prohibited for !root, var_off=%s\n", ptr_regno, tn_… in check_stack_read()
2722 err = check_stack_read_fixed_off(env, state, off, size, dst_regno); in check_stack_read()
2728 err = check_stack_read_var_off(env, ptr_regno, off, size, dst_regno); in check_stack_read()
2743 static int check_stack_write(struct bpf_verifier_env *env, int ptr_regno, int off, int size, int va… in check_stack_write() argument
2746 struct bpf_reg_state *reg = reg_state(env, ptr_regno); in check_stack_write()
2747 struct bpf_func_state *state = func(env, reg); in check_stack_write()
2752 err = check_stack_write_fixed_off(env, state, off, size, value_regno, insn_idx); in check_stack_write()
2757 err = check_stack_write_var_off(env, state, ptr_regno, off, size, value_regno, insn_idx); in check_stack_write()
2762 static int check_map_access_type(struct bpf_verifier_env *env, u32 regno, int off, int size, enum b… in check_map_access_type() argument
2764 struct bpf_reg_state *regs = cur_regs(env); in check_map_access_type()
2768 …verbose(env, "write into map forbidden, value_size=%d off=%d size=%d\n", map->value_size, off, siz… in check_map_access_type()
2772 …verbose(env, "read from map forbidden, value_size=%d off=%d size=%d\n", map->value_size, off, size… in check_map_access_type()
2780 static int __check_mem_access(struct bpf_verifier_env *env, int regno, int off, int size, u32 mem_s… in __check_mem_access() argument
2790 reg = &cur_regs(env)[regno]; in __check_mem_access()
2793 … verbose(env, "invalid access to map value, value_size=%d off=%d size=%d\n", mem_size, off, size); in __check_mem_access()
2798 …verbose(env, "invalid access to packet, off=%d size=%d, R%d(id=%d,off=%d,r=%d)\n", off, size, regn… in __check_mem_access()
2803 … verbose(env, "invalid access to memory, mem_size=%u off=%d size=%d\n", mem_size, off, size); in __check_mem_access()
2810 static int check_mem_region_access(struct bpf_verifier_env *env, u32 regno, int off, int size, u32 … in check_mem_region_access() argument
2813 struct bpf_verifier_state *vstate = env->cur_state; in check_mem_region_access()
2822 if (env->log.level & BPF_LOG_LEVEL) { in check_mem_region_access()
2823 print_verifier_state(env, state); in check_mem_region_access()
2835 …verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n… in check_mem_region_access()
2838 err = __check_mem_access(env, regno, reg->smin_value + off, size, mem_size, zero_size_allowed); in check_mem_region_access()
2840 verbose(env, "R%d min value is outside of the allowed memory range\n", regno); in check_mem_region_access()
2849 … verbose(env, "R%d unbounded memory access, make sure to bounds check any such access\n", regno); in check_mem_region_access()
2852 err = __check_mem_access(env, regno, reg->umax_value + off, size, mem_size, zero_size_allowed); in check_mem_region_access()
2854 verbose(env, "R%d max value is outside of the allowed memory range\n", regno); in check_mem_region_access()
2862 static int check_map_access(struct bpf_verifier_env *env, u32 regno, int off, int size, bool zero_s… in check_map_access() argument
2864 struct bpf_verifier_state *vstate = env->cur_state; in check_map_access()
2870 err = check_mem_region_access(env, regno, off, size, map->value_size, zero_size_allowed); in check_map_access()
2884 verbose(env, "bpf_spin_lock cannot be accessed directly by load/store\n"); in check_map_access()
2898 static bool may_access_direct_pkt_data(struct bpf_verifier_env *env, const struct bpf_call_arg_meta… in may_access_direct_pkt_data() argument
2901 enum bpf_prog_type prog_type = resolve_prog_type(env->prog); in may_access_direct_pkt_data()
2927 env->seen_direct_write = true; in may_access_direct_pkt_data()
2932 env->seen_direct_write = true; in may_access_direct_pkt_data()
2942 static int check_packet_access(struct bpf_verifier_env *env, u32 regno, int off, int size, bool zer… in check_packet_access() argument
2944 struct bpf_reg_state *regs = cur_regs(env); in check_packet_access()
2957 …verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n… in check_packet_access()
2960 err = __check_mem_access(env, regno, off, size, reg->range, zero_size_allowed); in check_packet_access()
2962 verbose(env, "R%d offset is outside of the packet\n", regno); in check_packet_access()
2972 …env->prog->aux->max_pkt_offset = max_t(u32, env->prog->aux->max_pkt_offset, off + reg->umax_value … in check_packet_access()
2978 static int check_ctx_access(struct bpf_verifier_env *env, int insn_idx, int off, int size, enum bpf… in check_ctx_access() argument
2983 .log = &env->log, in check_ctx_access()
2986 if (env->ops->is_valid_access && env->ops->is_valid_access(off, size, t, env->prog, &info)) { in check_ctx_access()
2999 env->insn_aux_data[insn_idx].ctx_field_size = info.ctx_field_size; in check_ctx_access()
3002 if (env->prog->aux->max_ctx_offset < off + size) { in check_ctx_access()
3003 env->prog->aux->max_ctx_offset = off + size; in check_ctx_access()
3008 verbose(env, "invalid bpf_context access off=%d size=%d\n", off, size); in check_ctx_access()
3012 static int check_flow_keys_access(struct bpf_verifier_env *env, int off, int size) in check_flow_keys_access() argument
3015 verbose(env, "invalid access to flow keys off=%d size=%d\n", off, size); in check_flow_keys_access()
3021 static int check_sock_access(struct bpf_verifier_env *env, int insn_idx, u32 regno, int off, int si… in check_sock_access() argument
3024 struct bpf_reg_state *regs = cur_regs(env); in check_sock_access()
3030 …verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n… in check_sock_access()
3052 env->insn_aux_data[insn_idx].ctx_field_size = info.ctx_field_size; in check_sock_access()
3056 …verbose(env, "R%d invalid %s access off=%d size=%d\n", regno, reg_type_str(env, reg->type), off, s… in check_sock_access()
3061 static bool is_pointer_value(struct bpf_verifier_env *env, int regno) in is_pointer_value() argument
3063 return __is_pointer_value(env->allow_ptr_leaks, reg_state(env, regno)); in is_pointer_value()
3066 static bool is_ctx_reg(struct bpf_verifier_env *env, int regno) in is_ctx_reg() argument
3068 const struct bpf_reg_state *reg = reg_state(env, regno); in is_ctx_reg()
3073 static bool is_sk_reg(struct bpf_verifier_env *env, int regno) in is_sk_reg() argument
3075 const struct bpf_reg_state *reg = reg_state(env, regno); in is_sk_reg()
3080 static bool is_pkt_reg(struct bpf_verifier_env *env, int regno) in is_pkt_reg() argument
3082 const struct bpf_reg_state *reg = reg_state(env, regno); in is_pkt_reg()
3087 static bool is_flow_key_reg(struct bpf_verifier_env *env, int regno) in is_flow_key_reg() argument
3089 const struct bpf_reg_state *reg = reg_state(env, regno); in is_flow_key_reg()
3095 static int check_pkt_ptr_alignment(struct bpf_verifier_env *env, const struct bpf_reg_state *reg, i… in check_pkt_ptr_alignment() argument
3121 …verbose(env, "misaligned packet access off %d+%s+%d+%d size %d\n", ip_align, tn_buf, reg->off, off… in check_pkt_ptr_alignment()
3128 static int check_generic_ptr_alignment(struct bpf_verifier_env *env, const struct bpf_reg_state *re… in check_generic_ptr_alignment() argument
3143 …verbose(env, "misaligned %saccess off %s+%d+%d size %d\n", pointer_desc, tn_buf, reg->off, off, si… in check_generic_ptr_alignment()
3150 static int check_ptr_alignment(struct bpf_verifier_env *env, const struct bpf_reg_state *reg, int o… in check_ptr_alignment() argument
3153 bool strict = env->strict_alignment || strict_alignment_once; in check_ptr_alignment()
3162 return check_pkt_ptr_alignment(env, reg, off, size, strict); in check_ptr_alignment()
3195 return check_generic_ptr_alignment(env, reg, pointer_desc, off, size, strict); in check_ptr_alignment()
3198 static int update_stack_depth(struct bpf_verifier_env *env, const struct bpf_func_state *func, int … in update_stack_depth() argument
3200 u16 stack = env->subprog_info[func->subprogno].stack_depth; in update_stack_depth()
3207 env->subprog_info[func->subprogno].stack_depth = -off; in update_stack_depth()
3217 static int check_max_stack_depth(struct bpf_verifier_env *env) in check_max_stack_depth() argument
3220 struct bpf_subprog_info *subprog = env->subprog_info; in check_max_stack_depth()
3221 struct bpf_insn *insn = env->prog->insnsi; in check_max_stack_depth()
3251 …verbose(env, "tail_calls are not allowed when call stack of previous frames is %d bytes. Too large… in check_max_stack_depth()
3260 … verbose(env, "combined stack size of %d calls is %d. Too large\n", frame + 1, depth); in check_max_stack_depth()
3280 idx = find_subprog(env, i); in check_max_stack_depth()
3292 verbose(env, "the call stack of %d frames is too deep !\n", frame); in check_max_stack_depth()
3317 env->prog->aux->tail_call_reachable = true; in check_max_stack_depth()
3336 static int get_callee_stack_depth(struct bpf_verifier_env *env, const struct bpf_insn *insn, int id… in get_callee_stack_depth() argument
3340 subprog = find_subprog(env, start); in get_callee_stack_depth()
3345 return env->subprog_info[subprog].stack_depth; in get_callee_stack_depth()
3349 static int __check_ptr_off_reg(struct bpf_verifier_env *env, const struct bpf_reg_state *reg, int r… in __check_ptr_off_reg() argument
3357 …verbose(env, "dereference of modified %s ptr R%d off=%d disallowed\n", reg_type_str(env, reg->type… in __check_ptr_off_reg()
3366 … verbose(env, "variable %s access var_off=%s disallowed\n", reg_type_str(env, reg->type), tn_buf); in __check_ptr_off_reg()
3373 int check_ptr_off_reg(struct bpf_verifier_env *env, const struct bpf_reg_state *reg, int regno) in check_ptr_off_reg() argument
3375 return __check_ptr_off_reg(env, reg, regno, false); in check_ptr_off_reg()
3378 static int __check_buffer_access(struct bpf_verifier_env *env, const char *buf_info, const struct b… in __check_buffer_access() argument
3382 verbose(env, "R%d invalid %s buffer access: off=%d, size=%d\n", regno, buf_info, off, size); in __check_buffer_access()
3389 … verbose(env, "R%d invalid variable buffer offset: off=%d, var_off=%s\n", regno, off, tn_buf); in __check_buffer_access()
3396 static int check_tp_buffer_access(struct bpf_verifier_env *env, const struct bpf_reg_state *reg, in… in check_tp_buffer_access() argument
3401 err = __check_buffer_access(env, "tracepoint", reg, regno, off, size); in check_tp_buffer_access()
3406 if (off + size > env->prog->aux->max_tp_access) { in check_tp_buffer_access()
3407 env->prog->aux->max_tp_access = off + size; in check_tp_buffer_access()
3413 static int check_buffer_access(struct bpf_verifier_env *env, const struct bpf_reg_state *reg, int r… in check_buffer_access() argument
3418 err = __check_buffer_access(env, buf_info, reg, regno, off, size); in check_buffer_access()
3505 static int check_ptr_to_btf_access(struct bpf_verifier_env *env, struct bpf_reg_state *regs, int re… in check_ptr_to_btf_access() argument
3515 verbose(env, "R%d is ptr_%s invalid negative access: off=%d\n", regno, tname, off); in check_ptr_to_btf_access()
3522 …verbose(env, "R%d is ptr_%s invalid variable offset: off=%d, var_off=%s\n", regno, tname, off, tn_… in check_ptr_to_btf_access()
3526 if (env->ops->btf_struct_access) { in check_ptr_to_btf_access()
3527 ret = env->ops->btf_struct_access(&env->log, t, off, size, atype, &btf_id); in check_ptr_to_btf_access()
3530 verbose(env, "only read is supported\n"); in check_ptr_to_btf_access()
3534 ret = btf_struct_access(&env->log, t, off, size, atype, &btf_id); in check_ptr_to_btf_access()
3542 mark_btf_ld_reg(env, regs, value_regno, ret, btf_id); in check_ptr_to_btf_access()
3548 static int check_ptr_to_map_access(struct bpf_verifier_env *env, struct bpf_reg_state *regs, int re… in check_ptr_to_map_access() argument
3559 verbose(env, "map_ptr access not supported without CONFIG_DEBUG_INFO_BTF\n"); in check_ptr_to_map_access()
3564 verbose(env, "map_ptr access not supported for map type %d\n", map->map_type); in check_ptr_to_map_access()
3571 if (!env->allow_ptr_to_map_access) { in check_ptr_to_map_access()
3572 verbose(env, "%s access is allowed only to CAP_PERFMON and CAP_SYS_ADMIN\n", tname); in check_ptr_to_map_access()
3577 verbose(env, "R%d is %s invalid negative access: off=%d\n", regno, tname, off); in check_ptr_to_map_access()
3582 verbose(env, "only read from %s is supported\n", tname); in check_ptr_to_map_access()
3586 ret = btf_struct_access(&env->log, t, off, size, atype, &btf_id); in check_ptr_to_map_access()
3592 mark_btf_ld_reg(env, regs, value_regno, ret, btf_id); in check_ptr_to_map_access()
3625 static int check_stack_access_within_bounds(struct bpf_verifier_env *env, int regno, int off, int a… in check_stack_access_within_bounds() argument
3628 struct bpf_reg_state *regs = cur_regs(env); in check_stack_access_within_bounds()
3630 struct bpf_func_state *state = func(env, reg); in check_stack_access_within_bounds()
3653 verbose(env, "invalid unbounded variable-offset%s stack R%d\n", err_extra, regno); in check_stack_access_within_bounds()
3671 … verbose(env, "invalid%s stack R%d off=%d size=%d\n", err_extra, regno, off, access_size); in check_stack_access_within_bounds()
3676 … verbose(env, "invalid variable-offset%s stack R%d var_off=%s size=%d\n", err_extra, regno, tn_buf, in check_stack_access_within_bounds()
3689 static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regno, int off, int bpf… in check_mem_access() argument
3692 struct bpf_reg_state *regs = cur_regs(env); in check_mem_access()
3703 err = check_ptr_alignment(env, reg, off, size, strict_alignment_once); in check_mem_access()
3712 if (t == BPF_WRITE && value_regno >= 0 && is_pointer_value(env, value_regno)) { in check_mem_access()
3713 verbose(env, "R%d leaks addr into map\n", value_regno); in check_mem_access()
3716 err = check_map_access_type(env, regno, off, size, t); in check_mem_access()
3720 err = check_map_access(env, regno, off, size, false); in check_mem_access()
3737 mark_reg_unknown(env, regs, value_regno); in check_mem_access()
3744 verbose(env, "R%d invalid mem access '%s'\n", regno, reg_type_str(env, reg->type)); in check_mem_access()
3749 verbose(env, "R%d cannot write into %s\n", regno, reg_type_str(env, reg->type)); in check_mem_access()
3753 if (t == BPF_WRITE && value_regno >= 0 && is_pointer_value(env, value_regno)) { in check_mem_access()
3754 verbose(env, "R%d leaks addr into mem\n", value_regno); in check_mem_access()
3758 err = check_mem_region_access(env, regno, off, size, reg->mem_size, false); in check_mem_access()
3760 mark_reg_unknown(env, regs, value_regno); in check_mem_access()
3766 if (t == BPF_WRITE && value_regno >= 0 && is_pointer_value(env, value_regno)) { in check_mem_access()
3767 verbose(env, "R%d leaks addr into ctx\n", value_regno); in check_mem_access()
3771 err = check_ptr_off_reg(env, reg, regno); in check_mem_access()
3776 err = check_ctx_access(env, insn_idx, off, size, t, ®_type, &btf_id); in check_mem_access()
3778 verbose_linfo(env, insn_idx, "; "); in check_mem_access()
3786 mark_reg_unknown(env, regs, value_regno); in check_mem_access()
3788 mark_reg_known_zero(env, regs, value_regno); in check_mem_access()
3790 regs[value_regno].id = ++env->id_gen; in check_mem_access()
3806 err = check_stack_access_within_bounds(env, regno, off, size, ACCESS_DIRECT, t); in check_mem_access()
3810 state = func(env, reg); in check_mem_access()
3811 err = update_stack_depth(env, state, off); in check_mem_access()
3817 err = check_stack_read(env, regno, off, size, value_regno); in check_mem_access()
3819 err = check_stack_write(env, regno, off, size, value_regno, insn_idx); in check_mem_access()
3822 if (t == BPF_WRITE && !may_access_direct_pkt_data(env, NULL, t)) { in check_mem_access()
3823 verbose(env, "cannot write into packet\n"); in check_mem_access()
3826 if (t == BPF_WRITE && value_regno >= 0 && is_pointer_value(env, value_regno)) { in check_mem_access()
3827 verbose(env, "R%d leaks addr into packet\n", value_regno); in check_mem_access()
3830 err = check_packet_access(env, regno, off, size, false); in check_mem_access()
3832 mark_reg_unknown(env, regs, value_regno); in check_mem_access()
3835 if (t == BPF_WRITE && value_regno >= 0 && is_pointer_value(env, value_regno)) { in check_mem_access()
3836 verbose(env, "R%d leaks addr into flow keys\n", value_regno); in check_mem_access()
3840 err = check_flow_keys_access(env, off, size); in check_mem_access()
3842 mark_reg_unknown(env, regs, value_regno); in check_mem_access()
3846 verbose(env, "R%d cannot write into %s\n", regno, reg_type_str(env, reg->type)); in check_mem_access()
3849 err = check_sock_access(env, insn_idx, regno, off, size, t); in check_mem_access()
3851 mark_reg_unknown(env, regs, value_regno); in check_mem_access()
3854 err = check_tp_buffer_access(env, reg, regno, off, size); in check_mem_access()
3856 mark_reg_unknown(env, regs, value_regno); in check_mem_access()
3859 err = check_ptr_to_btf_access(env, regs, regno, off, size, t, value_regno); in check_mem_access()
3861 err = check_ptr_to_map_access(env, regs, regno, off, size, t, value_regno); in check_mem_access()
3869 verbose(env, "R%d cannot write into %s\n", regno, reg_type_str(env, reg->type)); in check_mem_access()
3873 max_access = &env->prog->aux->max_rdonly_access; in check_mem_access()
3876 max_access = &env->prog->aux->max_rdwr_access; in check_mem_access()
3879 err = check_buffer_access(env, reg, regno, off, size, false, buf_info, max_access); in check_mem_access()
3881 mark_reg_unknown(env, regs, value_regno); in check_mem_access()
3884 verbose(env, "R%d invalid mem access '%s'\n", regno, reg_type_str(env, reg->type)); in check_mem_access()
3895 static int check_xadd(struct bpf_verifier_env *env, int insn_idx, struct bpf_insn *insn) in check_xadd() argument
3900 verbose(env, "BPF_XADD uses reserved fields\n"); in check_xadd()
3905 err = check_reg_arg(env, insn->src_reg, SRC_OP); in check_xadd()
3911 err = check_reg_arg(env, insn->dst_reg, SRC_OP); in check_xadd()
3916 if (is_pointer_value(env, insn->src_reg)) { in check_xadd()
3917 verbose(env, "R%d leaks addr into mem\n", insn->src_reg); in check_xadd()
3921 …if (is_ctx_reg(env, insn->dst_reg) || is_pkt_reg(env, insn->dst_reg) || is_flow_key_reg(env, insn-… in check_xadd()
3922 is_sk_reg(env, insn->dst_reg)) { in check_xadd()
3923 verbose(env, "BPF_XADD stores into R%d %s is not allowed\n", insn->dst_reg, in check_xadd()
3924 reg_type_str(env, reg_state(env, insn->dst_reg)->type)); in check_xadd()
3929 …err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off, BPF_SIZE(insn->code), BPF_READ, -1… in check_xadd()
3935 …return check_mem_access(env, insn_idx, insn->dst_reg, insn->off, BPF_SIZE(insn->code), BPF_WRITE, … in check_xadd()
3947 static int check_stack_range_initialized(struct bpf_verifier_env *env, int regno, int off, int acce… in check_stack_range_initialized() argument
3951 struct bpf_reg_state *reg = reg_state(env, regno); in check_stack_range_initialized()
3952 struct bpf_func_state *state = func(env, reg); in check_stack_range_initialized()
3962 verbose(env, "invalid zero-sized read\n"); in check_stack_range_initialized()
3976 err = check_stack_access_within_bounds(env, regno, off, access_size, type, bounds_check_type); in check_stack_range_initialized()
3989 if (!env->bypass_spec_v1) { in check_stack_range_initialized()
3993 …verbose(env, "R%d%s variable offset stack access prohibited for !root, var_off=%s\n", regno, err_e… in check_stack_range_initialized()
4042 (state->stack[spi].spilled_ptr.type == SCALAR_VALUE || env->allow_ptr_leaks)) { in check_stack_range_initialized()
4044 __mark_reg_unknown(env, &state->stack[spi].spilled_ptr); in check_stack_range_initialized()
4054 …verbose(env, "invalid%s read from stack R%d off %d+%d size %d\n", err_extra, regno, min_off, i - m… in check_stack_range_initialized()
4060 …verbose(env, "invalid%s read from stack R%d var_off %s+%d size %d\n", err_extra, regno, tn_buf, i … in check_stack_range_initialized()
4068 …mark_reg_read(env, &state->stack[spi].spilled_ptr, state->stack[spi].spilled_ptr.parent, REG_LIVE_… in check_stack_range_initialized()
4070 return update_stack_depth(env, state, min_off); in check_stack_range_initialized()
4073 static int check_helper_mem_access(struct bpf_verifier_env *env, int regno, int access_size, bool z… in check_helper_mem_access() argument
4076 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; in check_helper_mem_access()
4083 return check_packet_access(env, regno, reg->off, access_size, zero_size_allowed); in check_helper_mem_access()
4085 if (check_map_access_type(env, regno, reg->off, access_size, in check_helper_mem_access()
4089 return check_map_access(env, regno, reg->off, access_size, zero_size_allowed); in check_helper_mem_access()
4091 …return check_mem_region_access(env, regno, reg->off, access_size, reg->mem_size, zero_size_allowed… in check_helper_mem_access()
4099 max_access = &env->prog->aux->max_rdonly_access; in check_helper_mem_access()
4102 max_access = &env->prog->aux->max_rdwr_access; in check_helper_mem_access()
4104 …return check_buffer_access(env, reg, regno, reg->off, access_size, zero_size_allowed, buf_info, ma… in check_helper_mem_access()
4106 …return check_stack_range_initialized(env, regno, reg->off, access_size, zero_size_allowed, ACCESS_… in check_helper_mem_access()
4114 verbose(env, "R%d type=%s ", regno, reg_type_str(env, reg->type)); in check_helper_mem_access()
4115 verbose(env, "expected=%s\n", reg_type_str(env, PTR_TO_STACK)); in check_helper_mem_access()
4139 static int process_spin_lock(struct bpf_verifier_env *env, int regno, bool is_lock) in process_spin_lock() argument
4141 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; in process_spin_lock()
4142 struct bpf_verifier_state *cur = env->cur_state; in process_spin_lock()
4148 …verbose(env, "R%d doesn't have constant offset. bpf_spin_lock has to be at the constant offset\n",… in process_spin_lock()
4152 verbose(env, "map '%s' has to have BTF in order to use bpf_spin_lock\n", map->name); in process_spin_lock()
4157 verbose(env, "map '%s' has more than one 'struct bpf_spin_lock'\n", map->name); in process_spin_lock()
4159 verbose(env, "map '%s' doesn't have 'struct bpf_spin_lock'\n", map->name); in process_spin_lock()
4161 verbose(env, "map '%s' is not a struct type or bpf_spin_lock is mangled\n", map->name); in process_spin_lock()
4166 verbose(env, "off %lld doesn't point to 'struct bpf_spin_lock'\n", val + reg->off); in process_spin_lock()
4171 verbose(env, "Locking two bpf_spin_locks are not allowed\n"); in process_spin_lock()
4177 verbose(env, "bpf_spin_unlock without taking a lock\n"); in process_spin_lock()
4181 verbose(env, "bpf_spin_unlock of different lock\n"); in process_spin_lock()
4220 static int resolve_map_arg_type(struct bpf_verifier_env *env, const struct bpf_call_arg_meta *meta, in resolve_map_arg_type() argument
4225 verbose(env, "invalid map_ptr to access map->type\n"); in resolve_map_arg_type()
4235 verbose(env, "invalid arg_type for sockmap/sockhash\n"); in resolve_map_arg_type()
4341 static int check_reg_type(struct bpf_verifier_env *env, u32 regno, enum bpf_arg_type arg_type, cons… in check_reg_type() argument
4343 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; in check_reg_type()
4350 verbose(env, "verifier internal error: unsupported arg type %d\n", arg_type); in check_reg_type()
4382 verbose(env, "R%d type=%s expected=", regno, reg_type_str(env, reg->type)); in check_reg_type()
4384 verbose(env, "%s, ", reg_type_str(env, compatible->types[j])); in check_reg_type()
4386 verbose(env, "%s\n", reg_type_str(env, compatible->types[j])); in check_reg_type()
4393 verbose(env, "verifier internal error: missing arg compatible BTF ID\n"); in check_reg_type()
4399 if (!btf_struct_ids_match(&env->log, reg->off, reg->btf_id, *arg_btf_id)) { in check_reg_type()
4400 … verbose(env, "R%d is of type %s but %s is expected\n", regno, kernel_type_name(reg->btf_id), in check_reg_type()
4409 static int check_func_arg(struct bpf_verifier_env *env, u32 arg, struct bpf_call_arg_meta *meta, in check_func_arg() argument
4413 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; in check_func_arg()
4422 err = check_reg_arg(env, regno, SRC_OP); in check_func_arg()
4428 if (is_pointer_value(env, regno)) { in check_func_arg()
4429 verbose(env, "R%d leaks addr into helper function\n", regno); in check_func_arg()
4435 if (type_is_pkt_pointer(type) && !may_access_direct_pkt_data(env, meta, BPF_READ)) { in check_func_arg()
4436 verbose(env, "helper access to the packet is not allowed\n"); in check_func_arg()
4441 err = resolve_map_arg_type(env, meta, &arg_type); in check_func_arg()
4454 err = check_reg_type(env, regno, arg_type, fn->arg_btf_id[arg]); in check_func_arg()
4481 err = __check_ptr_off_reg(env, reg, regno, type == PTR_TO_BTF_ID); in check_func_arg()
4491 … verbose(env, "verifier internal error: more than one arg with ref_obj_id R%d %u %u\n", regno, in check_func_arg()
4512 verbose(env, "invalid map_ptr to access map->key\n"); in check_func_arg()
4515 err = check_helper_mem_access(env, regno, meta->map_ptr->key_size, false, NULL); in check_func_arg()
4526 verbose(env, "invalid map_ptr to access map->value\n"); in check_func_arg()
4530 err = check_helper_mem_access(env, regno, meta->map_ptr->value_size, false, meta); in check_func_arg()
4533 verbose(env, "Helper has invalid btf_id in R%d\n", regno); in check_func_arg()
4539 if (process_spin_lock(env, regno, true)) { in check_func_arg()
4543 if (process_spin_lock(env, regno, false)) { in check_func_arg()
4547 verbose(env, "verifier internal error\n"); in check_func_arg()
4581 … verbose(env, "R%d min value is negative, either use unsigned or 'var &= const'\n", regno); in check_func_arg()
4586 err = check_helper_mem_access(env, regno - 1, 0, zero_size_allowed, meta); in check_func_arg()
4593 … verbose(env, "R%d unbounded memory access, use 'var &= const' or 'if (var < const)'\n", regno); in check_func_arg()
4596 err = check_helper_mem_access(env, regno - 1, reg->umax_value, zero_size_allowed, meta); in check_func_arg()
4598 err = mark_chain_precision(env, regno); in check_func_arg()
4602 verbose(env, "R%d unbounded size, use 'var &= const' or 'if (var < const)'\n", regno); in check_func_arg()
4609 err = check_helper_mem_access(env, regno, size, false, meta); in check_func_arg()
4613 err = check_ptr_alignment(env, reg, 0, size, true); in check_func_arg()
4619 static bool may_update_sockmap(struct bpf_verifier_env *env, int func_id) in may_update_sockmap() argument
4621 enum bpf_attach_type eatype = env->prog->expected_attach_type; in may_update_sockmap()
4622 enum bpf_prog_type type = resolve_prog_type(env->prog); in may_update_sockmap()
4649 verbose(env, "cannot update sockmap in this context\n"); in may_update_sockmap()
4653 static bool allow_tail_call_in_subprogs(struct bpf_verifier_env *env) in allow_tail_call_in_subprogs() argument
4655 return env->prog->jit_requested && IS_ENABLED(CONFIG_X86_64); in allow_tail_call_in_subprogs()
4658 static int check_map_func_compatibility(struct bpf_verifier_env *env, struct bpf_map *map, int func… in check_map_func_compatibility() argument
4729 !may_update_sockmap(env, func_id)) { in check_map_func_compatibility()
4737 !may_update_sockmap(env, func_id)) { in check_map_func_compatibility()
4773 if (env->subprog_cnt > 1 && !allow_tail_call_in_subprogs(env)) { in check_map_func_compatibility()
4774 … verbose(env, "tail_calls are not allowed in non-JITed programs with bpf-to-bpf calls\n"); in check_map_func_compatibility()
4861 …verbose(env, "cannot pass map_type %d into func %s#%d\n", map->map_type, func_id_name(func_id), fu… in check_map_func_compatibility()
4976 static void __clear_all_pkt_pointers(struct bpf_verifier_env *env, struct bpf_func_state *state) in __clear_all_pkt_pointers() argument
4983 mark_reg_unknown(env, regs, i); in __clear_all_pkt_pointers()
4993 __mark_reg_unknown(env, reg); in __clear_all_pkt_pointers()
4998 static void clear_all_pkt_pointers(struct bpf_verifier_env *env) in clear_all_pkt_pointers() argument
5000 struct bpf_verifier_state *vstate = env->cur_state; in clear_all_pkt_pointers()
5004 __clear_all_pkt_pointers(env, vstate->frame[i]); in clear_all_pkt_pointers()
5008 static void release_reg_references(struct bpf_verifier_env *env, struct bpf_func_state *state, int … in release_reg_references() argument
5015 mark_reg_unknown(env, regs, i); in release_reg_references()
5025 __mark_reg_unknown(env, reg); in release_reg_references()
5033 static int release_reference(struct bpf_verifier_env *env, int ref_obj_id) in release_reference() argument
5035 struct bpf_verifier_state *vstate = env->cur_state; in release_reference()
5039 err = release_reference_state(cur_func(env), ref_obj_id); in release_reference()
5045 release_reg_references(env, vstate->frame[i], ref_obj_id); in release_reference()
5051 static void clear_caller_saved_regs(struct bpf_verifier_env *env, struct bpf_reg_state *regs) in clear_caller_saved_regs() argument
5057 mark_reg_not_init(env, regs, caller_saved[i]); in clear_caller_saved_regs()
5058 check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); in clear_caller_saved_regs()
5062 static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, int *insn_idx) in check_func_call() argument
5064 struct bpf_verifier_state *state = env->cur_state; in check_func_call()
5071 verbose(env, "the call stack of %d frames is too deep\n", state->curframe + 2); in check_func_call()
5076 subprog = find_subprog(env, target_insn + 1); in check_func_call()
5078 verbose(env, "verifier bug. No program starts at insn %d\n", target_insn + 1); in check_func_call()
5084 verbose(env, "verifier bug. Frame %d already allocated\n", state->curframe + 1); in check_func_call()
5088 func_info_aux = env->prog->aux->func_info_aux; in check_func_call()
5092 err = btf_check_func_arg_match(env, subprog, caller->regs); in check_func_call()
5098 verbose(env, "Caller passes invalid args into func#%d\n", subprog); in check_func_call()
5101 if (env->log.level & BPF_LOG_LEVEL) { in check_func_call()
5102 verbose(env, "Func#%d is global and valid. Skipping.\n", subprog); in check_func_call()
5104 clear_caller_saved_regs(env, caller->regs); in check_func_call()
5107 mark_reg_unknown(env, caller->regs, BPF_REG_0); in check_func_call()
5125 init_func_state(env, callee, in check_func_call()
5143 clear_caller_saved_regs(env, caller->regs); in check_func_call()
5151 if (env->log.level & BPF_LOG_LEVEL) { in check_func_call()
5152 verbose(env, "caller:\n"); in check_func_call()
5153 print_verifier_state(env, caller); in check_func_call()
5154 verbose(env, "callee:\n"); in check_func_call()
5155 print_verifier_state(env, callee); in check_func_call()
5160 static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx) in prepare_func_exit() argument
5162 struct bpf_verifier_state *state = env->cur_state; in prepare_func_exit()
5176 verbose(env, "cannot return stack pointer to the caller\n"); in prepare_func_exit()
5192 if (env->log.level & BPF_LOG_LEVEL) { in prepare_func_exit()
5193 verbose(env, "returning from callee:\n"); in prepare_func_exit()
5194 print_verifier_state(env, callee); in prepare_func_exit()
5195 verbose(env, "to caller at %d:\n", *insn_idx); in prepare_func_exit()
5196 print_verifier_state(env, caller); in prepare_func_exit()
5222 static int record_func_map(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, int func_i… in record_func_map() argument
5224 struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx]; in record_func_map()
5234 verbose(env, "kernel subsystem misconfigured verifier\n"); in record_func_map()
5245 verbose(env, "write into map forbidden\n"); in record_func_map()
5257 static int record_func_key(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, int func_i… in record_func_key() argument
5259 struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx]; in record_func_key()
5260 struct bpf_reg_state *regs = cur_regs(env), *reg; in record_func_key()
5269 verbose(env, "kernel subsystem misconfigured verifier\n"); in record_func_key()
5282 err = mark_chain_precision(env, BPF_REG_3); in record_func_key()
5295 static int check_reference_leak(struct bpf_verifier_env *env) in check_reference_leak() argument
5297 struct bpf_func_state *state = cur_func(env); in check_reference_leak()
5301 …verbose(env, "Unreleased reference id=%d alloc_insn=%d\n", state->refs[i].id, state->refs[i].insn_… in check_reference_leak()
5306 static int check_helper_call(struct bpf_verifier_env *env, int func_id, int insn_idx) in check_helper_call() argument
5318 verbose(env, "invalid func %s#%d\n", func_id_name(func_id), func_id); in check_helper_call()
5322 if (env->ops->get_func_proto) { in check_helper_call()
5323 fn = env->ops->get_func_proto(func_id, env->prog); in check_helper_call()
5326 verbose(env, "unknown func %s#%d\n", func_id_name(func_id), func_id); in check_helper_call()
5331 if (!env->prog->gpl_compatible && fn->gpl_only) { in check_helper_call()
5332 verbose(env, "cannot call GPL-restricted function from non-GPL compatible program\n"); in check_helper_call()
5336 if (fn->allowed && !fn->allowed(env->prog)) { in check_helper_call()
5337 verbose(env, "helper call is not allowed in probe\n"); in check_helper_call()
5344 …verbose(env, "kernel subsystem misconfigured func %s#%d: r1 != ctx\n", func_id_name(func_id), func… in check_helper_call()
5353 verbose(env, "kernel subsystem misconfigured func %s#%d\n", func_id_name(func_id), func_id); in check_helper_call()
5360 err = check_func_arg(env, i, &meta, fn); in check_helper_call()
5366 err = record_func_map(env, &meta, func_id, insn_idx); in check_helper_call()
5371 err = record_func_key(env, &meta, func_id, insn_idx); in check_helper_call()
5380 err = check_mem_access(env, insn_idx, meta.regno, i, BPF_B, BPF_WRITE, -1, false); in check_helper_call()
5387 err = check_reference_leak(env); in check_helper_call()
5389 verbose(env, "tail_call would lead to reference leak\n"); in check_helper_call()
5393 err = release_reference(env, meta.ref_obj_id); in check_helper_call()
5395 …verbose(env, "func %s#%d reference has not been acquired before\n", func_id_name(func_id), func_id… in check_helper_call()
5400 regs = cur_regs(env); in check_helper_call()
5405 verbose(env, "get_local_storage() doesn't support non-zero flags\n"); in check_helper_call()
5411 mark_reg_not_init(env, regs, caller_saved[i]); in check_helper_call()
5412 check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); in check_helper_call()
5423 mark_reg_unknown(env, regs, BPF_REG_0); in check_helper_call()
5428 mark_reg_known_zero(env, regs, BPF_REG_0); in check_helper_call()
5434 verbose(env, "kernel subsystem misconfigured verifier\n"); in check_helper_call()
5440 regs[BPF_REG_0].id = ++env->id_gen; in check_helper_call()
5443 mark_reg_known_zero(env, regs, BPF_REG_0); in check_helper_call()
5446 mark_reg_known_zero(env, regs, BPF_REG_0); in check_helper_call()
5449 mark_reg_known_zero(env, regs, BPF_REG_0); in check_helper_call()
5452 mark_reg_known_zero(env, regs, BPF_REG_0); in check_helper_call()
5458 mark_reg_known_zero(env, regs, BPF_REG_0); in check_helper_call()
5469 verbose(env, "unable to resolve the size of type '%s': %ld\n", tname, PTR_ERR(ret)); in check_helper_call()
5488 mark_reg_known_zero(env, regs, BPF_REG_0); in check_helper_call()
5492 …verbose(env, "invalid return type %u of func %s#%d\n", base_type(ret_type), func_id_name(func_id),… in check_helper_call()
5497 …verbose(env, "unknown return type %u of func %s#%d\n", base_type(ret_type), func_id_name(func_id),… in check_helper_call()
5502 regs[BPF_REG_0].id = ++env->id_gen; in check_helper_call()
5509 int id = acquire_reference_state(env, insn_idx); in check_helper_call()
5521 err = check_map_func_compatibility(env, meta.map_ptr, func_id); in check_helper_call()
5526 …if ((func_id == BPF_FUNC_get_stack || func_id == BPF_FUNC_get_task_stack) && !env->prog->has_callc… in check_helper_call()
5537 verbose(env, err_str, func_id_name(func_id), func_id); in check_helper_call()
5541 env->prog->has_callchain_buf = true; in check_helper_call()
5545 env->prog->call_get_stack = true; in check_helper_call()
5549 clear_all_pkt_pointers(env); in check_helper_call()
5598 static bool check_reg_sane_offset(struct bpf_verifier_env *env, const struct bpf_reg_state *reg, en… in check_reg_sane_offset() argument
5605 … verbose(env, "math between %s pointer and %lld is not allowed\n", reg_type_str(env, type), val); in check_reg_sane_offset()
5610 verbose(env, "%s pointer offset %d is not allowed\n", reg_type_str(env, type), reg->off); in check_reg_sane_offset()
5615 … verbose(env, "math between %s pointer and register with unbounded min value is not allowed\n", in check_reg_sane_offset()
5616 reg_type_str(env, type)); in check_reg_sane_offset()
5621 … verbose(env, "value %lld makes %s pointer be out of bounds\n", smin, reg_type_str(env, type)); in check_reg_sane_offset()
5628 static struct bpf_insn_aux_data *cur_aux(struct bpf_verifier_env *env) in cur_aux() argument
5630 return &env->insn_aux_data[env->insn_idx]; in cur_aux()
5670 static bool can_skip_alu_sanitation(const struct bpf_verifier_env *env, const struct bpf_insn *insn) in can_skip_alu_sanitation() argument
5672 return env->bypass_spec_v1 || BPF_SRC(insn->code) == BPF_K; in can_skip_alu_sanitation()
5690 static int sanitize_val_alu(struct bpf_verifier_env *env, struct bpf_insn *insn) in sanitize_val_alu() argument
5692 struct bpf_insn_aux_data *aux = cur_aux(env); in sanitize_val_alu()
5694 if (can_skip_alu_sanitation(env, insn)) { in sanitize_val_alu()
5711 static struct bpf_verifier_state *sanitize_speculative_path(struct bpf_verifier_env *env, const str… in sanitize_speculative_path() argument
5717 branch = push_stack(env, next_idx, curr_idx, true); in sanitize_speculative_path()
5721 mark_reg_unknown(env, regs, insn->dst_reg); in sanitize_speculative_path()
5723 mark_reg_unknown(env, regs, insn->dst_reg); in sanitize_speculative_path()
5724 mark_reg_unknown(env, regs, insn->src_reg); in sanitize_speculative_path()
5730 static int sanitize_ptr_alu(struct bpf_verifier_env *env, struct bpf_insn *insn, const struct bpf_r… in sanitize_ptr_alu() argument
5734 struct bpf_insn_aux_data *aux = commit_window ? cur_aux(env) : &info->aux; in sanitize_ptr_alu()
5735 struct bpf_verifier_state *vstate = env->cur_state; in sanitize_ptr_alu()
5745 if (can_skip_alu_sanitation(env, insn)) { in sanitize_ptr_alu()
5785 env->explore_alu_limits = true; in sanitize_ptr_alu()
5819 ret = sanitize_speculative_path(env, NULL, env->insn_idx + 1, env->insn_idx); in sanitize_ptr_alu()
5826 static void sanitize_mark_insn_seen(struct bpf_verifier_env *env) in sanitize_mark_insn_seen() argument
5828 struct bpf_verifier_state *vstate = env->cur_state; in sanitize_mark_insn_seen()
5836 env->insn_aux_data[env->insn_idx].seen = env->pass_cnt; in sanitize_mark_insn_seen()
5840 static int sanitize_err(struct bpf_verifier_env *env, const struct bpf_insn *insn, int reason, in sanitize_err() argument
5849 …verbose(env, "R%d has unknown scalar with mixed signed bounds, %s\n", off_reg == dst_reg ? dst : s… in sanitize_err()
5852 …verbose(env, "R%d has pointer with unsupported alu operation, %s\n", off_reg == dst_reg ? src : ds… in sanitize_err()
5855 … verbose(env, "R%d tried to %s from different maps, paths or scalars, %s\n", dst, op, err); in sanitize_err()
5858 verbose(env, "R%d tried to %s beyond pointer bounds, %s\n", dst, op, err); in sanitize_err()
5861 verbose(env, "R%d could not be pushed for speculative verification, %s\n", dst, err); in sanitize_err()
5864 verbose(env, "verifier internal error: unknown reason (%d)\n", reason); in sanitize_err()
5881 static int check_stack_access_for_ptr_arithmetic(struct bpf_verifier_env *env, int regno, in check_stack_access_for_ptr_arithmetic() argument
5888 …verbose(env, "R%d variable stack access prohibited for !root, var_off=%s off=%d\n", regno, tn_buf,… in check_stack_access_for_ptr_arithmetic()
5893 verbose(env, in check_stack_access_for_ptr_arithmetic()
5903 static int sanitize_check_bounds(struct bpf_verifier_env *env, const struct bpf_insn *insn, in sanitize_check_bounds() argument
5911 if (env->bypass_spec_v1) { in sanitize_check_bounds()
5917 …if (check_stack_access_for_ptr_arithmetic(env, dst, dst_reg, dst_reg->off + dst_reg->var_off.value… in sanitize_check_bounds()
5922 if (check_map_access(env, dst, dst_reg->off, 1, false)) { in sanitize_check_bounds()
5923 verbose(env, in sanitize_check_bounds()
5942 static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, struct bpf_insn *insn, in adjust_ptr_min_max_vals() argument
5945 struct bpf_verifier_state *vstate = env->cur_state; in adjust_ptr_min_max_vals()
5964 __mark_reg_unknown(env, dst_reg); in adjust_ptr_min_max_vals()
5970 if (opcode == BPF_SUB && env->allow_ptr_leaks) { in adjust_ptr_min_max_vals()
5971 __mark_reg_unknown(env, dst_reg); in adjust_ptr_min_max_vals()
5975 verbose(env, "R%d 32-bit pointer arithmetic prohibited\n", dst); in adjust_ptr_min_max_vals()
5980 verbose(env, "R%d pointer arithmetic on %s prohibited, null-check it first\n", dst, in adjust_ptr_min_max_vals()
5981 reg_type_str(env, ptr_reg->type)); in adjust_ptr_min_max_vals()
5998 … verbose(env, "R%d pointer arithmetic on %s prohibited\n", dst, reg_type_str(env, ptr_reg->type)); in adjust_ptr_min_max_vals()
6013 …if (!check_reg_sane_offset(env, off_reg, ptr_reg->type) || !check_reg_sane_offset(env, ptr_reg, pt… in adjust_ptr_min_max_vals()
6021 ret = sanitize_ptr_alu(env, insn, ptr_reg, off_reg, dst_reg, &info, false); in adjust_ptr_min_max_vals()
6023 return sanitize_err(env, insn, ret, off_reg, dst_reg); in adjust_ptr_min_max_vals()
6070 dst_reg->id = ++env->id_gen; in adjust_ptr_min_max_vals()
6078 verbose(env, "R%d tried to subtract pointer from scalar\n", dst); in adjust_ptr_min_max_vals()
6086 verbose(env, "R%d subtraction from stack pointer prohibited\n", dst); in adjust_ptr_min_max_vals()
6125 dst_reg->id = ++env->id_gen; in adjust_ptr_min_max_vals()
6136 …verbose(env, "R%d bitwise operator %s on pointer prohibited\n", dst, bpf_alu_string[opcode >> 0x4]… in adjust_ptr_min_max_vals()
6140 …verbose(env, "R%d pointer arithmetic with %s operator prohibited\n", dst, bpf_alu_string[opcode >>… in adjust_ptr_min_max_vals()
6144 if (!check_reg_sane_offset(env, dst_reg, ptr_reg->type)) { in adjust_ptr_min_max_vals()
6150 if (sanitize_check_bounds(env, insn, dst_reg) < 0) { in adjust_ptr_min_max_vals()
6154 ret = sanitize_ptr_alu(env, insn, dst_reg, off_reg, dst_reg, &info, true); in adjust_ptr_min_max_vals()
6156 return sanitize_err(env, insn, ret, off_reg, dst_reg); in adjust_ptr_min_max_vals()
6712 static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, struct bpf_insn *insn, in adjust_scalar_min_max_vals() argument
6715 struct bpf_reg_state *regs = cur_regs(env); in adjust_scalar_min_max_vals()
6743 __mark_reg_unknown(env, dst_reg); in adjust_scalar_min_max_vals()
6753 __mark_reg_unknown(env, dst_reg); in adjust_scalar_min_max_vals()
6759 __mark_reg_unknown(env, dst_reg); in adjust_scalar_min_max_vals()
6764 ret = sanitize_val_alu(env, insn); in adjust_scalar_min_max_vals()
6766 return sanitize_err(env, insn, ret, NULL, NULL); in adjust_scalar_min_max_vals()
6820 mark_reg_unknown(env, regs, insn->dst_reg); in adjust_scalar_min_max_vals()
6834 mark_reg_unknown(env, regs, insn->dst_reg); in adjust_scalar_min_max_vals()
6848 mark_reg_unknown(env, regs, insn->dst_reg); in adjust_scalar_min_max_vals()
6858 mark_reg_unknown(env, regs, insn->dst_reg); in adjust_scalar_min_max_vals()
6874 static int adjust_reg_min_max_vals(struct bpf_verifier_env *env, struct bpf_insn *insn) in adjust_reg_min_max_vals() argument
6876 struct bpf_verifier_state *vstate = env->cur_state; in adjust_reg_min_max_vals()
6901 if (opcode == BPF_SUB && env->allow_ptr_leaks) { in adjust_reg_min_max_vals()
6902 mark_reg_unknown(env, regs, insn->dst_reg); in adjust_reg_min_max_vals()
6905 verbose(env, "R%d pointer %s pointer prohibited\n", insn->dst_reg, in adjust_reg_min_max_vals()
6913 err = mark_chain_precision(env, insn->dst_reg); in adjust_reg_min_max_vals()
6917 return adjust_ptr_min_max_vals(env, insn, src_reg, dst_reg); in adjust_reg_min_max_vals()
6921 err = mark_chain_precision(env, insn->src_reg); in adjust_reg_min_max_vals()
6925 return adjust_ptr_min_max_vals(env, insn, dst_reg, src_reg); in adjust_reg_min_max_vals()
6935 return adjust_ptr_min_max_vals(env, insn, ptr_reg, src_reg); in adjust_reg_min_max_vals()
6941 print_verifier_state(env, state); in adjust_reg_min_max_vals()
6942 verbose(env, "verifier internal error: unexpected ptr_reg\n"); in adjust_reg_min_max_vals()
6946 print_verifier_state(env, state); in adjust_reg_min_max_vals()
6947 verbose(env, "verifier internal error: no src_reg\n"); in adjust_reg_min_max_vals()
6950 return adjust_scalar_min_max_vals(env, insn, dst_reg, *src_reg); in adjust_reg_min_max_vals()
6954 static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn) in check_alu_op() argument
6956 struct bpf_reg_state *regs = cur_regs(env); in check_alu_op()
6963 verbose(env, "BPF_NEG uses reserved fields\n"); in check_alu_op()
6970 verbose(env, "BPF_END uses reserved fields\n"); in check_alu_op()
6976 err = check_reg_arg(env, insn->dst_reg, SRC_OP); in check_alu_op()
6981 if (is_pointer_value(env, insn->dst_reg)) { in check_alu_op()
6982 verbose(env, "R%d pointer arithmetic prohibited\n", insn->dst_reg); in check_alu_op()
6987 err = check_reg_arg(env, insn->dst_reg, DST_OP); in check_alu_op()
6994 verbose(env, "BPF_MOV uses reserved fields\n"); in check_alu_op()
6998 err = check_reg_arg(env, insn->src_reg, SRC_OP); in check_alu_op()
7004 verbose(env, "BPF_MOV uses reserved fields\n"); in check_alu_op()
7010 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); in check_alu_op()
7028 src_reg->id = ++env->id_gen; in check_alu_op()
7035 if (is_pointer_value(env, insn->src_reg)) { in check_alu_op()
7036 verbose(env, "R%d partial copy of pointer\n", insn->src_reg); in check_alu_op()
7046 dst_reg->subreg_def = env->insn_idx + 1; in check_alu_op()
7048 mark_reg_unknown(env, regs, insn->dst_reg); in check_alu_op()
7059 mark_reg_unknown(env, regs, insn->dst_reg); in check_alu_op()
7068 verbose(env, "invalid BPF_ALU opcode %x\n", opcode); in check_alu_op()
7073 verbose(env, "BPF_ALU uses reserved fields\n"); in check_alu_op()
7077 err = check_reg_arg(env, insn->src_reg, SRC_OP); in check_alu_op()
7083 verbose(env, "BPF_ALU uses reserved fields\n"); in check_alu_op()
7089 err = check_reg_arg(env, insn->dst_reg, SRC_OP); in check_alu_op()
7095 verbose(env, "div by zero\n"); in check_alu_op()
7102 verbose(env, "invalid shift %d\n", insn->imm); in check_alu_op()
7107 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); in check_alu_op()
7111 return adjust_reg_min_max_vals(env, insn); in check_alu_op()
7822 static int check_cond_jmp_op(struct bpf_verifier_env *env, struct bpf_insn *insn, int *insn_idx) in check_cond_jmp_op() argument
7824 struct bpf_verifier_state *this_branch = env->cur_state; in check_cond_jmp_op()
7835 verbose(env, "invalid BPF_JMP/JMP32 opcode %x\n", opcode); in check_cond_jmp_op()
7841 verbose(env, "BPF_JMP/JMP32 uses reserved fields\n"); in check_cond_jmp_op()
7846 err = check_reg_arg(env, insn->src_reg, SRC_OP); in check_cond_jmp_op()
7851 if (is_pointer_value(env, insn->src_reg)) { in check_cond_jmp_op()
7852 verbose(env, "R%d pointer comparison prohibited\n", insn->src_reg); in check_cond_jmp_op()
7858 verbose(env, "BPF_JMP/JMP32 uses reserved fields\n"); in check_cond_jmp_op()
7864 err = check_reg_arg(env, insn->dst_reg, SRC_OP); in check_cond_jmp_op()
7885 err = mark_chain_precision(env, insn->dst_reg); in check_cond_jmp_op()
7888 err = mark_chain_precision(env, insn->src_reg); in check_cond_jmp_op()
7900 … if (!env->bypass_spec_v1 && !sanitize_speculative_path(env, insn, *insn_idx + 1, *insn_idx)) { in check_cond_jmp_op()
7910 …if (!env->bypass_spec_v1 && !sanitize_speculative_path(env, insn, *insn_idx + insn->off + 1, *insn… in check_cond_jmp_op()
7916 other_branch = push_stack(env, *insn_idx + insn->off + 1, *insn_idx, false); in check_cond_jmp_op()
7971 is_pointer_value(env, insn->dst_reg)) { in check_cond_jmp_op()
7972 verbose(env, "R%d pointer comparison prohibited\n", insn->dst_reg); in check_cond_jmp_op()
7975 if (env->log.level & BPF_LOG_LEVEL) { in check_cond_jmp_op()
7976 print_verifier_state(env, this_branch->frame[this_branch->curframe]); in check_cond_jmp_op()
7982 static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn) in check_ld_imm() argument
7984 struct bpf_insn_aux_data *aux = cur_aux(env); in check_ld_imm()
7985 struct bpf_reg_state *regs = cur_regs(env); in check_ld_imm()
7991 verbose(env, "invalid BPF_LD_IMM insn\n"); in check_ld_imm()
7995 verbose(env, "BPF_LD_IMM64 uses reserved fields\n"); in check_ld_imm()
7999 err = check_reg_arg(env, insn->dst_reg, DST_OP); in check_ld_imm()
8017 mark_reg_known_zero(env, regs, insn->dst_reg); in check_ld_imm()
8030 verbose(env, "bpf verifier is misconfigured\n"); in check_ld_imm()
8036 map = env->used_maps[aux->map_index]; in check_ld_imm()
8043 dst_reg->id = ++env->id_gen; in check_ld_imm()
8048 verbose(env, "bpf verifier is misconfigured\n"); in check_ld_imm()
8082 static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn) in check_ld_abs() argument
8084 struct bpf_reg_state *regs = cur_regs(env); in check_ld_abs()
8089 if (!may_access_skb(resolve_prog_type(env->prog))) { in check_ld_abs()
8090 verbose(env, "BPF_LD_[ABS|IND] instructions not allowed for this program type\n"); in check_ld_abs()
8094 if (!env->ops->gen_ld_abs) { in check_ld_abs()
8095 verbose(env, "bpf verifier is misconfigured\n"); in check_ld_abs()
8101 verbose(env, "BPF_LD_[ABS|IND] uses reserved fields\n"); in check_ld_abs()
8106 err = check_reg_arg(env, ctx_reg, SRC_OP); in check_ld_abs()
8115 err = check_reference_leak(env); in check_ld_abs()
8117 verbose(env, "BPF_LD_[ABS|IND] cannot be mixed with socket references\n"); in check_ld_abs()
8121 if (env->cur_state->active_spin_lock) { in check_ld_abs()
8122 verbose(env, "BPF_LD_[ABS|IND] cannot be used inside bpf_spin_lock-ed region\n"); in check_ld_abs()
8127 verbose(env, "at the time of BPF_LD_ABS|IND R6 != pointer to skb\n"); in check_ld_abs()
8133 err = check_reg_arg(env, insn->src_reg, SRC_OP); in check_ld_abs()
8139 err = check_ptr_off_reg(env, ®s[ctx_reg], ctx_reg); in check_ld_abs()
8146 mark_reg_not_init(env, regs, caller_saved[i]); in check_ld_abs()
8147 check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); in check_ld_abs()
8154 mark_reg_unknown(env, regs, BPF_REG_0); in check_ld_abs()
8156 regs[BPF_REG_0].subreg_def = env->insn_idx + 1; in check_ld_abs()
8160 static int check_return_code(struct bpf_verifier_env *env) in check_return_code() argument
8163 const struct bpf_prog *prog = env->prog; in check_return_code()
8166 enum bpf_prog_type prog_type = resolve_prog_type(env->prog); in check_return_code()
8168 const bool is_subprog = env->cur_state->frame[0]->subprogno; in check_return_code()
8182 err = check_reg_arg(env, BPF_REG_0, SRC_OP); in check_return_code()
8187 if (is_pointer_value(env, BPF_REG_0)) { in check_return_code()
8188 verbose(env, "R0 leaks addr as return value\n"); in check_return_code()
8192 reg = cur_regs(env) + BPF_REG_0; in check_return_code()
8195 verbose(env, "At subprogram exit the register R0 is not a scalar value (%s)\n", in check_return_code()
8196 reg_type_str(env, reg->type)); in check_return_code()
8204 if (env->prog->expected_attach_type == BPF_CGROUP_UDP4_RECVMSG || in check_return_code()
8205 env->prog->expected_attach_type == BPF_CGROUP_UDP6_RECVMSG || in check_return_code()
8206 env->prog->expected_attach_type == BPF_CGROUP_INET4_GETPEERNAME || in check_return_code()
8207 env->prog->expected_attach_type == BPF_CGROUP_INET6_GETPEERNAME || in check_return_code()
8208 env->prog->expected_attach_type == BPF_CGROUP_INET4_GETSOCKNAME || in check_return_code()
8209 env->prog->expected_attach_type == BPF_CGROUP_INET6_GETSOCKNAME) { in check_return_code()
8214 if (env->prog->expected_attach_type == BPF_CGROUP_INET_EGRESS) { in check_return_code()
8226 if (!env->prog->aux->attach_btf_id) { in check_return_code()
8232 switch (env->prog->expected_attach_type) { in check_return_code()
8258 …verbose(env, "At program exit the register R0 is not a known value (%s)\n", reg_type_str(env, reg-… in check_return_code()
8265 verbose(env, "At program exit the register R0 "); in check_return_code()
8268 verbose(env, "has value %s", tn_buf); in check_return_code()
8270 verbose(env, "has unknown scalar value"); in check_return_code()
8273 verbose(env, " should have been in %s\n", tn_buf); in check_return_code()
8278 env->prog->enforce_expected_attach_type = 1; in check_return_code()
8323 static u32 state_htab_size(struct bpf_verifier_env *env) in state_htab_size() argument
8325 return env->prog->len; in state_htab_size()
8328 static struct bpf_verifier_state_list **explored_state(struct bpf_verifier_env *env, int idx) in explored_state() argument
8330 struct bpf_verifier_state *cur = env->cur_state; in explored_state()
8333 return &env->explored_states[(idx ^ state->callsite) % state_htab_size(env)]; in explored_state()
8336 static void init_explored_state(struct bpf_verifier_env *env, int idx) in init_explored_state() argument
8338 env->insn_aux_data[idx].prune_point = true; in init_explored_state()
8346 static int push_insn(int t, int w, int e, struct bpf_verifier_env *env, bool loop_ok) in push_insn() argument
8348 int *insn_stack = env->cfg.insn_stack; in push_insn()
8349 int *insn_state = env->cfg.insn_state; in push_insn()
8359 if (w < 0 || w >= env->prog->len) { in push_insn()
8360 verbose_linfo(env, t, "%d: ", t); in push_insn()
8361 verbose(env, "jump out of range from insn %d to %d\n", t, w); in push_insn()
8367 init_explored_state(env, w); in push_insn()
8374 if (env->cfg.cur_stack >= env->prog->len) { in push_insn()
8377 insn_stack[env->cfg.cur_stack++] = w; in push_insn()
8380 if (loop_ok && env->bpf_capable) { in push_insn()
8383 verbose_linfo(env, t, "%d: ", t); in push_insn()
8384 verbose_linfo(env, w, "%d: ", w); in push_insn()
8385 verbose(env, "back-edge from insn %d to %d\n", t, w); in push_insn()
8391 verbose(env, "insn state internal bug\n"); in push_insn()
8400 static int check_cfg(struct bpf_verifier_env *env) in check_cfg() argument
8402 struct bpf_insn *insns = env->prog->insnsi; in check_cfg()
8403 int insn_cnt = env->prog->len; in check_cfg()
8408 insn_state = env->cfg.insn_state = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL); in check_cfg()
8413 insn_stack = env->cfg.insn_stack = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL); in check_cfg()
8421 env->cfg.cur_stack = 1; in check_cfg()
8424 if (env->cfg.cur_stack == 0) { in check_cfg()
8427 t = insn_stack[env->cfg.cur_stack - 1]; in check_cfg()
8434 ret = push_insn(t, t + 1, FALLTHROUGH, env, false); in check_cfg()
8441 init_explored_state(env, t + 1); in check_cfg()
8444 init_explored_state(env, t); in check_cfg()
8445 ret = push_insn(t, t + insns[t].imm + 1, BRANCH, env, false); in check_cfg()
8458 ret = push_insn(t, t + insns[t].off + 1, FALLTHROUGH, env, true); in check_cfg()
8468 init_explored_state(env, t + insns[t].off + 1); in check_cfg()
8473 init_explored_state(env, t + 1); in check_cfg()
8477 init_explored_state(env, t); in check_cfg()
8478 ret = push_insn(t, t + 1, FALLTHROUGH, env, true); in check_cfg()
8485 ret = push_insn(t, t + insns[t].off + 1, BRANCH, env, true); in check_cfg()
8496 ret = push_insn(t, t + 1, FALLTHROUGH, env, false); in check_cfg()
8506 if (env->cfg.cur_stack-- <= 0) { in check_cfg()
8507 verbose(env, "pop stack internal bug\n"); in check_cfg()
8517 verbose(env, "unreachable insn %d\n", i); in check_cfg()
8527 env->cfg.insn_state = env->cfg.insn_stack = NULL; in check_cfg()
8531 static int check_abnormal_return(struct bpf_verifier_env *env) in check_abnormal_return() argument
8535 for (i = 1; i < env->subprog_cnt; i++) { in check_abnormal_return()
8536 if (env->subprog_info[i].has_ld_abs) { in check_abnormal_return()
8537 verbose(env, "LD_ABS is not allowed in subprogs without BTF\n"); in check_abnormal_return()
8540 if (env->subprog_info[i].has_tail_call) { in check_abnormal_return()
8541 verbose(env, "tail_call is not allowed in subprogs without BTF\n"); in check_abnormal_return()
8552 static int check_btf_func(struct bpf_verifier_env *env, const union bpf_attr *attr, union bpf_attr … in check_btf_func() argument
8568 if (check_abnormal_return(env)) { in check_btf_func()
8574 if (nfuncs != env->subprog_cnt) { in check_btf_func()
8575 verbose(env, "number of funcs in func_info doesn't match number of subprogs\n"); in check_btf_func()
8581 verbose(env, "invalid func info rec size %u\n", urec_size); in check_btf_func()
8585 prog = env->prog; in check_btf_func()
8604 verbose(env, "nonzero tailing record in func info"); in check_btf_func()
8624 … verbose(env, "nonzero insn_off %u for the first func info record", krecord[i].insn_off); in check_btf_func()
8628 …verbose(env, "same or smaller insn offset (%u) than previous func info record (%u)", krecord[i].in… in check_btf_func()
8633 if (env->subprog_info[i].start != krecord[i].insn_off) { in check_btf_func()
8634 verbose(env, "func_info BTF section doesn't match subprog layout in BPF program\n"); in check_btf_func()
8641 verbose(env, "invalid type id %d in func info", krecord[i].type_id); in check_btf_func()
8653 if (i && !scalar_return && env->subprog_info[i].has_ld_abs) { in check_btf_func()
8654 verbose(env, "LD_ABS is only allowed in functions that return 'int'.\n"); in check_btf_func()
8657 if (i && !scalar_return && env->subprog_info[i].has_tail_call) { in check_btf_func()
8658 verbose(env, "tail_call is only allowed in functions that return 'int'.\n"); in check_btf_func()
8677 static void adjust_btf_func(struct bpf_verifier_env *env) in adjust_btf_func() argument
8679 struct bpf_prog_aux *aux = env->prog->aux; in adjust_btf_func()
8686 for (i = 0; i < env->subprog_cnt; i++) { in adjust_btf_func()
8687 aux->func_info[i].insn_off = env->subprog_info[i].start; in adjust_btf_func()
8695 static int check_btf_line(struct bpf_verifier_env *env, const union bpf_attr *attr, union bpf_attr … in check_btf_line() argument
8726 prog = env->prog; in check_btf_line()
8730 sub = env->subprog_info; in check_btf_line()
8738 verbose(env, "nonzero tailing record in line_info"); in check_btf_line()
8763 …verbose(env, "Invalid line_info[%u].insn_off:%u (prev_offset:%u prog->len:%u)\n", i, linfo[i].insn… in check_btf_line()
8770 verbose(env, "Invalid insn code at line_info[%u].insn_off\n", i); in check_btf_line()
8776 verbose(env, "Invalid line_info[%u].line_off or .file_name_off\n", i); in check_btf_line()
8781 if (s != env->subprog_cnt) { in check_btf_line()
8786 verbose(env, "missing bpf_line_info for func#%u\n", s); in check_btf_line()
8796 if (s != env->subprog_cnt) { in check_btf_line()
8797 …verbose(env, "missing bpf_line_info for %u funcs starting from func#%u\n", env->subprog_cnt - s, s… in check_btf_line()
8812 static int check_btf_info(struct bpf_verifier_env *env, const union bpf_attr *attr, union bpf_attr … in check_btf_info() argument
8818 if (check_abnormal_return(env)) { in check_btf_info()
8828 env->prog->aux->btf = btf; in check_btf_info()
8830 err = check_btf_func(env, attr, uattr); in check_btf_info()
8835 err = check_btf_line(env, attr, uattr); in check_btf_info()
8882 static void clean_func_state(struct bpf_verifier_env *env, struct bpf_func_state *st) in clean_func_state() argument
8895 verifier_mark_reg_not_init(env, &st->regs[i]); in clean_func_state()
8904 verifier_mark_reg_not_init(env, &st->stack[i].spilled_ptr); in clean_func_state()
8912 static void clean_verifier_state(struct bpf_verifier_env *env, struct bpf_verifier_state *st) in clean_verifier_state() argument
8922 clean_func_state(env, st->frame[i]); in clean_verifier_state()
8957 static void clean_live_states(struct bpf_verifier_env *env, int insn, struct bpf_verifier_state *cu… in clean_live_states() argument
8962 sl = *explored_state(env, insn); in clean_live_states()
8975 clean_verifier_state(env, &sl->state); in clean_live_states()
8982 static bool regsafe(struct bpf_verifier_env *env, struct bpf_reg_state *rold, struct bpf_reg_state … in regsafe() argument
9014 if (env->explore_alu_limits) { in regsafe()
9109 static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old, struct bpf_func_sta… in stacksafe() argument
9160 if (!regsafe(env, &old->stack[spi].spilled_ptr, &cur->stack[spi].spilled_ptr, idmap)) { in stacksafe()
9211 static bool func_states_equal(struct bpf_verifier_env *env, struct bpf_func_state *old, struct bpf_… in func_states_equal() argument
9215 memset(env->idmap_scratch, 0, sizeof(env->idmap_scratch)); in func_states_equal()
9217 if (!regsafe(env, &old->regs[i], &cur->regs[i], env->idmap_scratch)) { in func_states_equal()
9222 if (!stacksafe(env, old, cur, env->idmap_scratch)) { in func_states_equal()
9233 static bool states_equal(struct bpf_verifier_env *env, struct bpf_verifier_state *old, struct bpf_v… in states_equal() argument
9259 if (!func_states_equal(env, old->frame[i], cur->frame[i])) { in states_equal()
9269 static int propagate_liveness_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, in propagate_liveness_reg() argument
9288 err = mark_reg_read(env, reg, parent_reg, flag); in propagate_liveness_reg()
9303 static int propagate_liveness(struct bpf_verifier_env *env, const struct bpf_verifier_state *vstate, in propagate_liveness() argument
9323 err = propagate_liveness_reg(env, &state_reg[i], &parent_reg[i]); in propagate_liveness()
9328 mark_insn_zext(env, &parent_reg[i]); in propagate_liveness()
9336 err = propagate_liveness_reg(env, state_reg, parent_reg); in propagate_liveness()
9348 static int propagate_precision(struct bpf_verifier_env *env, const struct bpf_verifier_state *old) in propagate_precision() argument
9360 if (env->log.level & BPF_LOG_LEVEL2) { in propagate_precision()
9361 verbose(env, "propagating r%d\n", i); in propagate_precision()
9363 err = mark_chain_precision(env, i); in propagate_precision()
9377 if (env->log.level & BPF_LOG_LEVEL2) { in propagate_precision()
9378 verbose(env, "propagating fp%d\n", (-i - 1) * BPF_REG_SIZE); in propagate_precision()
9380 err = mark_chain_precision_stack(env, i); in propagate_precision()
9407 static int is_state_visited(struct bpf_verifier_env *env, int insn_idx) in is_state_visited() argument
9411 struct bpf_verifier_state *cur = env->cur_state, *new; in is_state_visited()
9413 bool add_new_state = env->test_state_freq ? true : false; in is_state_visited()
9415 cur->last_insn_idx = env->prev_insn_idx; in is_state_visited()
9416 if (!env->insn_aux_data[insn_idx].prune_point) { in is_state_visited()
9431 …if (env->jmps_processed - env->prev_jmps_processed >= 2 && env->insn_processed - env->prev_insn_pr… in is_state_visited()
9435 pprev = explored_state(env, insn_idx); in is_state_visited()
9438 clean_live_states(env, insn_idx, cur); in is_state_visited()
9446 if (states_maybe_looping(&sl->state, cur) && states_equal(env, &sl->state, cur)) { in is_state_visited()
9447 verbose_linfo(env, insn_idx, "; "); in is_state_visited()
9448 verbose(env, "infinite loop detected at insn %d\n", insn_idx); in is_state_visited()
9463 if (env->jmps_processed - env->prev_jmps_processed < 20 && in is_state_visited()
9464 env->insn_processed - env->prev_insn_processed < 100) { in is_state_visited()
9469 if (states_equal(env, &sl->state, cur)) { in is_state_visited()
9481 err = propagate_liveness(env, &sl->state, cur); in is_state_visited()
9488 err = err ?: push_jmp_history(env, cur); in is_state_visited()
9489 err = err ?: propagate_precision(env, &sl->state); in is_state_visited()
9521 env->peak_states--; in is_state_visited()
9527 sl->next = env->free_list; in is_state_visited()
9528 env->free_list = sl; in is_state_visited()
9538 if (env->max_states_per_insn < states_cnt) { in is_state_visited()
9539 env->max_states_per_insn = states_cnt; in is_state_visited()
9542 if (!env->bpf_capable && states_cnt > BPF_COMPLEXITY_LIMIT_STATES) { in is_state_visited()
9543 return push_jmp_history(env, cur); in is_state_visited()
9547 return push_jmp_history(env, cur); in is_state_visited()
9563 env->total_states++; in is_state_visited()
9564 env->peak_states++; in is_state_visited()
9565 env->prev_jmps_processed = env->jmps_processed; in is_state_visited()
9566 env->prev_insn_processed = env->insn_processed; in is_state_visited()
9582 new_sl->next = *explored_state(env, insn_idx); in is_state_visited()
9583 *explored_state(env, insn_idx) = new_sl; in is_state_visited()
9652 static int do_check(struct bpf_verifier_env *env) in do_check() argument
9654 bool pop_log = !(env->log.level & BPF_LOG_LEVEL2); in do_check()
9655 struct bpf_verifier_state *state = env->cur_state; in do_check()
9656 struct bpf_insn *insns = env->prog->insnsi; in do_check()
9658 int insn_cnt = env->prog->len; in do_check()
9667 env->prev_insn_idx = prev_insn_idx; in do_check()
9668 if (env->insn_idx >= insn_cnt) { in do_check()
9669 verbose(env, "invalid insn idx %d insn_cnt %d\n", env->insn_idx, insn_cnt); in do_check()
9673 insn = &insns[env->insn_idx]; in do_check()
9676 if (++env->insn_processed > BPF_COMPLEXITY_LIMIT_INSNS) { in do_check()
9677 verbose(env, "BPF program is too large. Processed %d insn\n", env->insn_processed); in do_check()
9681 err = is_state_visited(env, env->insn_idx); in do_check()
9687 if (env->log.level & BPF_LOG_LEVEL) { in do_check()
9689 verbose(env, "\nfrom %d to %d%s: safe\n", env->prev_insn_idx, env->insn_idx, in do_check()
9690 env->cur_state->speculative ? " (speculative execution)" : ""); in do_check()
9692 verbose(env, "%d: safe\n", env->insn_idx); in do_check()
9706 … if ((env->log.level & BPF_LOG_LEVEL2) || ((env->log.level & BPF_LOG_LEVEL) && do_print_state)) { in do_check()
9707 if (env->log.level & BPF_LOG_LEVEL2) { in do_check()
9708 verbose(env, "%d:", env->insn_idx); in do_check()
9710 verbose(env, "\nfrom %d to %d%s:", env->prev_insn_idx, env->insn_idx, in do_check()
9711 env->cur_state->speculative ? " (speculative execution)" : ""); in do_check()
9713 print_verifier_state(env, state->frame[state->curframe]); in do_check()
9717 if (env->log.level & BPF_LOG_LEVEL) { in do_check()
9720 .private_data = env, in do_check()
9723 verbose_linfo(env, env->insn_idx, "; "); in do_check()
9724 verbose(env, "%d: ", env->insn_idx); in do_check()
9725 print_bpf_insn(&cbs, insn, env->allow_ptr_leaks); in do_check()
9728 if (bpf_prog_is_dev_bound(env->prog->aux)) { in do_check()
9729 err = bpf_prog_offload_verify_insn(env, env->insn_idx, env->prev_insn_idx); in do_check()
9735 regs = cur_regs(env); in do_check()
9736 sanitize_mark_insn_seen(env); in do_check()
9737 prev_insn_idx = env->insn_idx; in do_check()
9740 err = check_alu_op(env, insn); in do_check()
9750 err = check_reg_arg(env, insn->src_reg, SRC_OP); in do_check()
9755 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); in do_check()
9765 …err = check_mem_access(env, env->insn_idx, insn->src_reg, insn->off, BPF_SIZE(insn->code), BPF_REA… in do_check()
9771 prev_src_type = &env->insn_aux_data[env->insn_idx].ptr_type; in do_check()
9787 verbose(env, "same insn cannot be used with different pointers\n"); in do_check()
9793 err = check_xadd(env, env->insn_idx, insn); in do_check()
9797 env->insn_idx++; in do_check()
9802 err = check_reg_arg(env, insn->src_reg, SRC_OP); in do_check()
9807 err = check_reg_arg(env, insn->dst_reg, SRC_OP); in do_check()
9815 …err = check_mem_access(env, env->insn_idx, insn->dst_reg, insn->off, BPF_SIZE(insn->code), BPF_WRI… in do_check()
9821 prev_dst_type = &env->insn_aux_data[env->insn_idx].ptr_type; in do_check()
9826 verbose(env, "same insn cannot be used with different pointers\n"); in do_check()
9831 verbose(env, "BPF_ST uses reserved fields\n"); in do_check()
9835 err = check_reg_arg(env, insn->dst_reg, SRC_OP); in do_check()
9839 if (is_ctx_reg(env, insn->dst_reg)) { in do_check()
9840 verbose(env, "BPF_ST stores into R%d %s is not allowed\n", insn->dst_reg, in do_check()
9841 reg_type_str(env, reg_state(env, insn->dst_reg)->type)); in do_check()
9846 …err = check_mem_access(env, env->insn_idx, insn->dst_reg, insn->off, BPF_SIZE(insn->code), BPF_WRI… in do_check()
9853 env->jmps_processed++; in do_check()
9858 verbose(env, "BPF_CALL uses reserved fields\n"); in do_check()
9862 if (env->cur_state->active_spin_lock && in do_check()
9864 verbose(env, "function calls are not allowed while holding a lock\n"); in do_check()
9868 err = check_func_call(env, insn, &env->insn_idx); in do_check()
9870 err = check_helper_call(env, insn->imm, env->insn_idx); in do_check()
9878 verbose(env, "BPF_JA uses reserved fields\n"); in do_check()
9881 env->insn_idx += insn->off + 1; in do_check()
9886 verbose(env, "BPF_EXIT uses reserved fields\n"); in do_check()
9889 if (env->cur_state->active_spin_lock) { in do_check()
9890 verbose(env, "bpf_spin_unlock is missing\n"); in do_check()
9895 err = prepare_func_exit(env, &env->insn_idx); in do_check()
9903 err = check_reference_leak(env); in do_check()
9908 err = check_return_code(env); in do_check()
9913 update_branch_counts(env, env->cur_state); in do_check()
9914 err = pop_stack(env, &prev_insn_idx, &env->insn_idx, pop_log); in do_check()
9925 err = check_cond_jmp_op(env, insn, &env->insn_idx); in do_check()
9933 err = check_ld_abs(env, insn); in do_check()
9938 err = check_ld_imm(env, insn); in do_check()
9942 env->insn_idx++; in do_check()
9943 sanitize_mark_insn_seen(env); in do_check()
9945 verbose(env, "invalid BPF_LD mode\n"); in do_check()
9949 verbose(env, "unknown insn class %d\n", class); in do_check()
9952 env->insn_idx++; in do_check()
9959 static int check_pseudo_btf_id(struct bpf_verifier_env *env, struct bpf_insn *insn, struct bpf_insn… in check_pseudo_btf_id() argument
9972 …verbose(env, "kernel is missing BTF, make sure CONFIG_DEBUG_INFO_BTF=y is specified in Kconfig.\n"… in check_pseudo_btf_id()
9977 verbose(env, "reserved field (insn[1].imm) is used in pseudo_btf_id ldimm64 insn.\n"); in check_pseudo_btf_id()
9983 verbose(env, "ldimm64 insn specifies invalid btf_id %d.\n", id); in check_pseudo_btf_id()
9988 verbose(env, "pseudo btf_id %d in ldimm64 isn't KIND_VAR.\n", id); in check_pseudo_btf_id()
9995 verbose(env, "ldimm64 failed to find the address for kernel symbol '%s'.\n", sym_name); in check_pseudo_btf_id()
10028 … verbose(env, "ldimm64 unable to resolve the size of type '%s': %ld\n", tname, PTR_ERR(ret)); in check_pseudo_btf_id()
10071 static int check_map_prog_compatibility(struct bpf_verifier_env *env, struct bpf_map *map, struct b… in check_map_prog_compatibility() argument
10094 verbose(env, "perf_event programs can only use preallocated hash map\n"); in check_map_prog_compatibility()
10098 verbose(env, "trace type programs can only use preallocated hash map\n"); in check_map_prog_compatibility()
10103 env, in check_map_prog_compatibility()
10108 verbose(env, "tracing progs cannot use bpf_spin_lock yet\n"); in check_map_prog_compatibility()
10113 verbose(env, "offload device mismatch between prog and map\n"); in check_map_prog_compatibility()
10118 verbose(env, "bpf_struct_ops map cannot be used in prog\n"); in check_map_prog_compatibility()
10128 verbose(env, "Sleepable programs can only use preallocated hash maps\n"); in check_map_prog_compatibility()
10133 verbose(env, "Sleepable programs can only use array and hash maps\n"); in check_map_prog_compatibility()
10153 static int resolve_pseudo_ldimm64(struct bpf_verifier_env *env) in resolve_pseudo_ldimm64() argument
10155 struct bpf_insn *insn = env->prog->insnsi; in resolve_pseudo_ldimm64()
10156 int insn_cnt = env->prog->len; in resolve_pseudo_ldimm64()
10159 err = bpf_prog_calc_tag(env->prog); in resolve_pseudo_ldimm64()
10166 verbose(env, "BPF_LDX uses reserved fields\n"); in resolve_pseudo_ldimm64()
10172 verbose(env, "BPF_STX uses reserved fields\n"); in resolve_pseudo_ldimm64()
10184 verbose(env, "invalid bpf_ld_imm64 insn\n"); in resolve_pseudo_ldimm64()
10194 aux = &env->insn_aux_data[i]; in resolve_pseudo_ldimm64()
10195 err = check_pseudo_btf_id(env, insn, aux); in resolve_pseudo_ldimm64()
10207 verbose(env, "unrecognized bpf_ld_imm64 insn\n"); in resolve_pseudo_ldimm64()
10214 verbose(env, "fd %d is not pointing to valid bpf_map\n", insn[0].imm); in resolve_pseudo_ldimm64()
10218 err = check_map_prog_compatibility(env, map, env->prog); in resolve_pseudo_ldimm64()
10224 aux = &env->insn_aux_data[i]; in resolve_pseudo_ldimm64()
10231 verbose(env, "direct value offset of %u is not allowed\n", off); in resolve_pseudo_ldimm64()
10237 verbose(env, "no direct value access support for this map type\n"); in resolve_pseudo_ldimm64()
10244 … verbose(env, "invalid access to map value pointer, value_size=%u off=%u\n", map->value_size, off); in resolve_pseudo_ldimm64()
10257 for (j = 0; j < env->used_map_cnt; j++) { in resolve_pseudo_ldimm64()
10258 if (env->used_maps[j] == map) { in resolve_pseudo_ldimm64()
10265 if (env->used_map_cnt >= MAX_USED_MAPS) { in resolve_pseudo_ldimm64()
10277 aux->map_index = env->used_map_cnt; in resolve_pseudo_ldimm64()
10278 env->used_maps[env->used_map_cnt++] = map; in resolve_pseudo_ldimm64()
10280 if (bpf_map_is_cgroup_storage(map) && bpf_cgroup_storage_assign(env->prog->aux, map)) { in resolve_pseudo_ldimm64()
10281 verbose(env, "only one cgroup storage of each type is allowed\n"); in resolve_pseudo_ldimm64()
10295 verbose(env, "unknown opcode %02x\n", insn->code); in resolve_pseudo_ldimm64()
10308 static void release_maps(struct bpf_verifier_env *env) in release_maps() argument
10310 __bpf_free_used_maps(env->prog->aux, env->used_maps, env->used_map_cnt); in release_maps()
10314 static void convert_pseudo_ld_imm64(struct bpf_verifier_env *env) in convert_pseudo_ld_imm64() argument
10316 struct bpf_insn *insn = env->prog->insnsi; in convert_pseudo_ld_imm64()
10317 int insn_cnt = env->prog->len; in convert_pseudo_ld_imm64()
10331 static void adjust_insn_aux_data(struct bpf_verifier_env *env, struct bpf_insn_aux_data *new_data, in adjust_insn_aux_data() argument
10334 struct bpf_insn_aux_data *old_data = env->insn_aux_data; in adjust_insn_aux_data()
10344 old_data[off].zext_dst = insn_has_def32(env, insn + off + cnt - 1); in adjust_insn_aux_data()
10356 new_data[i].zext_dst = insn_has_def32(env, insn + i); in adjust_insn_aux_data()
10358 env->insn_aux_data = new_data; in adjust_insn_aux_data()
10362 static void adjust_subprog_starts(struct bpf_verifier_env *env, u32 off, u32 len) in adjust_subprog_starts() argument
10370 for (i = 0; i <= env->subprog_cnt; i++) { in adjust_subprog_starts()
10371 if (env->subprog_info[i].start <= off) { in adjust_subprog_starts()
10374 env->subprog_info[i].start += len - 1; in adjust_subprog_starts()
10393 static struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, const struct bpf… in bpf_patch_insn_data() argument
10400 new_data = vzalloc(array_size(env->prog->len + len - 1, sizeof(struct bpf_insn_aux_data))); in bpf_patch_insn_data()
10406 new_prog = bpf_patch_insn_single(env->prog, off, patch, len); in bpf_patch_insn_data()
10409 … verbose(env, "insn %d cannot be patched due to 16-bit range\n", env->insn_aux_data[off].orig_idx); in bpf_patch_insn_data()
10414 adjust_insn_aux_data(env, new_data, new_prog, off, len); in bpf_patch_insn_data()
10415 adjust_subprog_starts(env, off, len); in bpf_patch_insn_data()
10420 static int adjust_subprog_starts_after_remove(struct bpf_verifier_env *env, u32 off, u32 cnt) in adjust_subprog_starts_after_remove() argument
10425 for (i = 0; i < env->subprog_cnt; i++) { in adjust_subprog_starts_after_remove()
10426 if (env->subprog_info[i].start >= off) { in adjust_subprog_starts_after_remove()
10431 for (j = i; j < env->subprog_cnt; j++) { in adjust_subprog_starts_after_remove()
10432 if (env->subprog_info[j].start >= off + cnt) { in adjust_subprog_starts_after_remove()
10439 if (env->subprog_info[j].start != off + cnt) { in adjust_subprog_starts_after_remove()
10444 struct bpf_prog_aux *aux = env->prog->aux; in adjust_subprog_starts_after_remove()
10448 move = env->subprog_cnt + 1 - j; in adjust_subprog_starts_after_remove()
10450 memmove(env->subprog_info + i, env->subprog_info + j, sizeof(*env->subprog_info) * move); in adjust_subprog_starts_after_remove()
10451 env->subprog_cnt -= j - i; in adjust_subprog_starts_after_remove()
10465 if (env->subprog_info[i].start == off) { in adjust_subprog_starts_after_remove()
10471 for (; i <= env->subprog_cnt; i++) { in adjust_subprog_starts_after_remove()
10472 env->subprog_info[i].start -= cnt; in adjust_subprog_starts_after_remove()
10478 static int bpf_adj_linfo_after_remove(struct bpf_verifier_env *env, u32 off, u32 cnt) in bpf_adj_linfo_after_remove() argument
10480 struct bpf_prog *prog = env->prog; in bpf_adj_linfo_after_remove()
10531 for (i = 0; i <= env->subprog_cnt; i++) { in bpf_adj_linfo_after_remove()
10532 if (env->subprog_info[i].linfo_idx > l_off) { in bpf_adj_linfo_after_remove()
10536 if (env->subprog_info[i].linfo_idx >= l_off + l_cnt) { in bpf_adj_linfo_after_remove()
10537 env->subprog_info[i].linfo_idx -= l_cnt; in bpf_adj_linfo_after_remove()
10539 env->subprog_info[i].linfo_idx = l_off; in bpf_adj_linfo_after_remove()
10547 static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt) in verifier_remove_insns() argument
10549 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; in verifier_remove_insns()
10550 unsigned int orig_prog_len = env->prog->len; in verifier_remove_insns()
10553 if (bpf_prog_is_dev_bound(env->prog->aux)) { in verifier_remove_insns()
10554 bpf_prog_offload_remove_insns(env, off, cnt); in verifier_remove_insns()
10557 err = bpf_remove_insns(env->prog, off, cnt); in verifier_remove_insns()
10562 err = adjust_subprog_starts_after_remove(env, off, cnt); in verifier_remove_insns()
10567 err = bpf_adj_linfo_after_remove(env, off, cnt); in verifier_remove_insns()
10588 static void sanitize_dead_code(struct bpf_verifier_env *env) in sanitize_dead_code() argument
10590 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; in sanitize_dead_code()
10592 struct bpf_insn *insn = env->prog->insnsi; in sanitize_dead_code()
10593 const int insn_cnt = env->prog->len; in sanitize_dead_code()
10621 static void opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env) in opt_hard_wire_dead_code_branches() argument
10623 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; in opt_hard_wire_dead_code_branches()
10625 struct bpf_insn *insn = env->prog->insnsi; in opt_hard_wire_dead_code_branches()
10626 const int insn_cnt = env->prog->len; in opt_hard_wire_dead_code_branches()
10642 if (bpf_prog_is_dev_bound(env->prog->aux)) { in opt_hard_wire_dead_code_branches()
10643 bpf_prog_offload_replace_insn(env, i, &ja); in opt_hard_wire_dead_code_branches()
10650 static int opt_remove_dead_code(struct bpf_verifier_env *env) in opt_remove_dead_code() argument
10652 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; in opt_remove_dead_code()
10653 int insn_cnt = env->prog->len; in opt_remove_dead_code()
10667 err = verifier_remove_insns(env, i, j); in opt_remove_dead_code()
10671 insn_cnt = env->prog->len; in opt_remove_dead_code()
10677 static int opt_remove_nops(struct bpf_verifier_env *env) in opt_remove_nops() argument
10680 struct bpf_insn *insn = env->prog->insnsi; in opt_remove_nops()
10681 int insn_cnt = env->prog->len; in opt_remove_nops()
10689 err = verifier_remove_insns(env, i, 1); in opt_remove_nops()
10700 static int opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env, const union bpf_attr *attr) in opt_subreg_zext_lo32_rnd_hi32() argument
10703 struct bpf_insn_aux_data *aux = env->insn_aux_data; in opt_subreg_zext_lo32_rnd_hi32()
10704 int i, patch_len, delta = 0, len = env->prog->len; in opt_subreg_zext_lo32_rnd_hi32()
10705 struct bpf_insn *insns = env->prog->insnsi; in opt_subreg_zext_lo32_rnd_hi32()
10737 if (is_reg64(env, &insn, insn.dst_reg, NULL, DST_OP)) { in opt_subreg_zext_lo32_rnd_hi32()
10768 new_prog = bpf_patch_insn_data(env, adj_idx, patch, patch_len); in opt_subreg_zext_lo32_rnd_hi32()
10772 env->prog = new_prog; in opt_subreg_zext_lo32_rnd_hi32()
10774 aux = env->insn_aux_data; in opt_subreg_zext_lo32_rnd_hi32()
10786 static int convert_ctx_accesses(struct bpf_verifier_env *env) in convert_ctx_accesses() argument
10788 const struct bpf_verifier_ops *ops = env->ops; in convert_ctx_accesses()
10790 const int insn_cnt = env->prog->len; in convert_ctx_accesses()
10797 if (ops->gen_prologue || env->seen_direct_write) { in convert_ctx_accesses()
10799 verbose(env, "bpf verifier is misconfigured\n"); in convert_ctx_accesses()
10802 cnt = ops->gen_prologue(insn_buf, env->seen_direct_write, env->prog); in convert_ctx_accesses()
10804 verbose(env, "bpf verifier is misconfigured\n"); in convert_ctx_accesses()
10807 new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt); in convert_ctx_accesses()
10812 env->prog = new_prog; in convert_ctx_accesses()
10817 if (bpf_prog_is_dev_bound(env->prog->aux)) { in convert_ctx_accesses()
10821 insn = env->prog->insnsi + delta; in convert_ctx_accesses()
10841 if (type == BPF_WRITE && env->insn_aux_data[i + delta].sanitize_stack_spill) { in convert_ctx_accesses()
10848 new_prog = bpf_patch_insn_data(env, i + delta, patch, cnt); in convert_ctx_accesses()
10854 env->prog = new_prog; in convert_ctx_accesses()
10863 switch (env->insn_aux_data[i + delta].ptr_type) { in convert_ctx_accesses()
10883 env->prog->aux->num_exentries++; in convert_ctx_accesses()
10884 } else if (resolve_prog_type(env->prog) != BPF_PROG_TYPE_STRUCT_OPS) { in convert_ctx_accesses()
10885 verbose(env, "Writes through BTF pointers are not allowed\n"); in convert_ctx_accesses()
10893 ctx_field_size = env->insn_aux_data[i + delta].ctx_field_size; in convert_ctx_accesses()
10908 verbose(env, "bpf verifier narrow ctx access misconfigured\n"); in convert_ctx_accesses()
10924 cnt = convert_ctx_access(type, insn, insn_buf, env->prog, &target_size); in convert_ctx_accesses()
10926 verbose(env, "bpf verifier is misconfigured\n"); in convert_ctx_accesses()
10933 verbose(env, "bpf verifier narrow ctx load misconfigured\n"); in convert_ctx_accesses()
10949 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in convert_ctx_accesses()
10957 env->prog = new_prog; in convert_ctx_accesses()
10964 static int jit_subprogs(struct bpf_verifier_env *env) in jit_subprogs() argument
10966 struct bpf_prog *prog = env->prog, **func, *tmp; in jit_subprogs()
10973 if (env->subprog_cnt <= 1) { in jit_subprogs()
10985 subprog = find_subprog(env, i + insn->imm + 1); in jit_subprogs()
10997 env->insn_aux_data[i].call_imm = insn->imm; in jit_subprogs()
11008 func = kcalloc(env->subprog_cnt, sizeof(prog), GFP_KERNEL); in jit_subprogs()
11013 for (i = 0; i < env->subprog_cnt; i++) { in jit_subprogs()
11015 subprog_end = env->subprog_info[i + 1].start; in jit_subprogs()
11054 func[i]->aux->stack_depth = env->subprog_info[i].stack_depth; in jit_subprogs()
11059 func[i]->aux->linfo_idx = env->subprog_info[i].linfo_idx; in jit_subprogs()
11068 func[i]->aux->tail_call_reachable = env->subprog_info[i].tail_call_reachable; in jit_subprogs()
11081 for (i = 0; i < env->subprog_cnt; i++) { in jit_subprogs()
11103 func[i]->aux->func_cnt = env->subprog_cnt; in jit_subprogs()
11105 for (i = 0; i < env->subprog_cnt; i++) { in jit_subprogs()
11109 verbose(env, "JIT doesn't support bpf-to-bpf calls\n"); in jit_subprogs()
11119 for (i = 0; i < env->subprog_cnt; i++) { in jit_subprogs()
11132 insn->off = env->insn_aux_data[i].call_imm; in jit_subprogs()
11133 subprog = find_subprog(env, i + insn->off + 1); in jit_subprogs()
11140 prog->aux->func_cnt = env->subprog_cnt; in jit_subprogs()
11156 for (i = 0; i < env->subprog_cnt; i++) { in jit_subprogs()
11172 insn->imm = env->insn_aux_data[i].call_imm; in jit_subprogs()
11178 static int fixup_call_args(struct bpf_verifier_env *env) in fixup_call_args() argument
11181 struct bpf_prog *prog = env->prog; in fixup_call_args()
11187 if (env->prog->jit_requested && !bpf_prog_is_dev_bound(env->prog->aux)) { in fixup_call_args()
11188 err = jit_subprogs(env); in fixup_call_args()
11197 if (env->subprog_cnt > 1 && env->prog->aux->tail_call_reachable) { in fixup_call_args()
11201 verbose(env, "tail_calls are not allowed in non-JITed programs with bpf-to-bpf calls\n"); in fixup_call_args()
11208 depth = get_callee_stack_depth(env, insn, i); in fixup_call_args()
11224 static int fixup_bpf_calls(struct bpf_verifier_env *env) in fixup_bpf_calls() argument
11226 struct bpf_prog *prog = env->prog; in fixup_bpf_calls()
11262 new_prog = bpf_patch_insn_data(env, i + delta, patchlet, cnt); in fixup_bpf_calls()
11268 env->prog = prog = new_prog; in fixup_bpf_calls()
11274 cnt = env->ops->gen_ld_abs(insn, insn_buf); in fixup_bpf_calls()
11276 verbose(env, "bpf verifier is misconfigured\n"); in fixup_bpf_calls()
11280 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in fixup_bpf_calls()
11286 env->prog = prog = new_prog; in fixup_bpf_calls()
11299 aux = &env->insn_aux_data[i + delta]; in fixup_bpf_calls()
11335 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf_in, cnt); in fixup_bpf_calls()
11341 env->prog = prog = new_prog; in fixup_bpf_calls()
11369 if (!allow_tail_call_in_subprogs(env)) { in fixup_bpf_calls()
11382 aux = &env->insn_aux_data[i + delta]; in fixup_bpf_calls()
11383 … if (env->bpf_capable && !expect_blinding && prog->jit_requested && !bpf_map_key_poisoned(aux) && in fixup_bpf_calls()
11394 verbose(env, "adding tail call poke descriptor failed\n"); in fixup_bpf_calls()
11413 verbose(env, "tail_call abusing map_ptr\n"); in fixup_bpf_calls()
11422 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in fixup_bpf_calls()
11428 env->prog = prog = new_prog; in fixup_bpf_calls()
11441 aux = &env->insn_aux_data[i + delta]; in fixup_bpf_calls()
11454 verbose(env, "bpf verifier is misconfigured\n"); in fixup_bpf_calls()
11458 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in fixup_bpf_calls()
11464 env->prog = prog = new_prog; in fixup_bpf_calls()
11514 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in fixup_bpf_calls()
11520 env->prog = prog = new_prog; in fixup_bpf_calls()
11526 fn = env->ops->get_func_proto(insn->imm, env->prog); in fixup_bpf_calls()
11531 … verbose(env, "kernel subsystem misconfigured func %s#%d\n", func_id_name(insn->imm), insn->imm); in fixup_bpf_calls()
11541 verbose(env, "bpf verifier is misconfigured\n"); in fixup_bpf_calls()
11547 verbose(env, "tracking tail call prog failed\n"); in fixup_bpf_calls()
11555 static void free_states(struct bpf_verifier_env *env) in free_states() argument
11560 sl = env->free_list; in free_states()
11567 env->free_list = NULL; in free_states()
11569 if (!env->explored_states) { in free_states()
11573 for (i = 0; i < state_htab_size(env); i++) { in free_states()
11574 sl = env->explored_states[i]; in free_states()
11582 env->explored_states[i] = NULL; in free_states()
11586 static int do_check_common(struct bpf_verifier_env *env, int subprog) in do_check_common() argument
11588 bool pop_log = !(env->log.level & BPF_LOG_LEVEL2); in do_check_common()
11593 env->prev_linfo = NULL; in do_check_common()
11594 env->pass_cnt++; in do_check_common()
11608 env->cur_state = state; in do_check_common()
11609 init_func_state(env, state->frame[0], BPF_MAIN_FUNC /* callsite */, 0 /* frameno */, subprog); in do_check_common()
11612 if (subprog || env->prog->type == BPF_PROG_TYPE_EXT) { in do_check_common()
11613 ret = btf_prepare_func_args(env, subprog, regs); in do_check_common()
11619 mark_reg_known_zero(env, regs, i); in do_check_common()
11621 mark_reg_unknown(env, regs, i); in do_check_common()
11627 mark_reg_known_zero(env, regs, BPF_REG_1); in do_check_common()
11628 ret = btf_check_func_arg_match(env, subprog, regs); in do_check_common()
11643 ret = do_check(env); in do_check_common()
11648 if (env->cur_state) { in do_check_common()
11649 free_verifier_state(env->cur_state, true); in do_check_common()
11650 env->cur_state = NULL; in do_check_common()
11652 while (!pop_stack(env, NULL, NULL, false)) { in do_check_common()
11656 bpf_vlog_reset(&env->log, 0); in do_check_common()
11658 free_states(env); in do_check_common()
11679 static int do_check_subprogs(struct bpf_verifier_env *env) in do_check_subprogs() argument
11681 struct bpf_prog_aux *aux = env->prog->aux; in do_check_subprogs()
11688 for (i = 1; i < env->subprog_cnt; i++) { in do_check_subprogs()
11692 env->insn_idx = env->subprog_info[i].start; in do_check_subprogs()
11693 WARN_ON_ONCE(env->insn_idx == 0); in do_check_subprogs()
11694 ret = do_check_common(env, i); in do_check_subprogs()
11697 } else if (env->log.level & BPF_LOG_LEVEL) { in do_check_subprogs()
11698 verbose(env, "Func#%d is safe for any args that match its prototype\n", i); in do_check_subprogs()
11704 static int do_check_main(struct bpf_verifier_env *env) in do_check_main() argument
11708 env->insn_idx = 0; in do_check_main()
11709 ret = do_check_common(env, 0); in do_check_main()
11711 env->prog->aux->stack_depth = env->subprog_info[0].stack_depth; in do_check_main()
11716 static void print_verification_stats(struct bpf_verifier_env *env) in print_verification_stats() argument
11720 if (env->log.level & BPF_LOG_STATS) { in print_verification_stats()
11721 …verbose(env, "verification time %lld usec\n", div_u64(env->verification_time, VERIFIER_ONETHOUSAND… in print_verification_stats()
11722 verbose(env, "stack depth "); in print_verification_stats()
11723 for (i = 0; i < env->subprog_cnt; i++) { in print_verification_stats()
11724 u32 depth = env->subprog_info[i].stack_depth; in print_verification_stats()
11726 verbose(env, "%d", depth); in print_verification_stats()
11727 if (i + 1 < env->subprog_cnt) { in print_verification_stats()
11728 verbose(env, "+"); in print_verification_stats()
11731 verbose(env, "\n"); in print_verification_stats()
11733 verbose(env, in print_verification_stats()
11736 … env->insn_processed, BPF_COMPLEXITY_LIMIT_INSNS, env->max_states_per_insn, env->total_states, in print_verification_stats()
11737 env->peak_states, env->longest_mark_read_walk); in print_verification_stats()
11740 static int check_struct_ops_btf_id(struct bpf_verifier_env *env) in check_struct_ops_btf_id() argument
11745 struct bpf_prog *prog = env->prog; in check_struct_ops_btf_id()
11750 verbose(env, "struct ops programs must have a GPL compatible license\n"); in check_struct_ops_btf_id()
11757 verbose(env, "attach_btf_id %u is not a supported struct\n", btf_id); in check_struct_ops_btf_id()
11764 verbose(env, "attach to invalid member idx %u of struct %s\n", member_idx, st_ops->name); in check_struct_ops_btf_id()
11772 …verbose(env, "attach to invalid member %s(@idx %u) of struct %s\n", mname, member_idx, st_ops->nam… in check_struct_ops_btf_id()
11779 verbose(env, "attach to unsupported member %s of struct %s\n", mname, st_ops->name); in check_struct_ops_btf_id()
11786 env->ops = st_ops->verifier_ops; in check_struct_ops_btf_id()
12062 static int check_attach_btf_id(struct bpf_verifier_env *env) in check_attach_btf_id() argument
12064 struct bpf_prog *prog = env->prog; in check_attach_btf_id()
12073 verbose(env, "Only fentry/fexit/fmod_ret and lsm programs can be sleepable\n"); in check_attach_btf_id()
12078 return check_struct_ops_btf_id(env); in check_attach_btf_id()
12085 ret = bpf_check_attach_target(&env->log, prog, tgt_prog, btf_id, &tgt_info); in check_attach_btf_id()
12095 env->ops = bpf_verifier_ops[tgt_prog->type]; in check_attach_btf_id()
12119 ret = bpf_lsm_verify_prog(&env->log, prog); in check_attach_btf_id()
12150 struct bpf_verifier_env *env; in bpf_check() local
12163 env = kzalloc(sizeof(struct bpf_verifier_env), GFP_KERNEL); in bpf_check()
12164 if (!env) { in bpf_check()
12167 log = &env->log; in bpf_check()
12170 env->insn_aux_data = vzalloc(array_size(sizeof(struct bpf_insn_aux_data), len)); in bpf_check()
12172 if (!env->insn_aux_data) { in bpf_check()
12176 env->insn_aux_data[i].orig_idx = i; in bpf_check()
12178 env->prog = *prog; in bpf_check()
12179 env->ops = bpf_verifier_ops[env->prog->type]; in bpf_check()
12206 verbose(env, "in-kernel BTF is malformed\n"); in bpf_check()
12211 env->strict_alignment = !!(attr->prog_flags & BPF_F_STRICT_ALIGNMENT); in bpf_check()
12213 env->strict_alignment = true; in bpf_check()
12216 env->strict_alignment = false; in bpf_check()
12219 env->allow_ptr_leaks = bpf_allow_ptr_leaks(); in bpf_check()
12220 env->allow_uninit_stack = bpf_allow_uninit_stack(); in bpf_check()
12221 env->allow_ptr_to_map_access = bpf_allow_ptr_to_map_access(); in bpf_check()
12222 env->bypass_spec_v1 = bpf_bypass_spec_v1(); in bpf_check()
12223 env->bypass_spec_v4 = bpf_bypass_spec_v4(); in bpf_check()
12224 env->bpf_capable = bpf_capable(); in bpf_check()
12227 env->test_state_freq = attr->prog_flags & BPF_F_TEST_STATE_FREQ; in bpf_check()
12230 …env->explored_states = kvcalloc(state_htab_size(env), sizeof(struct bpf_verifier_state_list *), GF… in bpf_check()
12232 if (!env->explored_states) { in bpf_check()
12236 ret = check_subprogs(env); in bpf_check()
12241 ret = check_btf_info(env, attr, uattr); in bpf_check()
12246 ret = check_attach_btf_id(env); in bpf_check()
12251 ret = resolve_pseudo_ldimm64(env); in bpf_check()
12256 if (bpf_prog_is_dev_bound(env->prog->aux)) { in bpf_check()
12257 ret = bpf_prog_offload_verifier_prep(env->prog); in bpf_check()
12263 ret = check_cfg(env); in bpf_check()
12268 ret = do_check_subprogs(env); in bpf_check()
12269 ret = ret ?: do_check_main(env); in bpf_check()
12271 if (ret == 0 && bpf_prog_is_dev_bound(env->prog->aux)) { in bpf_check()
12272 ret = bpf_prog_offload_finalize(env); in bpf_check()
12276 kvfree(env->explored_states); in bpf_check()
12279 ret = check_max_stack_depth(env); in bpf_check()
12285 opt_hard_wire_dead_code_branches(env); in bpf_check()
12288 ret = opt_remove_dead_code(env); in bpf_check()
12291 ret = opt_remove_nops(env); in bpf_check()
12295 sanitize_dead_code(env); in bpf_check()
12301 ret = convert_ctx_accesses(env); in bpf_check()
12305 ret = fixup_bpf_calls(env); in bpf_check()
12311 if (ret == 0 && !bpf_prog_is_dev_bound(env->prog->aux)) { in bpf_check()
12312 ret = opt_subreg_zext_lo32_rnd_hi32(env, attr); in bpf_check()
12313 env->prog->aux->verifier_zext = bpf_jit_needs_zext() ? !ret : false; in bpf_check()
12317 ret = fixup_call_args(env); in bpf_check()
12320 env->verification_time = ktime_get_ns() - start_time; in bpf_check()
12321 print_verification_stats(env); in bpf_check()
12331 if (ret == 0 && env->used_map_cnt) { in bpf_check()
12333 …env->prog->aux->used_maps = kmalloc_array(env->used_map_cnt, sizeof(env->used_maps[0]), GFP_KERNEL… in bpf_check()
12335 if (!env->prog->aux->used_maps) { in bpf_check()
12340 … memcpy(env->prog->aux->used_maps, env->used_maps, sizeof(env->used_maps[0]) * env->used_map_cnt); in bpf_check()
12341 env->prog->aux->used_map_cnt = env->used_map_cnt; in bpf_check()
12346 convert_pseudo_ld_imm64(env); in bpf_check()
12350 adjust_btf_func(env); in bpf_check()
12354 if (!env->prog->aux->used_maps) { in bpf_check()
12358 release_maps(env); in bpf_check()
12364 if (env->prog->type == BPF_PROG_TYPE_EXT) { in bpf_check()
12365 env->prog->expected_attach_type = 0; in bpf_check()
12368 *prog = env->prog; in bpf_check()
12373 vfree(env->insn_aux_data); in bpf_check()
12375 kfree(env); in bpf_check()