Lines Matching refs:state
223 static void bpf_map_key_store(struct bpf_insn_aux_data *aux, u64 state) in bpf_map_key_store() argument
227 aux->map_key_state = state | BPF_MAP_KEY_SEEN | in bpf_map_key_store()
580 const struct bpf_func_state *state) in print_verifier_state() argument
586 if (state->frameno) in print_verifier_state()
587 verbose(env, " frame%d:", state->frameno); in print_verifier_state()
589 reg = &state->regs[i]; in print_verifier_state()
667 for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) { in print_verifier_state()
673 if (state->stack[i].slot_type[j] != STACK_INVALID) in print_verifier_state()
676 state->stack[i].slot_type[j]]; in print_verifier_state()
682 print_liveness(env, state->stack[i].spilled_ptr.live); in print_verifier_state()
683 if (is_spilled_reg(&state->stack[i])) { in print_verifier_state()
684 reg = &state->stack[i].spilled_ptr; in print_verifier_state()
695 if (state->acquired_refs && state->refs[0].id) { in print_verifier_state()
696 verbose(env, " refs=%d", state->refs[0].id); in print_verifier_state()
697 for (i = 1; i < state->acquired_refs; i++) in print_verifier_state()
698 if (state->refs[i].id) in print_verifier_state()
699 verbose(env, ",%d", state->refs[i].id); in print_verifier_state()
726 static int realloc_##NAME##_state(struct bpf_func_state *state, int size, \ in COPY_STATE_FN()
729 u32 old_size = state->COUNT; \ in COPY_STATE_FN()
736 state->COUNT = slot * SIZE; \ in COPY_STATE_FN()
738 kfree(state->FIELD); \ in COPY_STATE_FN()
739 state->FIELD = NULL; \ in COPY_STATE_FN()
748 if (state->FIELD) \ in COPY_STATE_FN()
749 memcpy(new_##FIELD, state->FIELD, \ in COPY_STATE_FN()
754 state->COUNT = slot * SIZE; \ in COPY_STATE_FN()
755 kfree(state->FIELD); \ in COPY_STATE_FN()
756 state->FIELD = new_##FIELD; \ in COPY_STATE_FN()
772 static int realloc_func_state(struct bpf_func_state *state, int stack_size,
775 int err = realloc_reference_state(state, refs_size, copy_old);
778 return realloc_stack_state(state, stack_size, copy_old);
788 struct bpf_func_state *state = cur_func(env); in acquire_reference_state() local
789 int new_ofs = state->acquired_refs; in acquire_reference_state()
792 err = realloc_reference_state(state, state->acquired_refs + 1, true); in acquire_reference_state()
796 state->refs[new_ofs].id = id; in acquire_reference_state()
797 state->refs[new_ofs].insn_idx = insn_idx; in acquire_reference_state()
803 static int release_reference_state(struct bpf_func_state *state, int ptr_id) in release_reference_state() argument
807 last_idx = state->acquired_refs - 1; in release_reference_state()
808 for (i = 0; i < state->acquired_refs; i++) { in release_reference_state()
809 if (state->refs[i].id == ptr_id) { in release_reference_state()
811 memcpy(&state->refs[i], &state->refs[last_idx], in release_reference_state()
812 sizeof(*state->refs)); in release_reference_state()
813 memset(&state->refs[last_idx], 0, sizeof(*state->refs)); in release_reference_state()
814 state->acquired_refs--; in release_reference_state()
833 static void free_func_state(struct bpf_func_state *state) in free_func_state() argument
835 if (!state) in free_func_state()
837 kfree(state->refs); in free_func_state()
838 kfree(state->stack); in free_func_state()
839 kfree(state); in free_func_state()
842 static void clear_jmp_history(struct bpf_verifier_state *state) in clear_jmp_history() argument
844 kfree(state->jmp_history); in clear_jmp_history()
845 state->jmp_history = NULL; in clear_jmp_history()
846 state->jmp_history_cnt = 0; in clear_jmp_history()
849 static void free_verifier_state(struct bpf_verifier_state *state, in free_verifier_state() argument
854 for (i = 0; i <= state->curframe; i++) { in free_verifier_state()
855 free_func_state(state->frame[i]); in free_verifier_state()
856 state->frame[i] = NULL; in free_verifier_state()
858 clear_jmp_history(state); in free_verifier_state()
860 kfree(state); in free_verifier_state()
1414 struct bpf_func_state *state) in init_reg_state() argument
1416 struct bpf_reg_state *regs = state->regs; in init_reg_state()
1429 regs[BPF_REG_FP].frameno = state->frameno; in init_reg_state()
1434 struct bpf_func_state *state, in init_func_state() argument
1437 state->callsite = callsite; in init_func_state()
1438 state->frameno = frameno; in init_func_state()
1439 state->subprogno = subprogno; in init_func_state()
1440 init_reg_state(env, state); in init_func_state()
1572 const struct bpf_reg_state *state, in mark_reg_read() argument
1575 bool writes = parent == state->parent; /* Observe write marks */ in mark_reg_read()
1580 if (writes && state->live & REG_LIVE_WRITTEN) in mark_reg_read()
1608 state = parent; in mark_reg_read()
1609 parent = state->parent; in mark_reg_read()
1738 struct bpf_func_state *state = vstate->frame[vstate->curframe]; in check_reg_arg() local
1740 struct bpf_reg_state *reg, *regs = state->regs; in check_reg_arg()
2450 static void save_register_state(struct bpf_func_state *state, in save_register_state() argument
2456 copy_register_state(&state->stack[spi].spilled_ptr, reg); in save_register_state()
2458 state->stack[spi].spilled_ptr.live |= REG_LIVE_WRITTEN; in save_register_state()
2461 state->stack[spi].slot_type[i - 1] = STACK_SPILL; in save_register_state()
2465 scrub_spilled_slot(&state->stack[spi].slot_type[i - 1]); in save_register_state()
2478 struct bpf_func_state *state, in check_stack_write_fixed_off() argument
2488 err = realloc_func_state(state, round_up(slot + 1, BPF_REG_SIZE), in check_stack_write_fixed_off()
2489 state->acquired_refs, true); in check_stack_write_fixed_off()
2496 is_spilled_reg(&state->stack[spi]) && in check_stack_write_fixed_off()
2509 u8 type = state->stack[spi].slot_type[i]; in check_stack_write_fixed_off()
2534 save_register_state(state, spi, reg, size); in check_stack_write_fixed_off()
2537 state->stack[spi].spilled_ptr.id = 0; in check_stack_write_fixed_off()
2544 save_register_state(state, spi, &fake_reg, size); in check_stack_write_fixed_off()
2552 if (state != cur && reg->type == PTR_TO_STACK) { in check_stack_write_fixed_off()
2556 save_register_state(state, spi, reg, size); in check_stack_write_fixed_off()
2561 state->stack[spi].spilled_ptr.type = NOT_INIT; in check_stack_write_fixed_off()
2563 if (is_spilled_reg(&state->stack[spi])) in check_stack_write_fixed_off()
2565 scrub_spilled_slot(&state->stack[spi].slot_type[i]); in check_stack_write_fixed_off()
2576 state->stack[spi].spilled_ptr.live |= REG_LIVE_WRITTEN; in check_stack_write_fixed_off()
2590 state->stack[spi].slot_type[(slot - i) % BPF_REG_SIZE] = in check_stack_write_fixed_off()
2617 struct bpf_func_state *state, in check_stack_write_var_off() argument
2640 err = realloc_func_state(state, round_up(-min_off, BPF_REG_SIZE), in check_stack_write_var_off()
2641 state->acquired_refs, true); in check_stack_write_var_off()
2653 stype = &state->stack[spi].slot_type[slot % BPF_REG_SIZE]; in check_stack_write_var_off()
2673 state->stack[spi].spilled_ptr.type = NOT_INIT; in check_stack_write_var_off()
2719 struct bpf_func_state *state = vstate->frame[vstate->curframe]; in mark_reg_stack_read() local
2736 __mark_reg_const_zero(&state->regs[dst_regno]); in mark_reg_stack_read()
2747 state->regs[dst_regno].precise = true; in mark_reg_stack_read()
2750 mark_reg_unknown(env, state->regs, dst_regno); in mark_reg_stack_read()
2752 state->regs[dst_regno].live |= REG_LIVE_WRITTEN; in mark_reg_stack_read()
2770 struct bpf_func_state *state = vstate->frame[vstate->curframe]; in check_stack_read_fixed_off() local
2799 s32 subreg_def = state->regs[dst_regno].subreg_def; in check_stack_read_fixed_off()
2801 copy_register_state(&state->regs[dst_regno], reg); in check_stack_read_fixed_off()
2802 state->regs[dst_regno].subreg_def = subreg_def; in check_stack_read_fixed_off()
2814 mark_reg_unknown(env, state->regs, dst_regno); in check_stack_read_fixed_off()
2816 state->regs[dst_regno].live |= REG_LIVE_WRITTEN; in check_stack_read_fixed_off()
2822 copy_register_state(&state->regs[dst_regno], reg); in check_stack_read_fixed_off()
2827 state->regs[dst_regno].live |= REG_LIVE_WRITTEN; in check_stack_read_fixed_off()
2923 struct bpf_func_state *state = func(env, reg); in check_stack_read() local
2951 err = check_stack_read_fixed_off(env, state, off, size, in check_stack_read()
2980 struct bpf_func_state *state = func(env, reg); in check_stack_write() local
2985 err = check_stack_write_fixed_off(env, state, off, size, in check_stack_write()
2991 err = check_stack_write_var_off(env, state, in check_stack_write()
3058 struct bpf_func_state *state = vstate->frame[vstate->curframe]; in check_mem_region_access() local
3059 struct bpf_reg_state *reg = &state->regs[regno]; in check_mem_region_access()
3067 print_verifier_state(env, state); in check_mem_region_access()
3116 struct bpf_func_state *state = vstate->frame[vstate->curframe]; in check_map_access() local
3117 struct bpf_reg_state *reg = &state->regs[regno]; in check_map_access()
3887 struct bpf_func_state *state, in check_stack_slot_within_bounds() argument
3895 min_valid_off = -state->allocated_stack; in check_stack_slot_within_bounds()
3914 struct bpf_func_state *state = func(env, reg); in check_stack_access_within_bounds() local
3941 err = check_stack_slot_within_bounds(min_off, state, type); in check_stack_access_within_bounds()
3972 struct bpf_func_state *state; in check_mem_access() local
4076 state = func(env, reg); in check_mem_access()
4077 err = update_stack_depth(env, state, off); in check_mem_access()
4223 struct bpf_func_state *state = func(env, reg); in check_stack_range_initialized() local
4293 if (state->allocated_stack <= slot) in check_stack_range_initialized()
4295 stype = &state->stack[spi].slot_type[slot % BPF_REG_SIZE]; in check_stack_range_initialized()
4306 if (is_spilled_reg(&state->stack[spi]) && in check_stack_range_initialized()
4307 state->stack[spi].spilled_ptr.type == PTR_TO_BTF_ID) in check_stack_range_initialized()
4310 if (is_spilled_reg(&state->stack[spi]) && in check_stack_range_initialized()
4311 (state->stack[spi].spilled_ptr.type == SCALAR_VALUE || in check_stack_range_initialized()
4314 __mark_reg_unknown(env, &state->stack[spi].spilled_ptr); in check_stack_range_initialized()
4316 scrub_spilled_slot(&state->stack[spi].slot_type[j]); in check_stack_range_initialized()
4337 mark_reg_read(env, &state->stack[spi].spilled_ptr, in check_stack_range_initialized()
4338 state->stack[spi].spilled_ptr.parent, in check_stack_range_initialized()
4341 return update_stack_depth(env, state, min_off); in check_stack_range_initialized()
5223 struct bpf_func_state *state; in clear_all_pkt_pointers() local
5226 bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({ in clear_all_pkt_pointers()
5239 struct bpf_func_state *state = vstate->frame[vstate->curframe]; in mark_pkt_end() local
5240 struct bpf_reg_state *reg = &state->regs[regn]; in mark_pkt_end()
5264 struct bpf_func_state *state; in release_reference() local
5272 bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({ in release_reference()
5299 struct bpf_verifier_state *state = env->cur_state; in check_func_call() local
5305 if (state->curframe + 1 >= MAX_CALL_FRAMES) { in check_func_call()
5307 state->curframe + 2); in check_func_call()
5319 caller = state->frame[state->curframe]; in check_func_call()
5320 if (state->frame[state->curframe + 1]) { in check_func_call()
5322 state->curframe + 1); in check_func_call()
5356 state->frame[state->curframe + 1] = callee; in check_func_call()
5365 state->curframe + 1 /* frameno within this callchain */, in check_func_call()
5382 state->curframe++; in check_func_call()
5398 struct bpf_verifier_state *state = env->cur_state; in prepare_func_exit() local
5403 callee = state->frame[state->curframe]; in prepare_func_exit()
5416 state->curframe--; in prepare_func_exit()
5417 caller = state->frame[state->curframe]; in prepare_func_exit()
5435 state->frame[state->curframe + 1] = NULL; in prepare_func_exit()
5541 struct bpf_func_state *state = cur_func(env); in check_reference_leak() local
5544 for (i = 0; i < state->acquired_refs; i++) { in check_reference_leak()
5546 state->refs[i].id, state->refs[i].insn_idx); in check_reference_leak()
5548 return state->acquired_refs ? -EINVAL : 0; in check_reference_leak()
6211 struct bpf_func_state *state = vstate->frame[vstate->curframe]; in adjust_ptr_min_max_vals() local
6212 struct bpf_reg_state *regs = state->regs, *dst_reg; in adjust_ptr_min_max_vals()
7168 struct bpf_func_state *state = vstate->frame[vstate->curframe]; in adjust_reg_min_max_vals() local
7169 struct bpf_reg_state *regs = state->regs, *dst_reg, *src_reg; in adjust_reg_min_max_vals()
7237 print_verifier_state(env, state); in adjust_reg_min_max_vals()
7242 print_verifier_state(env, state); in adjust_reg_min_max_vals()
7428 struct bpf_func_state *state; in find_good_pkt_pointers() local
7495 bpf_for_each_reg_in_vstate(vstate, state, reg, ({ in find_good_pkt_pointers()
7956 static void mark_ptr_or_null_reg(struct bpf_func_state *state, in mark_ptr_or_null_reg() argument
8028 struct bpf_func_state *state = vstate->frame[vstate->curframe]; in mark_ptr_or_null_regs() local
8029 struct bpf_reg_state *regs = state->regs, *reg; in mark_ptr_or_null_regs()
8038 WARN_ON_ONCE(release_reference_state(state, id)); in mark_ptr_or_null_regs()
8040 bpf_for_each_reg_in_vstate(vstate, state, reg, ({ in mark_ptr_or_null_regs()
8041 mark_ptr_or_null_reg(state, reg, id, is_null); in mark_ptr_or_null_regs()
8153 struct bpf_func_state *state; in find_equal_scalars() local
8156 bpf_for_each_reg_in_vstate(vstate, state, reg, ({ in find_equal_scalars()
8699 struct bpf_func_state *state = cur->frame[cur->curframe]; in explored_state() local
8701 return &env->explored_states[(idx ^ state->callsite) % state_htab_size(env)]; in explored_state()
9332 if (sl->state.branches) in clean_live_states()
9334 if (sl->state.insn_idx != insn || in clean_live_states()
9335 sl->state.curframe != cur->curframe) in clean_live_states()
9338 if (sl->state.frame[i]->callsite != cur->frame[i]->callsite) in clean_live_states()
9340 clean_verifier_state(env, &sl->state); in clean_live_states()
9653 struct bpf_func_state *state, *parent; in propagate_liveness() local
9665 state = vstate->frame[frame]; in propagate_liveness()
9667 state_reg = state->regs; in propagate_liveness()
9679 for (i = 0; i < state->allocated_stack / BPF_REG_SIZE && in propagate_liveness()
9682 state_reg = &state->stack[i].spilled_ptr; in propagate_liveness()
9699 struct bpf_func_state *state; in propagate_precision() local
9703 state = old->frame[fr]; in propagate_precision()
9704 state_reg = state->regs; in propagate_precision()
9717 for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) { in propagate_precision()
9718 if (!is_spilled_reg(&state->stack[i])) in propagate_precision()
9720 state_reg = &state->stack[i].spilled_ptr; in propagate_precision()
9789 if (sl->state.insn_idx != insn_idx) in is_state_visited()
9791 if (sl->state.branches) { in is_state_visited()
9792 if (states_maybe_looping(&sl->state, cur) && in is_state_visited()
9793 states_equal(env, &sl->state, cur)) { in is_state_visited()
9815 if (states_equal(env, &sl->state, cur)) { in is_state_visited()
9827 err = propagate_liveness(env, &sl->state, cur); in is_state_visited()
9835 err = err ? : propagate_precision(env, &sl->state); in is_state_visited()
9859 if (sl->state.frame[0]->regs[0].live & REG_LIVE_DONE) { in is_state_visited()
9860 u32 br = sl->state.branches; in is_state_visited()
9865 free_verifier_state(&sl->state, false); in is_state_visited()
9915 new = &new_sl->state; in is_state_visited()
10006 struct bpf_verifier_state *state = env->cur_state; in do_check() local
10067 print_verifier_state(env, state->frame[state->curframe]); in do_check()
10263 if (state->curframe) { in do_check()
11963 free_verifier_state(&sl->state, false); in free_states()
11977 free_verifier_state(&sl->state, false); in free_states()
11988 struct bpf_verifier_state *state; in do_check_common() local
11995 state = kzalloc(sizeof(struct bpf_verifier_state), GFP_KERNEL); in do_check_common()
11996 if (!state) in do_check_common()
11998 state->curframe = 0; in do_check_common()
11999 state->speculative = false; in do_check_common()
12000 state->branches = 1; in do_check_common()
12001 state->frame[0] = kzalloc(sizeof(struct bpf_func_state), GFP_KERNEL); in do_check_common()
12002 if (!state->frame[0]) { in do_check_common()
12003 kfree(state); in do_check_common()
12006 env->cur_state = state; in do_check_common()
12007 init_func_state(env, state->frame[0], in do_check_common()
12012 state->first_insn_idx = env->subprog_info[subprog].start; in do_check_common()
12013 state->last_insn_idx = -1; in do_check_common()
12015 regs = state->frame[state->curframe]->regs; in do_check_common()